From f7f13e0742a999c991ea3d2871dd99a042736c28 Mon Sep 17 00:00:00 2001 From: Johannes Kron Date: Wed, 12 Dec 2018 11:17:43 +0100 Subject: [PATCH] Add end-to-end test for ColorSpace information MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:8651 Change-Id: Ib7f6162c8dd41868b7d7acca8a6292b2d911c520 Reviewed-on: https://webrtc-review.googlesource.com/c/113941 Commit-Queue: Johannes Kron Reviewed-by: Niels Moller Reviewed-by: Sebastian Jansson Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#25992} --- api/video/video_frame.h | 4 + test/call_test.cc | 11 ++- test/constants.cc | 1 + test/constants.h | 1 + test/frame_generator.h | 3 + test/frame_generator_capturer.cc | 9 +++ test/frame_generator_capturer.h | 2 + video/end_to_end_tests/codec_tests.cc | 107 +++++++++++++++++++++----- 8 files changed, 112 insertions(+), 26 deletions(-) diff --git a/api/video/video_frame.h b/api/video/video_frame.h index 2c5d081ba6..cc61fee5b1 100644 --- a/api/video/video_frame.h +++ b/api/video/video_frame.h @@ -118,6 +118,10 @@ class RTC_EXPORT VideoFrame { const ColorSpace* color_space() const { return color_space_ ? &*color_space_ : nullptr; } + void set_color_space(ColorSpace* color_space) { + color_space_ = + color_space ? absl::make_optional(*color_space) : absl::nullopt; + } // Get render time in milliseconds. // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). diff --git a/test/call_test.cc b/test/call_test.cc index 6e25b85a56..5d2bbcadb2 100644 --- a/test/call_test.cc +++ b/test/call_test.cc @@ -31,10 +31,6 @@ namespace webrtc { namespace test { -namespace { -const int kVideoRotationRtpExtensionId = 4; -} - CallTest::CallTest() : clock_(Clock::GetRealTimeClock()), send_event_log_(RtcEventLog::CreateNull()), @@ -240,6 +236,7 @@ void CallTest::CreateVideoSendConfig(VideoSendStream::Config* video_config, bitrate_allocator_factory_.get(); video_config->rtp.payload_name = "FAKE"; video_config->rtp.payload_type = kFakeVideoSendPayloadType; + video_config->rtp.extmap_allow_mixed = true; video_config->rtp.extensions.push_back( RtpExtension(RtpExtension::kTransportSequenceNumberUri, kTransportSequenceNumberExtensionId)); @@ -255,8 +252,10 @@ void CallTest::CreateVideoSendConfig(VideoSendStream::Config* video_config, for (size_t i = 0; i < num_video_streams; ++i) video_config->rtp.ssrcs.push_back(kVideoSendSsrcs[num_used_ssrcs + i]); - video_config->rtp.extensions.push_back(RtpExtension( - RtpExtension::kVideoRotationUri, kVideoRotationRtpExtensionId)); + video_config->rtp.extensions.push_back( + RtpExtension(RtpExtension::kVideoRotationUri, kVideoRotationExtensionId)); + video_config->rtp.extensions.push_back( + RtpExtension(RtpExtension::kColorSpaceUri, kColorSpaceExtensionId)); } void CallTest::CreateAudioAndFecSendConfigs(size_t num_audio_streams, diff --git a/test/constants.cc b/test/constants.cc index 12ae8e8449..4f33d25291 100644 --- a/test/constants.cc +++ b/test/constants.cc @@ -21,6 +21,7 @@ const int kVideoRotationExtensionId = 9; const int kVideoContentTypeExtensionId = 10; const int kVideoTimingExtensionId = 11; const int kGenericDescriptorExtensionId = 12; +const int kColorSpaceExtensionId = 13; } // namespace test } // namespace webrtc diff --git a/test/constants.h b/test/constants.h index 3cee828b20..b1b87d6f63 100644 --- a/test/constants.h +++ b/test/constants.h @@ -19,5 +19,6 @@ extern const int kVideoRotationExtensionId; extern const int kVideoContentTypeExtensionId; extern const int kVideoTimingExtensionId; extern const int kGenericDescriptorExtensionId; +extern const int kColorSpaceExtensionId; } // namespace test } // namespace webrtc diff --git a/test/frame_generator.h b/test/frame_generator.h index a92c3b4b5a..45cc9f0f13 100644 --- a/test/frame_generator.h +++ b/test/frame_generator.h @@ -50,6 +50,9 @@ class FrameGenerator { virtual ~FrameGenerator() = default; // Returns video frame that remains valid until next call. + // TODO(kron): Return rtc::scoped_refptr instead of + // VideoFrame* and populate the VideoFrame struct in FrameGeneratorCapturer + // using VideoFrame::Builder. virtual VideoFrame* NextFrame() = 0; // Change the capture resolution. diff --git a/test/frame_generator_capturer.cc b/test/frame_generator_capturer.cc index 872918727d..6ca1ddb8bd 100644 --- a/test/frame_generator_capturer.cc +++ b/test/frame_generator_capturer.cc @@ -147,6 +147,12 @@ void FrameGeneratorCapturer::SetFakeRotation(VideoRotation rotation) { fake_rotation_ = rotation; } +void FrameGeneratorCapturer::SetFakeColorSpace( + absl::optional color_space) { + rtc::CritScope cs(&lock_); + fake_color_space_ = color_space; +} + bool FrameGeneratorCapturer::Init() { // This check is added because frame_generator_ might be file based and should // not crash because a file moved. @@ -173,6 +179,9 @@ void FrameGeneratorCapturer::InsertFrame() { frame->set_timestamp_us(clock_->TimeInMicroseconds()); frame->set_ntp_time_ms(clock_->CurrentNtpInMilliseconds()); frame->set_rotation(fake_rotation_); + if (fake_color_space_) { + frame->set_color_space(&fake_color_space_.value()); + } if (first_frame_capture_time_ == -1) { first_frame_capture_time_ = frame->ntp_time_ms(); } diff --git a/test/frame_generator_capturer.h b/test/frame_generator_capturer.h index 6b2ed69bf0..a5e60a5eb5 100644 --- a/test/frame_generator_capturer.h +++ b/test/frame_generator_capturer.h @@ -74,6 +74,7 @@ class FrameGeneratorCapturer : public TestVideoCapturer { void ForceFrame(); void SetFakeRotation(VideoRotation rotation); + void SetFakeColorSpace(absl::optional color_space); int64_t first_frame_capture_time() const { return first_frame_capture_time_; } @@ -101,6 +102,7 @@ class FrameGeneratorCapturer : public TestVideoCapturer { int target_capture_fps_ RTC_GUARDED_BY(&lock_); absl::optional wanted_fps_ RTC_GUARDED_BY(&lock_); VideoRotation fake_rotation_ = kVideoRotation_0; + absl::optional fake_color_space_ RTC_GUARDED_BY(&lock_); int64_t first_frame_capture_time_; // Must be the last field, so it will be deconstructed first as tasks diff --git a/video/end_to_end_tests/codec_tests.cc b/video/end_to_end_tests/codec_tests.cc index 67ef611a70..4482a09026 100644 --- a/video/end_to_end_tests/codec_tests.cc +++ b/video/end_to_end_tests/codec_tests.cc @@ -8,7 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "absl/types/optional.h" #include "api/test/video/function_video_encoder_factory.h" +#include "api/video/color_space.h" +#include "api/video/video_rotation.h" #include "media/engine/internaldecoderfactory.h" #include "media/engine/internalencoderfactory.h" #include "modules/video_coding/codecs/h264/include/h264.h" @@ -22,6 +25,35 @@ #include "test/gtest.h" namespace webrtc { +namespace { +HdrMetadata CreateTestHdrMetadata() { + // Random but reasonable HDR metadata. + HdrMetadata hdr_metadata; + hdr_metadata.mastering_metadata.luminance_max = 2000.0; + hdr_metadata.mastering_metadata.luminance_min = 2.0001; + hdr_metadata.mastering_metadata.primary_r.x = 0.3003; + hdr_metadata.mastering_metadata.primary_r.y = 0.4004; + hdr_metadata.mastering_metadata.primary_g.x = 0.3201; + hdr_metadata.mastering_metadata.primary_g.y = 0.4604; + hdr_metadata.mastering_metadata.primary_b.x = 0.3409; + hdr_metadata.mastering_metadata.primary_b.y = 0.4907; + hdr_metadata.mastering_metadata.white_point.x = 0.4103; + hdr_metadata.mastering_metadata.white_point.y = 0.4806; + hdr_metadata.max_content_light_level = 2345; + hdr_metadata.max_frame_average_light_level = 1789; + return hdr_metadata; +} + +ColorSpace CreateTestColorSpace(bool with_hdr_metadata) { + HdrMetadata hdr_metadata = CreateTestHdrMetadata(); + return ColorSpace( + ColorSpace::PrimaryID::kBT709, ColorSpace::TransferID::kGAMMA22, + ColorSpace::MatrixID::kSMPTE2085, ColorSpace::RangeID::kFull, + ColorSpace::ChromaSiting::kCollocated, + ColorSpace::ChromaSiting::kCollocated, + with_hdr_metadata ? &hdr_metadata : nullptr); +} +} // namespace class CodecEndToEndTest : public test::CallTest, public testing::WithParamInterface { @@ -37,6 +69,7 @@ class CodecObserver : public test::EndToEndTest, public: CodecObserver(int no_frames_to_wait_for, VideoRotation rotation_to_test, + absl::optional color_space_to_test, const std::string& payload_name, VideoEncoderFactory* encoder_factory, VideoDecoderFactory* decoder_factory) @@ -45,6 +78,7 @@ class CodecObserver : public test::EndToEndTest, // https://bugs.webrtc.org/6830 no_frames_to_wait_for_(no_frames_to_wait_for), expected_rotation_(rotation_to_test), + expected_color_space_(color_space_to_test), payload_name_(payload_name), encoder_factory_(encoder_factory), decoder_factory_(decoder_factory), @@ -75,6 +109,14 @@ class CodecObserver : public test::EndToEndTest, void OnFrame(const VideoFrame& video_frame) override { EXPECT_EQ(expected_rotation_, video_frame.rotation()); + // Test only if explicit color space has been specified since otherwise the + // color space is codec dependent. + if (expected_color_space_) { + EXPECT_EQ(expected_color_space_, + video_frame.color_space() + ? absl::make_optional(*video_frame.color_space()) + : absl::nullopt); + } if (++frame_counter_ == no_frames_to_wait_for_) observation_complete_.Set(); } @@ -82,11 +124,13 @@ class CodecObserver : public test::EndToEndTest, void OnFrameGeneratorCapturerCreated( test::FrameGeneratorCapturer* frame_generator_capturer) override { frame_generator_capturer->SetFakeRotation(expected_rotation_); + frame_generator_capturer->SetFakeColorSpace(expected_color_space_); } private: int no_frames_to_wait_for_; VideoRotation expected_rotation_; + absl::optional expected_color_space_; std::string payload_name_; VideoEncoderFactory* encoder_factory_; VideoDecoderFactory* decoder_factory_; @@ -103,8 +147,8 @@ TEST_P(CodecEndToEndTest, SendsAndReceivesVP8) { []() { return VP8Encoder::Create(); }); test::FunctionVideoDecoderFactory decoder_factory( []() { return VP8Decoder::Create(); }); - CodecObserver test(5, kVideoRotation_0, "VP8", &encoder_factory, - &decoder_factory); + CodecObserver test(5, kVideoRotation_0, absl::nullopt, "VP8", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -113,8 +157,8 @@ TEST_P(CodecEndToEndTest, SendsAndReceivesVP8Rotation90) { []() { return VP8Encoder::Create(); }); test::FunctionVideoDecoderFactory decoder_factory( []() { return VP8Decoder::Create(); }); - CodecObserver test(5, kVideoRotation_90, "VP8", &encoder_factory, - &decoder_factory); + CodecObserver test(5, kVideoRotation_90, absl::nullopt, "VP8", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -124,8 +168,8 @@ TEST_P(CodecEndToEndTest, SendsAndReceivesVP9) { []() { return VP9Encoder::Create(); }); test::FunctionVideoDecoderFactory decoder_factory( []() { return VP9Decoder::Create(); }); - CodecObserver test(500, kVideoRotation_0, "VP9", &encoder_factory, - &decoder_factory); + CodecObserver test(500, kVideoRotation_0, absl::nullopt, "VP9", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -134,8 +178,31 @@ TEST_P(CodecEndToEndTest, SendsAndReceivesVP9VideoRotation90) { []() { return VP9Encoder::Create(); }); test::FunctionVideoDecoderFactory decoder_factory( []() { return VP9Decoder::Create(); }); - CodecObserver test(5, kVideoRotation_90, "VP9", &encoder_factory, - &decoder_factory); + CodecObserver test(5, kVideoRotation_90, absl::nullopt, "VP9", + &encoder_factory, &decoder_factory); + RunBaseTest(&test); +} + +TEST_P(CodecEndToEndTest, SendsAndReceivesVP9ExplicitColorSpace) { + test::FunctionVideoEncoderFactory encoder_factory( + []() { return VP9Encoder::Create(); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return VP9Decoder::Create(); }); + CodecObserver test(5, kVideoRotation_90, + CreateTestColorSpace(/*with_hdr_metadata=*/false), "VP9", + &encoder_factory, &decoder_factory); + RunBaseTest(&test); +} + +TEST_P(CodecEndToEndTest, + SendsAndReceivesVP9ExplicitColorSpaceWithHdrMetadata) { + test::FunctionVideoEncoderFactory encoder_factory( + []() { return VP9Encoder::Create(); }); + test::FunctionVideoDecoderFactory decoder_factory( + []() { return VP9Decoder::Create(); }); + CodecObserver test(5, kVideoRotation_90, + CreateTestColorSpace(/*with_hdr_metadata=*/true), "VP9", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -154,8 +221,8 @@ TEST_P(CodecEndToEndTest, SendsAndReceivesMultiplex) { &internal_decoder_factory, SdpVideoFormat(cricket::kVp9CodecName)); }); - CodecObserver test(5, kVideoRotation_0, "multiplex", &encoder_factory, - &decoder_factory); + CodecObserver test(5, kVideoRotation_0, absl::nullopt, "multiplex", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -172,8 +239,8 @@ TEST_P(CodecEndToEndTest, SendsAndReceivesMultiplexVideoRotation90) { return absl::make_unique( &internal_decoder_factory, SdpVideoFormat(cricket::kVp9CodecName)); }); - CodecObserver test(5, kVideoRotation_90, "multiplex", &encoder_factory, - &decoder_factory); + CodecObserver test(5, kVideoRotation_90, absl::nullopt, "multiplex", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -200,8 +267,8 @@ TEST_P(EndToEndTestH264, SendsAndReceivesH264) { []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); test::FunctionVideoDecoderFactory decoder_factory( []() { return H264Decoder::Create(); }); - CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory, - &decoder_factory); + CodecObserver test(500, kVideoRotation_0, absl::nullopt, "H264", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -210,8 +277,8 @@ TEST_P(EndToEndTestH264, SendsAndReceivesH264VideoRotation90) { []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); test::FunctionVideoDecoderFactory decoder_factory( []() { return H264Decoder::Create(); }); - CodecObserver test(5, kVideoRotation_90, "H264", &encoder_factory, - &decoder_factory); + CodecObserver test(5, kVideoRotation_90, absl::nullopt, "H264", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -222,8 +289,8 @@ TEST_P(EndToEndTestH264, SendsAndReceivesH264PacketizationMode0) { [codec]() { return H264Encoder::Create(codec); }); test::FunctionVideoDecoderFactory decoder_factory( []() { return H264Decoder::Create(); }); - CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory, - &decoder_factory); + CodecObserver test(500, kVideoRotation_0, absl::nullopt, "H264", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } @@ -234,8 +301,8 @@ TEST_P(EndToEndTestH264, SendsAndReceivesH264PacketizationMode1) { [codec]() { return H264Encoder::Create(codec); }); test::FunctionVideoDecoderFactory decoder_factory( []() { return H264Decoder::Create(); }); - CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory, - &decoder_factory); + CodecObserver test(500, kVideoRotation_0, absl::nullopt, "H264", + &encoder_factory, &decoder_factory); RunBaseTest(&test); } #endif // defined(WEBRTC_USE_H264)