From 0a01ffcd3fd4c05c46513091b112f05ec90e9ee8 Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Fri, 22 Dec 2023 09:40:56 +0100 Subject: [PATCH] Add SVC support to the video codec tester MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:14852 Change-Id: Iaa060fea396b8ec317d8f20d0c1bdad21bf739db Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/331502 Commit-Queue: Sergey Silkin Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/main@{#41440} --- test/video_codec_tester.cc | 86 ++++++++++++++++++++++++++++- test/video_codec_tester_unittest.cc | 71 +++++++++++++++++++----- 2 files changed, 141 insertions(+), 16 deletions(-) diff --git a/test/video_codec_tester.cc b/test/video_codec_tester.cc index d8091c87ba..dff511c1bf 100644 --- a/test/video_codec_tester.cc +++ b/test/video_codec_tester.cc @@ -944,13 +944,54 @@ class Encoder : public EncodedImageCallback { void Flush() { task_queue_.PostTaskAndWait([this] { encoder_->Release(); }); + if (last_superframe_) { + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(last_superframe_->scalability_mode); + for (int sidx = *last_superframe_->encoded_frame.SpatialIndex() + 1; + sidx < num_spatial_layers; ++sidx) { + last_superframe_->encoded_frame.SetSpatialIndex(sidx); + DeliverEncodedFrame(last_superframe_->encoded_frame); + } + last_superframe_.reset(); + } } private: + struct Superframe { + EncodedImage encoded_frame; + rtc::scoped_refptr encoded_data; + ScalabilityMode scalability_mode; + }; + Result OnEncodedImage(const EncodedImage& encoded_frame, const CodecSpecificInfo* codec_specific_info) override { analyzer_->FinishEncode(encoded_frame); + if (last_superframe_ && last_superframe_->encoded_frame.RtpTimestamp() != + encoded_frame.RtpTimestamp()) { + // New temporal unit. We have frame of previous temporal unit (TU) stored + // which means that the previous TU used spatial prediction. If encoder + // dropped a frame of layer X in the previous TU, mark the stored frame + // as a frame belonging to layer >X and deliver it such that decoders of + // layer >X receive encoded lower layers. + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(last_superframe_->scalability_mode); + for (int sidx = *last_superframe_->encoded_frame.SpatialIndex() + 1; + sidx < num_spatial_layers; ++sidx) { + last_superframe_->encoded_frame.SetSpatialIndex(sidx); + DeliverEncodedFrame(last_superframe_->encoded_frame); + } + last_superframe_.reset(); + } + + const EncodedImage& superframe = + MakeSuperFrame(encoded_frame, codec_specific_info); + DeliverEncodedFrame(superframe); + + return Result(Result::Error::OK); + } + + void DeliverEncodedFrame(const EncodedImage& encoded_frame) { { MutexLock lock(&mutex_); auto it = callbacks_.find(encoded_frame.RtpTimestamp()); @@ -962,8 +1003,6 @@ class Encoder : public EncodedImageCallback { if (ivf_writer_ != nullptr) { ivf_writer_->Write(encoded_frame); } - - return Result(Result::Error::OK); } void Configure(const EncodingSettings& es) { @@ -1081,6 +1120,48 @@ class Encoder : public EncodedImageCallback { return true; } + static bool IsSvc(const EncodedImage& encoded_frame, + const CodecSpecificInfo* codec_specific_info) { + ScalabilityMode scalability_mode = *codec_specific_info->scalability_mode; + return (kFullSvcScalabilityModes.count(scalability_mode) || + (kKeySvcScalabilityModes.count(scalability_mode) && + encoded_frame.FrameType() == VideoFrameType::kVideoFrameKey)); + } + + const EncodedImage& MakeSuperFrame( + const EncodedImage& encoded_frame, + const CodecSpecificInfo* codec_specific_info) { + if (last_superframe_) { + // Append to base spatial layer frame(s). + RTC_CHECK_EQ(*encoded_frame.SpatialIndex(), + *last_superframe_->encoded_frame.SpatialIndex() + 1) + << "Inter-layer frame drops are not supported."; + size_t current_size = last_superframe_->encoded_data->size(); + last_superframe_->encoded_data->Realloc(current_size + + encoded_frame.size()); + memcpy(last_superframe_->encoded_data->data() + current_size, + encoded_frame.data(), encoded_frame.size()); + last_superframe_->encoded_frame.SetEncodedData( + last_superframe_->encoded_data); + last_superframe_->encoded_frame.SetSpatialIndex( + encoded_frame.SpatialIndex()); + return last_superframe_->encoded_frame; + } + + if (IsSvc(encoded_frame, codec_specific_info)) { + last_superframe_ = Superframe{ + .encoded_frame = EncodedImage(encoded_frame), + .encoded_data = EncodedImageBuffer::Create(encoded_frame.data(), + encoded_frame.size()), + .scalability_mode = *codec_specific_info->scalability_mode}; + last_superframe_->encoded_frame.SetEncodedData( + last_superframe_->encoded_data); + return last_superframe_->encoded_frame; + } + + return encoded_frame; + } + VideoEncoderFactory* const encoder_factory_; std::unique_ptr encoder_; VideoCodecAnalyzer* const analyzer_; @@ -1092,6 +1173,7 @@ class Encoder : public EncodedImageCallback { std::unique_ptr ivf_writer_; std::map sidx_ RTC_GUARDED_BY(mutex_); std::map callbacks_ RTC_GUARDED_BY(mutex_); + absl::optional last_superframe_; Mutex mutex_; }; diff --git a/test/video_codec_tester_unittest.cc b/test/video_codec_tester_unittest.cc index abd2f957ed..df5dca90a2 100644 --- a/test/video_codec_tester_unittest.cc +++ b/test/video_codec_tester_unittest.cc @@ -293,7 +293,7 @@ class MockCodedVideoSource : public CodedVideoSource { TEST_F(VideoCodecTesterTest, Slice) { std::unique_ptr stats = - RunEncodeDecodeTest("VP8", ScalabilityMode::kL2T2, + RunEncodeDecodeTest("VP9", ScalabilityMode::kL2T2, {{{.timestamp_rtp = 0, .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, .frame_size = DataSize::Bytes(1)}, @@ -307,11 +307,13 @@ TEST_F(VideoCodecTesterTest, Slice) { EXPECT_THAT(slice, ElementsAre(Field(&Frame::frame_size, DataSize::Bytes(1)), Field(&Frame::frame_size, DataSize::Bytes(2)), - Field(&Frame::frame_size, DataSize::Bytes(3)))); + Field(&Frame::frame_size, DataSize::Bytes(3)), + Field(&Frame::frame_size, DataSize::Bytes(0)))); slice = stats->Slice({.min_timestamp_rtp = 1}, /*merge=*/false); EXPECT_THAT(slice, - ElementsAre(Field(&Frame::frame_size, DataSize::Bytes(3)))); + ElementsAre(Field(&Frame::frame_size, DataSize::Bytes(3)), + Field(&Frame::frame_size, DataSize::Bytes(0)))); slice = stats->Slice({.max_timestamp_rtp = 0}, /*merge=*/false); EXPECT_THAT(slice, @@ -534,17 +536,58 @@ TEST_P(VideoCodecTesterTestScalability, EncodeDecode) { INSTANTIATE_TEST_SUITE_P( All, VideoCodecTesterTestScalability, - Values(ScalabilityTestParameters{ - .codec_type = "VP8", - .scalability_mode = ScalabilityMode::kS2T1, - .encoded_frame_sizes = {{{0, DataSize::Bytes(1)}, - {1, DataSize::Bytes(2)}}, - {{0, DataSize::Bytes(3)}, - // Emulate frame drop. - {1, DataSize::Bytes(0)}}}, - .expected_decode_frame_sizes = {DataSize::Bytes(1), DataSize::Bytes(2), - DataSize::Bytes(3)}, - })); + Values( + ScalabilityTestParameters{ + .codec_type = "VP8", + .scalability_mode = ScalabilityMode::kS2T1, + .encoded_frame_sizes = {{{0, DataSize::Bytes(1)}, + {1, DataSize::Bytes(2)}}, + {{0, DataSize::Bytes(4)}, + // Emulate frame drop. + {1, DataSize::Bytes(0)}}}, + .expected_decode_frame_sizes = {DataSize::Bytes(1), + DataSize::Bytes(2), + DataSize::Bytes(4)}, + }, + ScalabilityTestParameters{ + .codec_type = "VP9", + .scalability_mode = ScalabilityMode::kL2T1, + .encoded_frame_sizes = + {{{0, DataSize::Bytes(1)}, {1, DataSize::Bytes(2)}}, + {{0, DataSize::Bytes(4)}, {1, DataSize::Bytes(8)}}, + {{0, DataSize::Bytes(16)}, + // Emulate frame drop. + {1, DataSize::Bytes(0)}}}, + .expected_decode_frame_sizes = + {DataSize::Bytes(1), DataSize::Bytes(3), DataSize::Bytes(4), + DataSize::Bytes(12), DataSize::Bytes(16), DataSize::Bytes(16)}, + }, + ScalabilityTestParameters{ + .codec_type = "VP9", + .scalability_mode = ScalabilityMode::kL2T1_KEY, + .encoded_frame_sizes = + {{{0, DataSize::Bytes(1)}, {1, DataSize::Bytes(2)}}, + {{0, DataSize::Bytes(4)}, {1, DataSize::Bytes(8)}}, + {{0, DataSize::Bytes(16)}, + // Emulate frame drop. + {1, DataSize::Bytes(0)}}}, + .expected_decode_frame_sizes = + {DataSize::Bytes(1), DataSize::Bytes(3), DataSize::Bytes(4), + DataSize::Bytes(8), DataSize::Bytes(16)}, + }, + ScalabilityTestParameters{ + .codec_type = "VP9", + .scalability_mode = ScalabilityMode::kS2T1, + .encoded_frame_sizes = + {{{0, DataSize::Bytes(1)}, {1, DataSize::Bytes(2)}}, + {{0, DataSize::Bytes(4)}, {1, DataSize::Bytes(8)}}, + {{0, DataSize::Bytes(16)}, + // Emulate frame drop. + {1, DataSize::Bytes(0)}}}, + .expected_decode_frame_sizes = + {DataSize::Bytes(1), DataSize::Bytes(2), DataSize::Bytes(4), + DataSize::Bytes(8), DataSize::Bytes(16)}, + })); class VideoCodecTesterTestPacing : public ::testing::TestWithParam> {