From bc4796af945a38c2e734091ac6ac410d76482aac Mon Sep 17 00:00:00 2001 From: Byoungchan Lee Date: Wed, 21 Sep 2022 10:28:51 +0900 Subject: [PATCH] Add the dependency descriptor for H.264 temporal scalability And validate it using svc_e2e_tests. Bug: webrtc:13961 Change-Id: Ie7edcf5a0684f46e4d26155b77cebbebbd46d21f Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/269541 Reviewed-by: Philip Eliasson Commit-Queue: Daniel.L (Byoungchan) Lee Reviewed-by: Florent Castelli Cr-Commit-Position: refs/heads/main@{#38153} --- modules/video_coding/BUILD.gn | 2 + .../codecs/h264/h264_encoder_impl.cc | 53 +++++++++++++++++++ .../codecs/h264/h264_encoder_impl.h | 2 + pc/test/svc_e2e_tests.cc | 9 ++-- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 0a3af1b8f2..fd5f43ec22 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -524,6 +524,8 @@ rtc_library("webrtc_h264") { "../../rtc_base/system:rtc_export", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", + "svc:scalability_structures", + "svc:scalable_video_controller", "//third_party/libyuv", ] absl_deps = [ diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/modules/video_coding/codecs/h264/h264_encoder_impl.cc index f6d52c6539..fc3fd195fb 100644 --- a/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -22,6 +22,7 @@ #include "absl/strings/match.h" #include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/video_coding/svc/create_scalability_structure.h" #include "modules/video_coding/utility/simulcast_rate_allocator.h" #include "modules/video_coding/utility/simulcast_utility.h" #include "rtc_base/checks.h" @@ -157,6 +158,7 @@ H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec) encoders_.reserve(kMaxSimulcastStreams); configurations_.reserve(kMaxSimulcastStreams); tl0sync_limit_.reserve(kMaxSimulcastStreams); + svc_controllers_.reserve(kMaxSimulcastStreams); } H264EncoderImpl::~H264EncoderImpl() { @@ -196,6 +198,7 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst, encoded_images_.resize(number_of_streams); encoders_.resize(number_of_streams); pictures_.resize(number_of_streams); + svc_controllers_.resize(number_of_streams); configurations_.resize(number_of_streams); tl0sync_limit_.resize(number_of_streams); @@ -281,6 +284,32 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst, encoded_images_[i].set_size(0); tl0sync_limit_[i] = configurations_[i].num_temporal_layers; + absl::optional scalability_mode; + switch (configurations_[i].num_temporal_layers) { + case 0: + break; + case 1: + scalability_mode = ScalabilityMode::kL1T1; + break; + case 2: + scalability_mode = ScalabilityMode::kL1T2; + break; + case 3: + scalability_mode = ScalabilityMode::kL1T3; + break; + default: + RTC_DCHECK_NOTREACHED(); + } + if (scalability_mode.has_value()) { + svc_controllers_[i] = + CreateScalabilityStructure(scalability_mode.value()); + if (svc_controllers_[i] == nullptr) { + RTC_LOG(LS_ERROR) << "Failed to create scalability structure"; + Release(); + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + } } SimulcastRateAllocator init_allocator(codec_); @@ -305,6 +334,7 @@ int32_t H264EncoderImpl::Release() { encoded_images_.clear(); pictures_.clear(); tl0sync_limit_.clear(); + svc_controllers_.clear(); return WEBRTC_VIDEO_CODEC_OK; } @@ -467,6 +497,12 @@ int32_t H264EncoderImpl::Encode( SFrameBSInfo info; memset(&info, 0, sizeof(SFrameBSInfo)); + std::vector layer_frames; + if (svc_controllers_[i]) { + layer_frames = svc_controllers_[i]->NextFrameConfig(send_key_frame); + RTC_CHECK_EQ(layer_frames.size(), 1); + } + // Encode! int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info); if (enc_ret != 0) { @@ -510,6 +546,15 @@ int32_t H264EncoderImpl::Encode( codec_specific.codecSpecific.H264.temporal_idx = tid; codec_specific.codecSpecific.H264.base_layer_sync = tid > 0 && tid < tl0sync_limit_[i]; + if (svc_controllers_[i]) { + if (layer_frames[0].TemporalId() != tid) { + RTC_LOG(LS_WARNING) + << "Encoder produced a frame for layer S" << (i + 1) << "T" + << tid + 1 << " that wasn't requested."; + continue; + } + encoded_images_[i].SetTemporalIndex(tid); + } if (codec_specific.codecSpecific.H264.base_layer_sync) { tl0sync_limit_[i] = tid; } @@ -517,6 +562,14 @@ int32_t H264EncoderImpl::Encode( tl0sync_limit_[i] = configurations_[i].num_temporal_layers; } } + if (svc_controllers_[i]) { + codec_specific.generic_frame_info = + svc_controllers_[i]->OnEncodeDone(layer_frames[0]); + if (send_key_frame && codec_specific.generic_frame_info.has_value()) { + codec_specific.template_structure = + svc_controllers_[i]->DependencyStructure(); + } + } encoded_image_callback_->OnEncodedImage(encoded_images_[i], &codec_specific); } diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.h b/modules/video_coding/codecs/h264/h264_encoder_impl.h index 1163464421..780781be23 100644 --- a/modules/video_coding/codecs/h264/h264_encoder_impl.h +++ b/modules/video_coding/codecs/h264/h264_encoder_impl.h @@ -28,6 +28,7 @@ #include "api/video_codecs/video_encoder.h" #include "common_video/h264/h264_bitstream_parser.h" #include "modules/video_coding/codecs/h264/include/h264.h" +#include "modules/video_coding/svc/scalable_video_controller.h" #include "modules/video_coding/utility/quality_scaler.h" #include "third_party/openh264/src/codec/api/svc/codec_app_def.h" @@ -97,6 +98,7 @@ class H264EncoderImpl : public H264Encoder { std::vector> downscaled_buffers_; std::vector configurations_; std::vector encoded_images_; + std::vector> svc_controllers_; VideoCodec codec_; H264PacketizationMode packetization_mode_; diff --git a/pc/test/svc_e2e_tests.cc b/pc/test/svc_e2e_tests.cc index 4e15c6c6f7..8482c77ae8 100644 --- a/pc/test/svc_e2e_tests.cc +++ b/pc/test/svc_e2e_tests.cc @@ -335,11 +335,12 @@ INSTANTIATE_TEST_SUITE_P( SvcTest, Combine(ValuesIn({ SvcTestParameters::Create(kH264CodecName, "L1T1"), - // SSvcTestParameters::Create(kH264CodecName, "L1T2"), - // SSvcTestParameters::Create(kH264CodecName, "L1T3"), + SvcTestParameters::Create(kH264CodecName, "L1T2"), + SvcTestParameters::Create(kH264CodecName, "L1T3"), }), - Values(UseDependencyDescriptor::Disabled, - UseDependencyDescriptor::Enabled)), + // Like AV1, H.264 RTP format does not include SVC related + // information, so always use Dependency Descriptor. + Values(UseDependencyDescriptor::Enabled)), SvcTestNameGenerator); #endif