diff --git a/media/BUILD.gn b/media/BUILD.gn index 5563faea4c..1b8f4cc12d 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -502,6 +502,7 @@ rtc_library("rtc_internal_video_codecs") { "../api/video:video_frame", "../api/video:video_rtp_headers", "../api/video_codecs:rtc_software_fallback_wrappers", + "../api/video_codecs:scalability_mode", "../api/video_codecs:video_codecs_api", "../api/video_codecs:video_encoder_factory_template", "../api/video_codecs:video_encoder_factory_template_libvpx_vp8_adapter", @@ -517,6 +518,7 @@ rtc_library("rtc_internal_video_codecs") { "../rtc_base/system:rtc_export", "../system_wrappers:field_trial", "../test:fake_video_codecs", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings", ] diff --git a/media/engine/fake_video_codec_factory.cc b/media/engine/fake_video_codec_factory.cc index 1b62cff10c..f6f9627bf8 100644 --- a/media/engine/fake_video_codec_factory.cc +++ b/media/engine/fake_video_codec_factory.cc @@ -12,7 +12,9 @@ #include +#include "absl/container/inlined_vector.h" #include "api/environment/environment.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_encoder.h" @@ -33,8 +35,15 @@ namespace webrtc { std::vector FakeVideoEncoderFactory::GetSupportedFormats() const { + const absl::InlinedVector + kSupportedScalabilityModes = {webrtc::ScalabilityMode::kL1T1, + webrtc::ScalabilityMode::kL1T2, + webrtc::ScalabilityMode::kL1T3}; + return std::vector( - 1, SdpVideoFormat(kFakeCodecFactoryCodecName)); + 1, SdpVideoFormat(kFakeCodecFactoryCodecName, {}, + kSupportedScalabilityModes)); } std::unique_ptr FakeVideoEncoderFactory::Create( diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 781e997cc4..69b71fff7e 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -553,10 +553,17 @@ void FallbackToDefaultScalabilityModeIfNotSupported( // scalability mode of the first encoding when the others are inactive. continue; } + if (!encoding.scalability_mode.has_value() || !IsScalabilityModeSupportedByCodec(codec, *encoding.scalability_mode, config)) { - encoding.scalability_mode = webrtc::kDefaultScalabilityModeStr; + encoding.scalability_mode = + (encoding.scalability_mode != + std::string(webrtc::kDefaultScalabilityModeStr) && + IsScalabilityModeSupportedByCodec( + codec, webrtc::kDefaultScalabilityModeStr, config)) + ? webrtc::kDefaultScalabilityModeStr + : webrtc::kNoLayeringScalabilityModeStr; RTC_LOG(LS_INFO) << " -> " << *encoding.scalability_mode; } } diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index 83b3d22df9..222ae0ad89 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -8480,12 +8480,90 @@ TEST_F(WebRtcVideoChannelTest, FallbackForUnsetOrUnsupportedScalabilityMode) { EXPECT_TRUE(send_channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); } +#ifdef RTC_ENABLE_H265 +TEST_F( + WebRtcVideoChannelTest, + NoLayeringValueUsedIfModeIsUnsetOrUnsupportedByH265AndDefaultUnsupported) { + const absl::InlinedVector + kSupportedModes = {ScalabilityMode::kL1T1, ScalabilityMode::kL1T3}; + + encoder_factory_->AddSupportedVideoCodec(webrtc::SdpVideoFormat( + "H265", webrtc::CodecParameterMap(), kSupportedModes)); + cricket::VideoSenderParameters send_parameters; + send_parameters.codecs.push_back(GetEngineCodec("H265")); + EXPECT_TRUE(send_channel_->SetSenderParameters(send_parameters)); + + FakeVideoSendStream* stream = SetUpSimulcast(true, /*with_rtx=*/false); + + // Send a full size frame so all simulcast layers are used when reconfiguring. + webrtc::test::FrameForwarder frame_forwarder; + VideoOptions options; + EXPECT_TRUE( + send_channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder)); + send_channel_->SetSend(true); + frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); + + // Set scalability mode. + webrtc::RtpParameters parameters = + send_channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + parameters.encodings[0].scalability_mode = std::nullopt; + parameters.encodings[1].scalability_mode = "L1T3"; // Supported. + parameters.encodings[2].scalability_mode = "L3T3"; // Unsupported. + EXPECT_TRUE(send_channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + + // Verify that the new value is propagated down to the encoder. + // Check that WebRtcVideoSendStream updates VideoEncoderConfig correctly. + const std::optional kDefaultScalabilityMode = + webrtc::ScalabilityModeFromString(webrtc::kNoLayeringScalabilityModeStr); + EXPECT_EQ(2, stream->num_encoder_reconfigurations()); + webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); + EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams); + EXPECT_THAT(encoder_config.simulcast_layers, + ElementsAre(Field(&webrtc::VideoStream::scalability_mode, + kDefaultScalabilityMode), + Field(&webrtc::VideoStream::scalability_mode, + ScalabilityMode::kL1T3), + Field(&webrtc::VideoStream::scalability_mode, + kDefaultScalabilityMode))); + + // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of + // VideoStreams are created appropriately for the simulcast case. + EXPECT_THAT(stream->GetVideoStreams(), + ElementsAre(Field(&webrtc::VideoStream::scalability_mode, + kDefaultScalabilityMode), + Field(&webrtc::VideoStream::scalability_mode, + ScalabilityMode::kL1T3), + Field(&webrtc::VideoStream::scalability_mode, + kDefaultScalabilityMode))); + + // GetParameters. + parameters = send_channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_THAT( + parameters.encodings, + ElementsAre( + Field(&webrtc::RtpEncodingParameters::scalability_mode, + webrtc::kNoLayeringScalabilityModeStr), + Field(&webrtc::RtpEncodingParameters::scalability_mode, "L1T3"), + Field(&webrtc::RtpEncodingParameters::scalability_mode, + webrtc::kNoLayeringScalabilityModeStr))); + + // No parameters changed, encoder should not be reconfigured. + EXPECT_TRUE(send_channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); + EXPECT_EQ(2, stream->num_encoder_reconfigurations()); + + EXPECT_TRUE(send_channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} +#endif + TEST_F(WebRtcVideoChannelTest, DefaultValueUsedIfScalabilityModeIsUnsupportedByCodec) { - encoder_factory_->AddSupportedVideoCodec(webrtc::SdpVideoFormat( - "VP8", webrtc::CodecParameterMap(), {ScalabilityMode::kL1T1})); - encoder_factory_->AddSupportedVideoCodec(webrtc::SdpVideoFormat( - "VP9", webrtc::CodecParameterMap(), {ScalabilityMode::kL3T3})); + encoder_factory_->AddSupportedVideoCodec( + webrtc::SdpVideoFormat("VP8", webrtc::CodecParameterMap(), + {ScalabilityMode::kL1T1, ScalabilityMode::kL1T2})); + encoder_factory_->AddSupportedVideoCodec( + webrtc::SdpVideoFormat("VP9", webrtc::CodecParameterMap(), + {ScalabilityMode::kL1T2, ScalabilityMode::kL3T3})); cricket::VideoSenderParameters send_parameters; send_parameters.codecs.push_back(GetEngineCodec("VP9")); diff --git a/modules/video_coding/svc/scalability_mode_util.h b/modules/video_coding/svc/scalability_mode_util.h index 80c7283c25..f7689988bc 100644 --- a/modules/video_coding/svc/scalability_mode_util.h +++ b/modules/video_coding/svc/scalability_mode_util.h @@ -26,6 +26,10 @@ enum class ScalabilityModeResolutionRatio { static constexpr char kDefaultScalabilityModeStr[] = "L1T2"; +// Scalability mode to be used if falling back to default scalability mode is +// unsupported. +static constexpr char kNoLayeringScalabilityModeStr[] = "L1T1"; + RTC_EXPORT std::optional MakeScalabilityMode( int num_spatial_layers, int num_temporal_layers, diff --git a/test/peer_scenario/BUILD.gn b/test/peer_scenario/BUILD.gn index bf1781882f..132fac89ae 100644 --- a/test/peer_scenario/BUILD.gn +++ b/test/peer_scenario/BUILD.gn @@ -45,6 +45,7 @@ if (rtc_include_tests) { "../../api/transport:datagram_transport_interface", "../../api/transport:enums", "../../api/transport:field_trial_based_config", + "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_decoder_factory_template", "../../api/video_codecs:video_decoder_factory_template_dav1d_adapter", "../../api/video_codecs:video_decoder_factory_template_libvpx_vp8_adapter", @@ -85,6 +86,7 @@ if (rtc_include_tests) { "../network:emulated_network", "../scenario", "../time_controller", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/flags:flag", "//third_party/abseil-cpp/absl/memory", ] diff --git a/test/peer_scenario/peer_scenario_client.cc b/test/peer_scenario/peer_scenario_client.cc index f3923c5f9e..5a490edfec 100644 --- a/test/peer_scenario/peer_scenario_client.cc +++ b/test/peer_scenario/peer_scenario_client.cc @@ -13,6 +13,7 @@ #include #include +#include "absl/container/inlined_vector.h" #include "absl/memory/memory.h" #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" @@ -21,6 +22,7 @@ #include "api/task_queue/default_task_queue_factory.h" #include "api/test/create_time_controller.h" #include "api/transport/field_trial_based_config.h" +#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_decoder_factory_template.h" #include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h" #include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h" @@ -179,7 +181,13 @@ class LambdaSetRemoteDescriptionObserver class FakeVideoEncoderFactory : public VideoEncoderFactory { public: std::vector GetSupportedFormats() const override { - return {SdpVideoFormat::VP8()}; + const absl::InlinedVector + kSupportedScalabilityModes = {webrtc::ScalabilityMode::kL1T1, + webrtc::ScalabilityMode::kL1T2, + webrtc::ScalabilityMode::kL1T3}; + return { + SdpVideoFormat(cricket::kVp8CodecName, {}, kSupportedScalabilityModes)}; } std::unique_ptr Create(const Environment& env, const SdpVideoFormat& format) override {