From 9f096a8707262cfaf26bd704916cd8a65dd0d7ac Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Fri, 6 Sep 2024 12:06:15 +0200 Subject: [PATCH] Allow VideoEncoderSoftwareFallbackWrapper to return SIMULCAST_PARAMS_NOT_SUPPORTED MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now some HW encoders support simulcast. If parameters are not suitable for single encoder simulcast, the error code should be forwarded back to SimulcastEncoderAdapter instead of trying software fallback. Bug: webrtc:347737882 Change-Id: Id02ff1afc012cd46761d9530b1ce368d5dc480bb Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/361744 Commit-Queue: Ilya Nikolaevskiy Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/main@{#42972} --- ...video_encoder_software_fallback_wrapper.cc | 3 + .../simulcast_encoder_adapter_unittest.cc | 122 +++++++++++++++++- 2 files changed, 118 insertions(+), 7 deletions(-) diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc index c3e9ad9389..53efa515ba 100644 --- a/api/video_codecs/video_encoder_software_fallback_wrapper.cc +++ b/api/video_codecs/video_encoder_software_fallback_wrapper.cc @@ -344,6 +344,9 @@ int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( PrimeEncoder(current_encoder()); return ret; } + if (ret == WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED) { + return ret; + } RTC_LOG(LS_WARNING) << "[VESFW] Hardware encoder initialization failed with" << " error code: " << WebRtcVideoCodecErrorToString(ret); diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index a52ac96d56..8baccfd4f5 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -191,8 +191,8 @@ class MockVideoEncoderFactory : public VideoEncoderFactory { std::vector limits) { resolution_bitrate_limits_ = limits; } - void set_fallback_from_simulcast(bool value) { - fallback_from_simulcast_ = value; + void set_fallback_from_simulcast(std::optional return_value) { + fallback_from_simulcast_ = return_value; } void DestroyVideoEncoder(VideoEncoder* encoder); @@ -200,7 +200,7 @@ class MockVideoEncoderFactory : public VideoEncoderFactory { private: bool create_video_encoder_return_nullptr_ = false; int32_t init_encode_return_value_ = 0; - bool fallback_from_simulcast_ = false; + std::optional fallback_from_simulcast_; std::vector encoders_; std::vector encoder_names_; // Keep number of entries in sync with `kMaxSimulcastStreams`. @@ -226,7 +226,7 @@ class MockVideoEncoder : public VideoEncoder { const VideoEncoder::Settings& settings) override { codec_ = *codecSettings; if (codec_.numberOfSimulcastStreams > 1 && fallback_from_simulcast_) { - return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + return *fallback_from_simulcast_; } return init_encode_return_value_; } @@ -294,7 +294,7 @@ class MockVideoEncoder : public VideoEncoder { init_encode_return_value_ = value; } - void set_fallback_from_simulcast(bool value) { + void set_fallback_from_simulcast(std::optional value) { fallback_from_simulcast_ = value; } @@ -356,7 +356,7 @@ class MockVideoEncoder : public VideoEncoder { bool has_trusted_rate_controller_ = false; bool is_hardware_accelerated_ = false; int32_t init_encode_return_value_ = 0; - bool fallback_from_simulcast_ = false; + std::optional fallback_from_simulcast_; VideoEncoder::RateControlParameters last_set_rates_; FramerateFractions fps_allocation_; bool supports_simulcast_ = false; @@ -636,7 +636,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, InitEncode) { TEST_F(TestSimulcastEncoderAdapterFake, EarlyCallbackSetupNotLost) { helper_->factory()->set_supports_simulcast(true); - helper_->factory()->set_fallback_from_simulcast(true); + helper_->factory()->set_fallback_from_simulcast( + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); SetupCodecWithEarlyEncodeCompleteCallback( /*active_streams=*/{true, true, true}); for (size_t idx = 0; idx < 3; ++idx) { @@ -1808,6 +1809,113 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsFallback) { EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); } +TEST_F(TestSimulcastEncoderAdapterFake, + SupportsHardwareSimulcastWithBadParametrs) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + + // Enable support for fallback encoder factory and re-setup. + use_fallback_factory_ = true; + SetUp(); + + helper_->factory()->set_supports_simulcast(true); + // Make encoders reject the simulcast configuration despite supporting it + // because parameters are not good for simulcast (e.g. different temporal + // layers setting). + helper_->factory()->set_fallback_from_simulcast( + WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED); + + SetupCodec(); + + // Make sure we have bitrate for all layers. + DataRate max_bitrate = DataRate::Zero(); + for (int i = 0; i < 3; ++i) { + max_bitrate += + DataRate::KilobitsPerSec(codec_.simulcastStream[i].maxBitrate); + } + const auto rate_settings = VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(max_bitrate.bps(), 30)), + 30.0, max_bitrate); + adapter_->SetRates(rate_settings); + + std::vector primary_encoders = + helper_->factory()->encoders(); + std::vector fallback_encoders = + helper_->fallback_factory()->encoders(); + + ASSERT_EQ(3u, primary_encoders.size()); + ASSERT_EQ(3u, fallback_encoders.size()); + + // Create frame to test with. + rtc::scoped_refptr buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rtp_timestamp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + std::vector frame_types(3, VideoFrameType::kVideoFrameKey); + + // All primary encoders must be used. + for (auto codec : primary_encoders) { + EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + } + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, SupportsHardwareSimulcast) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + + // Enable support for fallback encoder factory and re-setup. + use_fallback_factory_ = true; + SetUp(); + + helper_->factory()->set_supports_simulcast(true); + helper_->factory()->set_fallback_from_simulcast(std::nullopt); + + SetupCodec(); + + // Make sure we have bitrate for all layers. + DataRate max_bitrate = DataRate::Zero(); + for (int i = 0; i < 3; ++i) { + max_bitrate += + DataRate::KilobitsPerSec(codec_.simulcastStream[i].maxBitrate); + } + const auto rate_settings = VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(max_bitrate.bps(), 30)), + 30.0, max_bitrate); + adapter_->SetRates(rate_settings); + + std::vector primary_encoders = + helper_->factory()->encoders(); + std::vector fallback_encoders = + helper_->fallback_factory()->encoders(); + + ASSERT_EQ(1u, primary_encoders.size()); + ASSERT_EQ(1u, fallback_encoders.size()); + + // Create frame to test with. + rtc::scoped_refptr buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rtp_timestamp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + std::vector frame_types(3, VideoFrameType::kVideoFrameKey); + + // A primary encoders must be used. + for (auto codec : primary_encoders) { + EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + } + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + TEST_F(TestSimulcastEncoderAdapterFake, SupportsPerSimulcastLayerMaxFramerate) { SimulcastTestFixtureImpl::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile),