diff --git a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc index 84229ddfbd..a63a7c27fd 100644 --- a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc +++ b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc @@ -145,6 +145,8 @@ class VideoEncoderSoftwareFallbackWrapperTestBase : public ::testing::Test { info.scaling_settings = ScalingSettings(kLowThreshold, kHighThreshold); info.supports_native_handle = supports_native_handle_; info.implementation_name = implementation_name_; + if (is_qp_trusted_) + info.is_qp_trusted = is_qp_trusted_; return info; } @@ -156,6 +158,7 @@ class VideoEncoderSoftwareFallbackWrapperTestBase : public ::testing::Test { int release_count_ = 0; mutable int supports_native_handle_count_ = 0; bool supports_native_handle_ = false; + bool is_qp_trusted_ = false; std::string implementation_name_ = "fake-encoder"; absl::optional last_video_frame_; }; @@ -418,6 +421,16 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, ReportsImplementationName) { CheckLastEncoderName("fake-encoder"); } +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + IsQpTrustedNotForwardedDuringFallback) { + // Fake encoder signals trusted QP, default (libvpx) does not. + fake_encoder_->is_qp_trusted_ = true; + EXPECT_TRUE(fake_encoder_->GetEncoderInfo().is_qp_trusted.value_or(false)); + UtilizeFallbackEncoder(); + EXPECT_FALSE(fallback_wrapper_->GetEncoderInfo().is_qp_trusted.has_value()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); +} + TEST_F(VideoEncoderSoftwareFallbackWrapperTest, ReportsFallbackImplementationName) { UtilizeFallbackEncoder(); diff --git a/api/video_codecs/video_encoder.cc b/api/video_codecs/video_encoder.cc index a7e9d7487c..ac5d84b50a 100644 --- a/api/video_codecs/video_encoder.cc +++ b/api/video_codecs/video_encoder.cc @@ -187,6 +187,9 @@ std::string VideoEncoder::EncoderInfo::ToString() const { oss << VideoFrameBufferTypeToString(preferred_pixel_formats.at(i)); } oss << "]"; + if (is_qp_trusted.has_value()) { + oss << ", is_qp_trusted = " << is_qp_trusted.value(); + } oss << "}"; return oss.str(); } diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h index 8191f47f9c..2bdf8d015d 100644 --- a/api/video_codecs/video_encoder.h +++ b/api/video_codecs/video_encoder.h @@ -260,6 +260,10 @@ class RTC_EXPORT VideoEncoder { // preferred pixel format. The order of the formats does not matter. absl::InlinedVector preferred_pixel_formats; + + // Indicates whether or not QP value encoder writes into frame/slice/tile + // header can be interpreted as average frame/slice/tile QP. + absl::optional is_qp_trusted; }; struct RTC_EXPORT RateControlParameters { diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index f7be8b3013..d4137f621c 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -876,6 +876,7 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.is_hardware_accelerated = encoder_impl_info.is_hardware_accelerated; encoder_info.has_internal_source = encoder_impl_info.has_internal_source; + encoder_info.is_qp_trusted = encoder_impl_info.is_qp_trusted; } else { encoder_info.implementation_name += ", "; encoder_info.implementation_name += encoder_impl_info.implementation_name; @@ -897,6 +898,12 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { // Has internal source only if all encoders have it. encoder_info.has_internal_source &= encoder_impl_info.has_internal_source; + + // Treat QP from frame/slice/tile header as average QP only if all + // encoders report it as average QP. + encoder_info.is_qp_trusted = + encoder_info.is_qp_trusted.value_or(true) & + encoder_impl_info.is_qp_trusted.value_or(true); } encoder_info.fps_allocation[i] = encoder_impl_info.fps_allocation[0]; encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple( diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index c946b60a1a..5a2bf8e7e3 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -241,6 +241,7 @@ class MockVideoEncoder : public VideoEncoder { info.has_internal_source = has_internal_source_; info.fps_allocation[0] = fps_allocation_; info.supports_simulcast = supports_simulcast_; + info.is_qp_trusted = is_qp_trusted_; return info; } @@ -308,6 +309,10 @@ class MockVideoEncoder : public VideoEncoder { video_format_ = video_format; } + void set_is_qp_trusted(absl::optional is_qp_trusted) { + is_qp_trusted_ = is_qp_trusted; + } + bool supports_simulcast() const { return supports_simulcast_; } SdpVideoFormat video_format() const { return video_format_; } @@ -326,6 +331,7 @@ class MockVideoEncoder : public VideoEncoder { VideoEncoder::RateControlParameters last_set_rates_; FramerateFractions fps_allocation_; bool supports_simulcast_ = false; + absl::optional is_qp_trusted_; SdpVideoFormat video_format_; VideoCodec codec_; @@ -1387,6 +1393,28 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReportsInternalSource) { EXPECT_FALSE(adapter_->GetEncoderInfo().has_internal_source); } +TEST_F(TestSimulcastEncoderAdapterFake, ReportsIsQpTrusted) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + adapter_->RegisterEncodeCompleteCallback(this); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + // All encoders have internal source, simulcast adapter reports true. + for (MockVideoEncoder* encoder : helper_->factory()->encoders()) { + encoder->set_is_qp_trusted(true); + } + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_TRUE(adapter_->GetEncoderInfo().is_qp_trusted.value_or(false)); + + // One encoder reports QP not trusted, simulcast adapter reports false. + helper_->factory()->encoders()[2]->set_is_qp_trusted(false); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + EXPECT_FALSE(adapter_->GetEncoderInfo().is_qp_trusted.value_or(true)); +} + TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) { SimulcastTestFixtureImpl::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile), diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 84981b3621..23a3593625 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -527,7 +527,8 @@ void VideoStreamEncoderResourceManager::ConfigureQualityScaler( IsResolutionScalingEnabled(degradation_preference_) && (scaling_settings.thresholds.has_value() || (encoder_settings_.has_value() && - encoder_settings_->encoder_config().is_quality_scaling_allowed)); + encoder_settings_->encoder_config().is_quality_scaling_allowed)) && + encoder_info.is_qp_trusted.value_or(true); // TODO(https://crbug.com/webrtc/11222): Should this move to // QualityScalerResource? diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 37deab0a73..b9c6f8f10e 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -888,6 +888,9 @@ class VideoStreamEncoderTest : public ::testing::Test { info.apply_alignment_to_all_simulcast_layers = apply_alignment_to_all_simulcast_layers_; info.preferred_pixel_formats = preferred_pixel_formats_; + if (is_qp_trusted_.has_value()) { + info.is_qp_trusted = is_qp_trusted_; + } return info; } @@ -1036,6 +1039,11 @@ class VideoStreamEncoderTest : public ::testing::Test { preferred_pixel_formats_ = std::move(pixel_formats); } + void SetIsQpTrusted(absl::optional trusted) { + MutexLock lock(&local_mutex_); + is_qp_trusted_ = trusted; + } + private: int32_t Encode(const VideoFrame& input_image, const std::vector* frame_types) override { @@ -1185,6 +1193,7 @@ class VideoStreamEncoderTest : public ::testing::Test { RTC_GUARDED_BY(local_mutex_); absl::InlinedVector preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_); + absl::optional is_qp_trusted_ RTC_GUARDED_BY(local_mutex_); }; class TestSink : public VideoStreamEncoder::EncoderSink { @@ -8191,6 +8200,66 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP not trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = true; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(true); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = false; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + #if !defined(WEBRTC_IOS) // TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is // disabled by default on iOS. @@ -8221,6 +8290,36 @@ TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) { video_stream_encoder_->Stop(); } + +TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(true); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = true; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} #endif // Test parameters: (VideoCodecType codec, bool allow_i420_conversion)