diff --git a/video/BUILD.gn b/video/BUILD.gn index 7e85a556d0..a23d4bb6ce 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -388,6 +388,8 @@ rtc_library("video_stream_encoder_impl") { "encoder_overshoot_detector.h", "frame_encode_metadata_writer.cc", "frame_encode_metadata_writer.h", + "quality_convergence_controller.cc", + "quality_convergence_controller.h", "quality_convergence_monitor.cc", "quality_convergence_monitor.h", "rate_utilization_tracker.cc", @@ -778,6 +780,7 @@ if (rtc_include_tests) { "frame_decode_timing_unittest.cc", "frame_encode_metadata_writer_unittest.cc", "picture_id_tests.cc", + "quality_convergence_controller_unittest.cc", "quality_convergence_monitor_unittest.cc", "quality_limitation_reason_tracker_unittest.cc", "quality_scaling_tests.cc", diff --git a/video/quality_convergence_controller.cc b/video/quality_convergence_controller.cc new file mode 100644 index 0000000000..7a46e11062 --- /dev/null +++ b/video/quality_convergence_controller.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/quality_convergence_controller.h" + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { +// TODO(https://crbug.com/328598314): Remove default values once HW encoders +// correctly report the minimum QP value. These thresholds correspond to the +// default configurations used for the software encoders. +constexpr int kVp8DefaultStaticQpThreshold = 15; +constexpr int kVp9DefaultStaticQpThreshold = 32; +constexpr int kAv1DefaultStaticQpThreshold = 40; + +int GetDefaultStaticQpThreshold(VideoCodecType codec) { + switch (codec) { + case kVideoCodecVP8: + return kVp8DefaultStaticQpThreshold; + case kVideoCodecVP9: + return kVp9DefaultStaticQpThreshold; + case kVideoCodecAV1: + return kAv1DefaultStaticQpThreshold; + case kVideoCodecGeneric: + case kVideoCodecH264: + case kVideoCodecH265: + // -1 will effectively disable the static QP threshold since QP values are + // always >= 0. + return -1; + } +} +} // namespace + +void QualityConvergenceController::Initialize( + int number_of_layers, + absl::optional static_qp_threshold, + VideoCodecType codec, + const FieldTrialsView& trials) { + RTC_CHECK(number_of_layers > 0); + number_of_layers_ = number_of_layers; + convergence_monitors_.clear(); + + int qp_threshold = + static_qp_threshold.value_or(GetDefaultStaticQpThreshold(codec)); + for (int i = 0; i < number_of_layers_; ++i) { + convergence_monitors_.push_back( + QualityConvergenceMonitor::Create(qp_threshold, codec, trials)); + } + initialized_ = true; +} + +bool QualityConvergenceController::AddSampleAndCheckTargetQuality( + int layer_index, + int qp, + bool is_refresh_frame) { + RTC_CHECK(initialized_); + if (layer_index < 0 || layer_index >= number_of_layers_) { + return false; + } + + convergence_monitors_[layer_index]->AddSample(qp, is_refresh_frame); + return convergence_monitors_[layer_index]->AtTargetQuality(); +} + +} // namespace webrtc diff --git a/video/quality_convergence_controller.h b/video/quality_convergence_controller.h new file mode 100644 index 0000000000..46f8419c41 --- /dev/null +++ b/video/quality_convergence_controller.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_ +#define VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/video/video_codec_type.h" +#include "video/quality_convergence_monitor.h" + +namespace webrtc { + +class QualityConvergenceController { + public: + void Initialize(int number_of_layers, + absl::optional static_qp_threshold, + VideoCodecType codec, + const FieldTrialsView& trials); + + // Add the supplied `qp` value to the detection window for specified layer. + // `is_refresh_frame` must only be `true` if the corresponding + // video frame is a refresh frame that is used to improve the visual quality. + // Returns `true` if the algorithm has determined that the supplied QP values + // have converged and reached the target quality for this layer. + bool AddSampleAndCheckTargetQuality(int layer_index, + int qp, + bool is_refresh_frame); + + private: + bool initialized_ = false; + int number_of_layers_ = 0; + std::vector> convergence_monitors_; +}; + +} // namespace webrtc + +#endif // VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_ diff --git a/video/quality_convergence_controller_unittest.cc b/video/quality_convergence_controller_unittest.cc new file mode 100644 index 0000000000..c1378e06a9 --- /dev/null +++ b/video/quality_convergence_controller_unittest.cc @@ -0,0 +1,66 @@ + +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "video/quality_convergence_controller.h" + +#include "test/gtest.h" +#include "test/scoped_key_value_config.h" + +namespace webrtc { +namespace { +constexpr int kStaticQpThreshold = 15; + +TEST(QualityConvergenceController, Singlecast) { + test::ScopedKeyValueConfig field_trials; + QualityConvergenceController controller; + controller.Initialize(1, kStaticQpThreshold, kVideoCodecVP8, field_trials); + + EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/0, kStaticQpThreshold + 1, /*is_refresh_frame=*/false)); + EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/false)); +} + +TEST(QualityConvergenceController, Simulcast) { + test::ScopedKeyValueConfig field_trials; + QualityConvergenceController controller; + controller.Initialize(2, kStaticQpThreshold, kVideoCodecVP8, field_trials); + + EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/0, kStaticQpThreshold + 1, /*is_refresh_frame=*/false)); + EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/false)); + + // Layer 0 reaches target quality. + EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/false)); + EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/false)); + + // Frames are repeated for both layers. Layer 0 still at target quality. + EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/true)); + EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/true)); +} + +TEST(QualityConvergenceController, InvalidLayerIndex) { + test::ScopedKeyValueConfig field_trials; + QualityConvergenceController controller; + controller.Initialize(2, kStaticQpThreshold, kVideoCodecVP8, field_trials); + + EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/-1, kStaticQpThreshold, /*is_refresh_frame=*/false)); + EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality( + /*layer_index=*/3, kStaticQpThreshold, /*is_refresh_frame=*/false)); +} + +} // namespace +} // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 92f730838c..af26db8719 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1254,6 +1254,10 @@ void VideoStreamEncoder::ReconfigureEncoder() { codec.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityLow); } + quality_convergence_controller_.Initialize( + codec.numberOfSimulcastStreams, encoder_->GetEncoderInfo().min_qp, + codec.codecType, env_.field_trials()); + send_codec_ = codec; // Keep the same encoder, as long as the video_format is unchanged. @@ -2085,14 +2089,23 @@ EncodedImage VideoStreamEncoder::AugmentEncodedImage( .Parse(codec_type, stream_idx, image_copy.data(), image_copy.size()) .value_or(-1); } + + // Check if the encoded image has reached target quality. + const size_t simulcast_index = encoded_image.SimulcastIndex().value_or(0); + bool at_target_quality = + quality_convergence_controller_.AddSampleAndCheckTargetQuality( + simulcast_index, image_copy.qp_, + image_copy.IsSteadyStateRefreshFrame()); + image_copy.SetAtTargetQuality(at_target_quality); TRACE_EVENT2("webrtc", "VideoStreamEncoder::AugmentEncodedImage", "stream_idx", stream_idx, "qp", image_copy.qp_); + TRACE_EVENT_INSTANT2("webrtc", "VideoStreamEncoder::AugmentEncodedImage", + TRACE_EVENT_SCOPE_GLOBAL, "simulcast_idx", + simulcast_index, "at_target_quality", at_target_quality); RTC_LOG(LS_VERBOSE) << __func__ << " ntp time " << encoded_image.NtpTimeMs() << " stream_idx " << stream_idx << " qp " - << image_copy.qp_; - image_copy.SetAtTargetQuality(codec_type == kVideoCodecVP8 && - image_copy.qp_ <= kVp8SteadyStateQpThreshold); - + << image_copy.qp_ << " at target quality " + << at_target_quality; return image_copy; } diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index fe04fbf018..579cd3238c 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -47,6 +47,7 @@ #include "video/encoder_bitrate_adjuster.h" #include "video/frame_cadence_adapter.h" #include "video/frame_encode_metadata_writer.h" +#include "video/quality_convergence_controller.h" #include "video/video_source_sink_controller.h" #include "video/video_stream_encoder_interface.h" #include "video/video_stream_encoder_observer.h" @@ -418,6 +419,11 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, QpParser qp_parser_; const bool qp_parsing_allowed_; + // The quality convergence controller is used to determine if a codec has + // reached its target quality. This is used for screenshare to determine when + // there's no need to continue encoding the same repeated frame. + QualityConvergenceController quality_convergence_controller_; + // Enables encoder switching on initialization failures. bool switch_encoder_on_init_failures_; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index aab791f80a..ac4aa3ef0c 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -701,6 +701,10 @@ class SimpleVideoStreamEncoderFactory { (EncodedImage & encoded_image, rtc::scoped_refptr buffer), (override)); + MOCK_METHOD(VideoEncoder::EncoderInfo, + GetEncoderInfo, + (), + (const, override)); }; SimpleVideoStreamEncoderFactory() { @@ -9475,6 +9479,12 @@ TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) { auto video_stream_encoder = factory.Create(std::move(adapter), &encoder_queue); + // Set minimum QP. + VideoEncoder::EncoderInfo info; + info.min_qp = kVp8SteadyStateQpThreshold; + EXPECT_CALL(factory.GetMockFakeEncoder(), GetEncoderInfo) + .WillRepeatedly(Return(info)); + // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder. VideoEncoderConfig video_encoder_config; test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);