Add QualityConvergenceController to VideoStreamEncoder

QualityConvergenceController is a layer between VideoStreamEncoder
and QualityConvergenceMonitor that takes care of the simulcast
logic.

Bug: chromium:328598314
Change-Id: Iad8a9d9138e69a60fd508a7ef038220947888f0a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/356420
Commit-Queue: Johannes Kron <kron@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42642}

video/BUILD.gn

@@ -388,6 +388,8 @@ rtc_library("video_stream_encoder_impl") {
     "encoder_overshoot_detector.h",
     "frame_encode_metadata_writer.cc",
     "frame_encode_metadata_writer.h",
+    "quality_convergence_controller.cc",
+    "quality_convergence_controller.h",
     "quality_convergence_monitor.cc",
     "quality_convergence_monitor.h",
     "rate_utilization_tracker.cc",
@@ -778,6 +780,7 @@ if (rtc_include_tests) {
       "frame_decode_timing_unittest.cc",
       "frame_encode_metadata_writer_unittest.cc",
       "picture_id_tests.cc",
+      "quality_convergence_controller_unittest.cc",
       "quality_convergence_monitor_unittest.cc",
       "quality_limitation_reason_tracker_unittest.cc",
       "quality_scaling_tests.cc",

video/quality_convergence_controller.cc

@@ -0,0 +1,73 @@
+/*
+ *  Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/quality_convergence_controller.h"
+
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace {
+// TODO(https://crbug.com/328598314): Remove default values once HW encoders
+// correctly report the minimum QP value. These thresholds correspond to the
+// default configurations used for the software encoders.
+constexpr int kVp8DefaultStaticQpThreshold = 15;
+constexpr int kVp9DefaultStaticQpThreshold = 32;
+constexpr int kAv1DefaultStaticQpThreshold = 40;
+
+int GetDefaultStaticQpThreshold(VideoCodecType codec) {
+  switch (codec) {
+    case kVideoCodecVP8:
+      return kVp8DefaultStaticQpThreshold;
+    case kVideoCodecVP9:
+      return kVp9DefaultStaticQpThreshold;
+    case kVideoCodecAV1:
+      return kAv1DefaultStaticQpThreshold;
+    case kVideoCodecGeneric:
+    case kVideoCodecH264:
+    case kVideoCodecH265:
+      // -1 will effectively disable the static QP threshold since QP values are
+      // always >= 0.
+      return -1;
+  }
+}
+}  // namespace
+
+void QualityConvergenceController::Initialize(
+    int number_of_layers,
+    absl::optional<int> static_qp_threshold,
+    VideoCodecType codec,
+    const FieldTrialsView& trials) {
+  RTC_CHECK(number_of_layers > 0);
+  number_of_layers_ = number_of_layers;
+  convergence_monitors_.clear();
+
+  int qp_threshold =
+      static_qp_threshold.value_or(GetDefaultStaticQpThreshold(codec));
+  for (int i = 0; i < number_of_layers_; ++i) {
+    convergence_monitors_.push_back(
+        QualityConvergenceMonitor::Create(qp_threshold, codec, trials));
+  }
+  initialized_ = true;
+}
+
+bool QualityConvergenceController::AddSampleAndCheckTargetQuality(
+    int layer_index,
+    int qp,
+    bool is_refresh_frame) {
+  RTC_CHECK(initialized_);
+  if (layer_index < 0 || layer_index >= number_of_layers_) {
+    return false;
+  }
+
+  convergence_monitors_[layer_index]->AddSample(qp, is_refresh_frame);
+  return convergence_monitors_[layer_index]->AtTargetQuality();
+}
+
+}  // namespace webrtc

video/quality_convergence_controller.h

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_
+#define VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/field_trials_view.h"
+#include "api/video/video_codec_type.h"
+#include "video/quality_convergence_monitor.h"
+
+namespace webrtc {
+
+class QualityConvergenceController {
+ public:
+  void Initialize(int number_of_layers,
+                  absl::optional<int> static_qp_threshold,
+                  VideoCodecType codec,
+                  const FieldTrialsView& trials);
+
+  // Add the supplied `qp` value to the detection window for specified layer.
+  // `is_refresh_frame` must only be `true` if the corresponding
+  // video frame is a refresh frame that is used to improve the visual quality.
+  // Returns `true` if the algorithm has determined that the supplied QP values
+  // have converged and reached the target quality for this layer.
+  bool AddSampleAndCheckTargetQuality(int layer_index,
+                                      int qp,
+                                      bool is_refresh_frame);
+
+ private:
+  bool initialized_ = false;
+  int number_of_layers_ = 0;
+  std::vector<std::unique_ptr<QualityConvergenceMonitor>> convergence_monitors_;
+};
+
+}  // namespace webrtc
+
+#endif  // VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_

video/quality_convergence_controller_unittest.cc

@@ -0,0 +1,66 @@
+
+/*
+ *  Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "video/quality_convergence_controller.h"
+
+#include "test/gtest.h"
+#include "test/scoped_key_value_config.h"
+
+namespace webrtc {
+namespace {
+constexpr int kStaticQpThreshold = 15;
+
+TEST(QualityConvergenceController, Singlecast) {
+  test::ScopedKeyValueConfig field_trials;
+  QualityConvergenceController controller;
+  controller.Initialize(1, kStaticQpThreshold, kVideoCodecVP8, field_trials);
+
+  EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/0, kStaticQpThreshold + 1, /*is_refresh_frame=*/false));
+  EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/false));
+}
+
+TEST(QualityConvergenceController, Simulcast) {
+  test::ScopedKeyValueConfig field_trials;
+  QualityConvergenceController controller;
+  controller.Initialize(2, kStaticQpThreshold, kVideoCodecVP8, field_trials);
+
+  EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/0, kStaticQpThreshold + 1, /*is_refresh_frame=*/false));
+  EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/false));
+
+  // Layer 0 reaches target quality.
+  EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/false));
+  EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/false));
+
+  // Frames are repeated for both layers. Layer 0 still at target quality.
+  EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/true));
+  EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/true));
+}
+
+TEST(QualityConvergenceController, InvalidLayerIndex) {
+  test::ScopedKeyValueConfig field_trials;
+  QualityConvergenceController controller;
+  controller.Initialize(2, kStaticQpThreshold, kVideoCodecVP8, field_trials);
+
+  EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/-1, kStaticQpThreshold, /*is_refresh_frame=*/false));
+  EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+      /*layer_index=*/3, kStaticQpThreshold, /*is_refresh_frame=*/false));
+}
+
+}  // namespace
+}  // namespace webrtc

video/video_stream_encoder.cc

@@ -1254,6 +1254,10 @@ void VideoStreamEncoder::ReconfigureEncoder() {
     codec.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityLow);
   }
 
+  quality_convergence_controller_.Initialize(
+      codec.numberOfSimulcastStreams, encoder_->GetEncoderInfo().min_qp,
+      codec.codecType, env_.field_trials());
+
   send_codec_ = codec;
 
   // Keep the same encoder, as long as the video_format is unchanged.
@@ -2085,14 +2089,23 @@ EncodedImage VideoStreamEncoder::AugmentEncodedImage(
             .Parse(codec_type, stream_idx, image_copy.data(), image_copy.size())
             .value_or(-1);
   }
+
+  // Check if the encoded image has reached target quality.
+  const size_t simulcast_index = encoded_image.SimulcastIndex().value_or(0);
+  bool at_target_quality =
+      quality_convergence_controller_.AddSampleAndCheckTargetQuality(
+          simulcast_index, image_copy.qp_,
+          image_copy.IsSteadyStateRefreshFrame());
+  image_copy.SetAtTargetQuality(at_target_quality);
   TRACE_EVENT2("webrtc", "VideoStreamEncoder::AugmentEncodedImage",
                "stream_idx", stream_idx, "qp", image_copy.qp_);
+  TRACE_EVENT_INSTANT2("webrtc", "VideoStreamEncoder::AugmentEncodedImage",
+                       TRACE_EVENT_SCOPE_GLOBAL, "simulcast_idx",
+                       simulcast_index, "at_target_quality", at_target_quality);
   RTC_LOG(LS_VERBOSE) << __func__ << " ntp time " << encoded_image.NtpTimeMs()
                       << " stream_idx " << stream_idx << " qp "
-                      << image_copy.qp_;
-  image_copy.SetAtTargetQuality(codec_type == kVideoCodecVP8 &&
-                                image_copy.qp_ <= kVp8SteadyStateQpThreshold);
-
+                      << image_copy.qp_ << " at target quality "
+                      << at_target_quality;
   return image_copy;
 }
 

video/video_stream_encoder.h

@@ -47,6 +47,7 @@
 #include "video/encoder_bitrate_adjuster.h"
 #include "video/frame_cadence_adapter.h"
 #include "video/frame_encode_metadata_writer.h"
+#include "video/quality_convergence_controller.h"
 #include "video/video_source_sink_controller.h"
 #include "video/video_stream_encoder_interface.h"
 #include "video/video_stream_encoder_observer.h"
@@ -418,6 +419,11 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
   QpParser qp_parser_;
   const bool qp_parsing_allowed_;
 
+  // The quality convergence controller is used to determine if a codec has
+  // reached its target quality. This is used for screenshare to determine when
+  // there's no need to continue encoding the same repeated frame.
+  QualityConvergenceController quality_convergence_controller_;
+
   // Enables encoder switching on initialization failures.
   bool switch_encoder_on_init_failures_;
 

video/video_stream_encoder_unittest.cc

@@ -701,6 +701,10 @@ class SimpleVideoStreamEncoderFactory {
                 (EncodedImage & encoded_image,
                  rtc::scoped_refptr<EncodedImageBuffer> buffer),
                 (override));
+    MOCK_METHOD(VideoEncoder::EncoderInfo,
+                GetEncoderInfo,
+                (),
+                (const, override));
   };
 
   SimpleVideoStreamEncoderFactory() {
@@ -9475,6 +9479,12 @@ TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
   auto video_stream_encoder =
       factory.Create(std::move(adapter), &encoder_queue);
 
+  // Set minimum QP.
+  VideoEncoder::EncoderInfo info;
+  info.min_qp = kVp8SteadyStateQpThreshold;
+  EXPECT_CALL(factory.GetMockFakeEncoder(), GetEncoderInfo)
+      .WillRepeatedly(Return(info));
+
   // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
   VideoEncoderConfig video_encoder_config;
   test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);