Video: add new metric for VP9/AV1 hw encoding with softwareBRC

Record the RMSE of encoding bitrate diff from output stream to target
bitrate set by WebRTC when the AV1/VP9 stream is removed.

Design doc: https://docs.google.com/document/d/1tE7Fquj1U6CwZ01QxxjoCaJS2o0LFYyIxvstLDx0qB0/edit#heading=h.c11ebhaja9tg

Bug: chromium:1358750
Change-Id: Ifa6f5aacfa8f6b0835f4609f102ae3ae1ee2897e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/298684
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39907}

AUTHORS

@@ -133,6 +133,7 @@ Yura Yaroshevich <yura.yaroshevich@gmail.com>
 Yuriy Pavlyshak <yuriy@appear.in>
 Yusuke Suzuki <utatane.tea@gmail.com>
 Pengfei Han <hanpfei@gmail.com>
+Yingying Ma <yingying.ma@intel.com>
 # END individuals section.
 
 # BEGIN organizations section.

video/encoder_bitrate_adjuster.cc

@@ -47,7 +47,9 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings)
     : utilize_bandwidth_headroom_(RateControlSettings::ParseFromFieldTrials()
                                       .BitrateAdjusterCanUseNetworkHeadroom()),
       frames_since_layout_change_(0),
-      min_bitrates_bps_{} {
+      min_bitrates_bps_{},
+      codec_(codec_settings.codecType),
+      codec_mode_(codec_settings.mode) {
   // TODO(https://crbug.com/webrtc/14891): If we want to support simulcast of
   // SVC streams, EncoderBitrateAdjuster needs to be updated to care about both
   // `simulcastStream` and `spatialLayers` at the same time.
@@ -90,7 +92,9 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
         ++active_tls[si];
         if (!overshoot_detectors_[si][ti]) {
           overshoot_detectors_[si][ti] =
-              std::make_unique<EncoderOvershootDetector>(kWindowSizeMs);
+              std::make_unique<EncoderOvershootDetector>(
+                  kWindowSizeMs, codec_,
+                  codec_mode_ == VideoCodecMode::kScreensharing);
           frames_since_layout_change_ = 0;
         }
       } else if (overshoot_detectors_[si][ti]) {

video/encoder_bitrate_adjuster.h

@@ -73,6 +73,12 @@ class EncoderBitrateAdjuster {
 
   // Minimum bitrates allowed, per spatial layer.
   uint32_t min_bitrates_bps_[kMaxSpatialLayers];
+
+  // Codec type used for encoding.
+  VideoCodecType codec_;
+
+  // Codec mode: { kRealtimeVideo, kScreensharing }.
+  VideoCodecMode codec_mode_;
 };
 
 }  // namespace webrtc

video/encoder_overshoot_detector.cc

@@ -11,6 +11,9 @@
 #include "video/encoder_overshoot_detector.h"
 
 #include <algorithm>
+#include <string>
+
+#include "system_wrappers/include/metrics.h"
 
 namespace webrtc {
 namespace {
@@ -20,7 +23,9 @@ namespace {
 static constexpr double kMaxMediaUnderrunFrames = 5.0;
 }  // namespace
 
-EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms)
+EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms,
+                                                   VideoCodecType codec,
+                                                   bool is_screenshare)
     : window_size_ms_(window_size_ms),
       time_last_update_ms_(-1),
       sum_network_utilization_factors_(0.0),
@@ -28,9 +33,16 @@ EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms)
       target_bitrate_(DataRate::Zero()),
       target_framerate_fps_(0),
       network_buffer_level_bits_(0),
-      media_buffer_level_bits_(0) {}
+      media_buffer_level_bits_(0),
+      codec_(codec),
+      is_screenshare_(is_screenshare),
+      frame_count_(0),
+      sum_diff_kbps_squared_(0),
+      sum_overshoot_percent_(0) {}
 
-EncoderOvershootDetector::~EncoderOvershootDetector() = default;
+EncoderOvershootDetector::~EncoderOvershootDetector() {
+  UpdateHistograms();
+}
 
 void EncoderOvershootDetector::SetTargetRate(DataRate target_bitrate,
                                              double target_framerate_fps,
@@ -57,6 +69,7 @@ void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) {
   // bitrate.
   LeakBits(time_ms);
 
+  const int64_t frame_size_bits = bytes * 8;
   // Ideal size of a frame given the current rates.
   const int64_t ideal_frame_size_bits = IdealFrameSizeBits();
   if (ideal_frame_size_bits == 0) {
@@ -64,14 +77,22 @@ void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) {
     return;
   }
 
-  const double network_utilization_factor = HandleEncodedFrame(
+  const double network_utilization_factor =
-      bytes * 8, ideal_frame_size_bits, time_ms, &network_buffer_level_bits_);
+      HandleEncodedFrame(frame_size_bits, ideal_frame_size_bits, time_ms,
-  const double media_utilization_factor = HandleEncodedFrame(
+                         &network_buffer_level_bits_);
-      bytes * 8, ideal_frame_size_bits, time_ms, &media_buffer_level_bits_);
+  const double media_utilization_factor =
+      HandleEncodedFrame(frame_size_bits, ideal_frame_size_bits, time_ms,
+                         &media_buffer_level_bits_);
 
   sum_network_utilization_factors_ += network_utilization_factor;
   sum_media_utilization_factors_ += media_utilization_factor;
 
+  // Calculate the bitrate diff in kbps
+  int64_t diff_kbits = (frame_size_bits - ideal_frame_size_bits) / 1000;
+  sum_diff_kbps_squared_ += diff_kbits * diff_kbits;
+  sum_overshoot_percent_ += diff_kbits * 100 * 1000 / ideal_frame_size_bits;
+  ++frame_count_;
+
   utilization_factors_.emplace_back(network_utilization_factor,
                                     media_utilization_factor, time_ms);
 }
@@ -142,6 +163,10 @@ absl::optional<double> EncoderOvershootDetector::GetMediaRateUtilizationFactor(
 }
 
 void EncoderOvershootDetector::Reset() {
+  UpdateHistograms();
+  sum_diff_kbps_squared_ = 0;
+  frame_count_ = 0;
+  sum_overshoot_percent_ = 0;
   time_last_update_ms_ = -1;
   utilization_factors_.clear();
   target_bitrate_ = DataRate::Zero();
@@ -201,4 +226,50 @@ void EncoderOvershootDetector::CullOldUpdates(int64_t time_ms) {
   }
 }
 
+void EncoderOvershootDetector::UpdateHistograms() {
+  if (frame_count_ == 0)
+    return;
+
+  int64_t bitrate_rmse = std::sqrt(sum_diff_kbps_squared_ / frame_count_);
+  int64_t average_overshoot_percent = sum_overshoot_percent_ / frame_count_;
+  const std::string rmse_histogram_prefix =
+      is_screenshare_ ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps."
+                      : "WebRTC.Video.RMSEOfEncodingBitrateInKbps.";
+  const std::string overshoot_histogram_prefix =
+      is_screenshare_ ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot."
+                      : "WebRTC.Video.EncodingBitrateOvershoot.";
+  // index = 1 represents screensharing histograms recording.
+  // index = 0 represents normal video histograms recording.
+  const int index = is_screenshare_ ? 1 : 0;
+  switch (codec_) {
+    case VideoCodecType::kVideoCodecAV1:
+      RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Av1",
+                                  bitrate_rmse);
+      RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Av1",
+                                  average_overshoot_percent);
+      break;
+    case VideoCodecType::kVideoCodecVP9:
+      RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Vp9",
+                                  bitrate_rmse);
+      RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Vp9",
+                                  average_overshoot_percent);
+      break;
+    case VideoCodecType::kVideoCodecVP8:
+      RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Vp8",
+                                  bitrate_rmse);
+      RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Vp8",
+                                  average_overshoot_percent);
+      break;
+    case VideoCodecType::kVideoCodecH264:
+      RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "H264",
+                                  bitrate_rmse);
+      RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "H264",
+                                  average_overshoot_percent);
+      break;
+    case VideoCodecType::kVideoCodecGeneric:
+    case VideoCodecType::kVideoCodecMultiplex:
+      break;
+  }
+}
+
 }  // namespace webrtc

video/encoder_overshoot_detector.h

@@ -15,12 +15,15 @@
 
 #include "absl/types/optional.h"
 #include "api/units/data_rate.h"
+#include "api/video_codecs/video_codec.h"
 
 namespace webrtc {
 
 class EncoderOvershootDetector {
  public:
-  explicit EncoderOvershootDetector(int64_t window_size_ms);
+  explicit EncoderOvershootDetector(int64_t window_size_ms,
+                                    VideoCodecType codec,
+                                    bool is_screenshare);
   ~EncoderOvershootDetector();
 
   void SetTargetRate(DataRate target_bitrate,
@@ -64,6 +67,7 @@ class EncoderOvershootDetector {
     double media_utilization_factor;
     int64_t update_time_ms;
   };
+  void UpdateHistograms();
   std::deque<BitrateUpdate> utilization_factors_;
   double sum_network_utilization_factors_;
   double sum_media_utilization_factors_;
@@ -71,6 +75,11 @@ class EncoderOvershootDetector {
   double target_framerate_fps_;
   int64_t network_buffer_level_bits_;
   int64_t media_buffer_level_bits_;
+  VideoCodecType codec_;
+  bool is_screenshare_;
+  int64_t frame_count_;
+  int64_t sum_diff_kbps_squared_;
+  int64_t sum_overshoot_percent_;
 };
 
 }  // namespace webrtc

video/encoder_overshoot_detector_unittest.cc

@@ -10,23 +10,58 @@
 
 #include "video/encoder_overshoot_detector.h"
 
+#include <string>
+
 #include "api/units/data_rate.h"
 #include "rtc_base/fake_clock.h"
 #include "rtc_base/time_utils.h"
+#include "system_wrappers/include/metrics.h"
 #include "test/gtest.h"
 
 namespace webrtc {
 
-class EncoderOvershootDetectorTest : public ::testing::Test {
+namespace {
+
+using ::testing::TestWithParam;
+using ::testing::ValuesIn;
+
+static std::string CodecTypeToHistogramSuffix(VideoCodecType codec) {
+  switch (codec) {
+    case kVideoCodecVP8:
+      return "Vp8";
+    case kVideoCodecVP9:
+      return "Vp9";
+    case kVideoCodecAV1:
+      return "Av1";
+    case kVideoCodecH264:
+      return "H264";
+    case kVideoCodecGeneric:
+      return "Generic";
+    case kVideoCodecMultiplex:
+      return "Multiplex";
+  }
+}
+
+struct TestParams {
+  VideoCodecType codec_type;
+  bool is_screenshare;
+};
+
+}  // namespace
+
+class EncoderOvershootDetectorTest : public TestWithParam<TestParams> {
  public:
   static constexpr int kDefaultBitrateBps = 300000;
   static constexpr double kDefaultFrameRateFps = 15;
   EncoderOvershootDetectorTest()
-      : detector_(kWindowSizeMs),
+      : detector_(kWindowSizeMs,
+                  GetParam().codec_type,
+                  GetParam().is_screenshare),
         target_bitrate_(DataRate::BitsPerSec(kDefaultBitrateBps)),
         target_framerate_fps_(kDefaultFrameRateFps) {}
 
  protected:
+  void SetUp() override { metrics::Reset(); }
   void RunConstantUtilizationTest(double actual_utilization_factor,
                                   double expected_utilization_factor,
                                   double allowed_error,
@@ -70,7 +105,7 @@ class EncoderOvershootDetectorTest : public ::testing::Test {
   double target_framerate_fps_;
 };
 
-TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) {
+TEST_P(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) {
   const int frame_size_bytes = 1000;
   const int64_t time_interval_ms = 33;
   detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
@@ -86,26 +121,26 @@ TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) {
       detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()).has_value());
 }
 
-TEST_F(EncoderOvershootDetectorTest, OptimalSize) {
+TEST_P(EncoderOvershootDetectorTest, OptimalSize) {
   // Optimally behaved encoder.
   // Allow some error margin due to rounding errors, eg due to frame
   // interval not being an integer.
   RunConstantUtilizationTest(1.0, 1.0, 0.01, kWindowSizeMs);
 }
 
-TEST_F(EncoderOvershootDetectorTest, Undershoot) {
+TEST_P(EncoderOvershootDetectorTest, Undershoot) {
   // Undershoot, reported utilization factor should be capped to 1.0 so
   // that we don't incorrectly boost encoder bitrate during movement.
   RunConstantUtilizationTest(0.5, 1.0, 0.00, kWindowSizeMs);
 }
 
-TEST_F(EncoderOvershootDetectorTest, Overshoot) {
+TEST_P(EncoderOvershootDetectorTest, Overshoot) {
   // Overshoot by 20%.
   // Allow some error margin due to rounding errors.
   RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs);
 }
 
-TEST_F(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) {
+TEST_P(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) {
   // Overshoot by 20%, but vary framerate and bitrate.
   // Allow some error margin due to rounding errors.
   RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs);
@@ -115,7 +150,7 @@ TEST_F(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) {
   RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2);
 }
 
-TEST_F(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) {
+TEST_P(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) {
   // Overshoot by 10%, keep framerate and bitrate constant.
   // Allow some error margin due to rounding errors.
   RunConstantUtilizationTest(1.1, 1.1, 0.01, kWindowSizeMs);
@@ -127,7 +162,7 @@ TEST_F(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) {
   RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2);
 }
 
-TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) {
+TEST_P(EncoderOvershootDetectorTest, PartialOvershoot) {
   const int ideal_frame_size_bytes =
       (target_bitrate_.bps() / target_framerate_fps_) / 8;
   detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
@@ -163,4 +198,83 @@ TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) {
       detector_.GetMediaRateUtilizationFactor(rtc::TimeMillis());
   EXPECT_NEAR(media_utilization_factor.value_or(-1), 1.00, 0.01);
 }
+
+TEST_P(EncoderOvershootDetectorTest, RecordsZeroErrorMetricWithNoOvershoot) {
+  DataSize ideal_frame_size =
+      target_bitrate_ / Frequency::Hertz(target_framerate_fps_);
+  detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
+                          rtc::TimeMillis());
+  detector_.OnEncodedFrame(ideal_frame_size.bytes(), rtc::TimeMillis());
+  detector_.Reset();
+
+  const VideoCodecType codec = GetParam().codec_type;
+  const bool is_screenshare = GetParam().is_screenshare;
+  const std::string rmse_histogram_prefix =
+      is_screenshare ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps."
+                     : "WebRTC.Video.RMSEOfEncodingBitrateInKbps.";
+  const std::string overshoot_histogram_prefix =
+      is_screenshare ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot."
+                     : "WebRTC.Video.EncodingBitrateOvershoot.";
+  // RMSE and overshoot percent = 0, since we used ideal frame size.
+  EXPECT_METRIC_EQ(1, metrics::NumSamples(rmse_histogram_prefix +
+                                          CodecTypeToHistogramSuffix(codec)));
+  EXPECT_METRIC_EQ(
+      1, metrics::NumEvents(
+             rmse_histogram_prefix + CodecTypeToHistogramSuffix(codec), 0));
+
+  EXPECT_METRIC_EQ(1, metrics::NumSamples(overshoot_histogram_prefix +
+                                          CodecTypeToHistogramSuffix(codec)));
+  EXPECT_METRIC_EQ(1, metrics::NumEvents(overshoot_histogram_prefix +
+                                             CodecTypeToHistogramSuffix(codec),
+                                         0));
+}
+
+TEST_P(EncoderOvershootDetectorTest,
+       RecordScreenshareZeroMetricWithNoOvershoot) {
+  DataSize ideal_frame_size =
+      target_bitrate_ / Frequency::Hertz(target_framerate_fps_);
+  // Use target frame size with 50% overshoot.
+  DataSize target_frame_size = ideal_frame_size * 3 / 2;
+  detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
+                          rtc::TimeMillis());
+  detector_.OnEncodedFrame(target_frame_size.bytes(), rtc::TimeMillis());
+  detector_.Reset();
+
+  const VideoCodecType codec = GetParam().codec_type;
+  const bool is_screenshare = GetParam().is_screenshare;
+  const std::string rmse_histogram_prefix =
+      is_screenshare ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps."
+                     : "WebRTC.Video.RMSEOfEncodingBitrateInKbps.";
+  const std::string overshoot_histogram_prefix =
+      is_screenshare ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot."
+                     : "WebRTC.Video.EncodingBitrateOvershoot.";
+  // Use ideal_frame_size_kbits to represnt ideal_frame_size.bytes()*8/1000,
+  // then rmse_in_kbps = ideal_frame_size_kbits/2
+  // since we use target frame size with 50% overshoot.
+  int64_t rmse_in_kbps = ideal_frame_size.bytes() * 8 / 1000 / 2;
+  EXPECT_METRIC_EQ(1, metrics::NumSamples(rmse_histogram_prefix +
+                                          CodecTypeToHistogramSuffix(codec)));
+  EXPECT_METRIC_EQ(1, metrics::NumEvents(rmse_histogram_prefix +
+                                             CodecTypeToHistogramSuffix(codec),
+                                         rmse_in_kbps));
+  // overshoot percent = 50, since we used ideal_frame_size * 3 / 2;
+  EXPECT_METRIC_EQ(1, metrics::NumSamples(overshoot_histogram_prefix +
+                                          CodecTypeToHistogramSuffix(codec)));
+  EXPECT_METRIC_EQ(1, metrics::NumEvents(overshoot_histogram_prefix +
+                                             CodecTypeToHistogramSuffix(codec),
+                                         50));
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    PerCodecType,
+    EncoderOvershootDetectorTest,
+    ValuesIn<TestParams>({{VideoCodecType::kVideoCodecVP8, false},
+                          {VideoCodecType::kVideoCodecVP8, true},
+                          {VideoCodecType::kVideoCodecVP9, false},
+                          {VideoCodecType::kVideoCodecVP9, true},
+                          {VideoCodecType::kVideoCodecAV1, false},
+                          {VideoCodecType::kVideoCodecAV1, true},
+                          {VideoCodecType::kVideoCodecH264, false},
+                          {VideoCodecType::kVideoCodecH264, true}}));
+
 }  // namespace webrtc