diff --git a/AUTHORS b/AUTHORS index b3cbb4db35..83a968d326 100644 --- a/AUTHORS +++ b/AUTHORS @@ -133,6 +133,7 @@ Yura Yaroshevich Yuriy Pavlyshak Yusuke Suzuki Pengfei Han +Yingying Ma # END individuals section. # BEGIN organizations section. diff --git a/video/encoder_bitrate_adjuster.cc b/video/encoder_bitrate_adjuster.cc index c672173895..465d517d21 100644 --- a/video/encoder_bitrate_adjuster.cc +++ b/video/encoder_bitrate_adjuster.cc @@ -47,7 +47,9 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings) : utilize_bandwidth_headroom_(RateControlSettings::ParseFromFieldTrials() .BitrateAdjusterCanUseNetworkHeadroom()), frames_since_layout_change_(0), - min_bitrates_bps_{} { + min_bitrates_bps_{}, + codec_(codec_settings.codecType), + codec_mode_(codec_settings.mode) { // TODO(https://crbug.com/webrtc/14891): If we want to support simulcast of // SVC streams, EncoderBitrateAdjuster needs to be updated to care about both // `simulcastStream` and `spatialLayers` at the same time. @@ -90,7 +92,9 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( ++active_tls[si]; if (!overshoot_detectors_[si][ti]) { overshoot_detectors_[si][ti] = - std::make_unique(kWindowSizeMs); + std::make_unique( + kWindowSizeMs, codec_, + codec_mode_ == VideoCodecMode::kScreensharing); frames_since_layout_change_ = 0; } } else if (overshoot_detectors_[si][ti]) { diff --git a/video/encoder_bitrate_adjuster.h b/video/encoder_bitrate_adjuster.h index 6dc3e5534f..6b35186b98 100644 --- a/video/encoder_bitrate_adjuster.h +++ b/video/encoder_bitrate_adjuster.h @@ -73,6 +73,12 @@ class EncoderBitrateAdjuster { // Minimum bitrates allowed, per spatial layer. uint32_t min_bitrates_bps_[kMaxSpatialLayers]; + + // Codec type used for encoding. + VideoCodecType codec_; + + // Codec mode: { kRealtimeVideo, kScreensharing }. + VideoCodecMode codec_mode_; }; } // namespace webrtc diff --git a/video/encoder_overshoot_detector.cc b/video/encoder_overshoot_detector.cc index 80b2ec12b0..2c4efdb5a6 100644 --- a/video/encoder_overshoot_detector.cc +++ b/video/encoder_overshoot_detector.cc @@ -11,6 +11,9 @@ #include "video/encoder_overshoot_detector.h" #include +#include + +#include "system_wrappers/include/metrics.h" namespace webrtc { namespace { @@ -20,7 +23,9 @@ namespace { static constexpr double kMaxMediaUnderrunFrames = 5.0; } // namespace -EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms) +EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms, + VideoCodecType codec, + bool is_screenshare) : window_size_ms_(window_size_ms), time_last_update_ms_(-1), sum_network_utilization_factors_(0.0), @@ -28,9 +33,16 @@ EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms) target_bitrate_(DataRate::Zero()), target_framerate_fps_(0), network_buffer_level_bits_(0), - media_buffer_level_bits_(0) {} + media_buffer_level_bits_(0), + codec_(codec), + is_screenshare_(is_screenshare), + frame_count_(0), + sum_diff_kbps_squared_(0), + sum_overshoot_percent_(0) {} -EncoderOvershootDetector::~EncoderOvershootDetector() = default; +EncoderOvershootDetector::~EncoderOvershootDetector() { + UpdateHistograms(); +} void EncoderOvershootDetector::SetTargetRate(DataRate target_bitrate, double target_framerate_fps, @@ -57,6 +69,7 @@ void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) { // bitrate. LeakBits(time_ms); + const int64_t frame_size_bits = bytes * 8; // Ideal size of a frame given the current rates. const int64_t ideal_frame_size_bits = IdealFrameSizeBits(); if (ideal_frame_size_bits == 0) { @@ -64,14 +77,22 @@ void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) { return; } - const double network_utilization_factor = HandleEncodedFrame( - bytes * 8, ideal_frame_size_bits, time_ms, &network_buffer_level_bits_); - const double media_utilization_factor = HandleEncodedFrame( - bytes * 8, ideal_frame_size_bits, time_ms, &media_buffer_level_bits_); + const double network_utilization_factor = + HandleEncodedFrame(frame_size_bits, ideal_frame_size_bits, time_ms, + &network_buffer_level_bits_); + const double media_utilization_factor = + HandleEncodedFrame(frame_size_bits, ideal_frame_size_bits, time_ms, + &media_buffer_level_bits_); sum_network_utilization_factors_ += network_utilization_factor; sum_media_utilization_factors_ += media_utilization_factor; + // Calculate the bitrate diff in kbps + int64_t diff_kbits = (frame_size_bits - ideal_frame_size_bits) / 1000; + sum_diff_kbps_squared_ += diff_kbits * diff_kbits; + sum_overshoot_percent_ += diff_kbits * 100 * 1000 / ideal_frame_size_bits; + ++frame_count_; + utilization_factors_.emplace_back(network_utilization_factor, media_utilization_factor, time_ms); } @@ -142,6 +163,10 @@ absl::optional EncoderOvershootDetector::GetMediaRateUtilizationFactor( } void EncoderOvershootDetector::Reset() { + UpdateHistograms(); + sum_diff_kbps_squared_ = 0; + frame_count_ = 0; + sum_overshoot_percent_ = 0; time_last_update_ms_ = -1; utilization_factors_.clear(); target_bitrate_ = DataRate::Zero(); @@ -201,4 +226,50 @@ void EncoderOvershootDetector::CullOldUpdates(int64_t time_ms) { } } +void EncoderOvershootDetector::UpdateHistograms() { + if (frame_count_ == 0) + return; + + int64_t bitrate_rmse = std::sqrt(sum_diff_kbps_squared_ / frame_count_); + int64_t average_overshoot_percent = sum_overshoot_percent_ / frame_count_; + const std::string rmse_histogram_prefix = + is_screenshare_ ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps." + : "WebRTC.Video.RMSEOfEncodingBitrateInKbps."; + const std::string overshoot_histogram_prefix = + is_screenshare_ ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot." + : "WebRTC.Video.EncodingBitrateOvershoot."; + // index = 1 represents screensharing histograms recording. + // index = 0 represents normal video histograms recording. + const int index = is_screenshare_ ? 1 : 0; + switch (codec_) { + case VideoCodecType::kVideoCodecAV1: + RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Av1", + bitrate_rmse); + RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Av1", + average_overshoot_percent); + break; + case VideoCodecType::kVideoCodecVP9: + RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Vp9", + bitrate_rmse); + RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Vp9", + average_overshoot_percent); + break; + case VideoCodecType::kVideoCodecVP8: + RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Vp8", + bitrate_rmse); + RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Vp8", + average_overshoot_percent); + break; + case VideoCodecType::kVideoCodecH264: + RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "H264", + bitrate_rmse); + RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "H264", + average_overshoot_percent); + break; + case VideoCodecType::kVideoCodecGeneric: + case VideoCodecType::kVideoCodecMultiplex: + break; + } +} + } // namespace webrtc diff --git a/video/encoder_overshoot_detector.h b/video/encoder_overshoot_detector.h index 1f8908e54f..12c4bba5db 100644 --- a/video/encoder_overshoot_detector.h +++ b/video/encoder_overshoot_detector.h @@ -15,12 +15,15 @@ #include "absl/types/optional.h" #include "api/units/data_rate.h" +#include "api/video_codecs/video_codec.h" namespace webrtc { class EncoderOvershootDetector { public: - explicit EncoderOvershootDetector(int64_t window_size_ms); + explicit EncoderOvershootDetector(int64_t window_size_ms, + VideoCodecType codec, + bool is_screenshare); ~EncoderOvershootDetector(); void SetTargetRate(DataRate target_bitrate, @@ -64,6 +67,7 @@ class EncoderOvershootDetector { double media_utilization_factor; int64_t update_time_ms; }; + void UpdateHistograms(); std::deque utilization_factors_; double sum_network_utilization_factors_; double sum_media_utilization_factors_; @@ -71,6 +75,11 @@ class EncoderOvershootDetector { double target_framerate_fps_; int64_t network_buffer_level_bits_; int64_t media_buffer_level_bits_; + VideoCodecType codec_; + bool is_screenshare_; + int64_t frame_count_; + int64_t sum_diff_kbps_squared_; + int64_t sum_overshoot_percent_; }; } // namespace webrtc diff --git a/video/encoder_overshoot_detector_unittest.cc b/video/encoder_overshoot_detector_unittest.cc index a3c44eb013..bdc2676281 100644 --- a/video/encoder_overshoot_detector_unittest.cc +++ b/video/encoder_overshoot_detector_unittest.cc @@ -10,23 +10,58 @@ #include "video/encoder_overshoot_detector.h" +#include + #include "api/units/data_rate.h" #include "rtc_base/fake_clock.h" #include "rtc_base/time_utils.h" +#include "system_wrappers/include/metrics.h" #include "test/gtest.h" namespace webrtc { -class EncoderOvershootDetectorTest : public ::testing::Test { +namespace { + +using ::testing::TestWithParam; +using ::testing::ValuesIn; + +static std::string CodecTypeToHistogramSuffix(VideoCodecType codec) { + switch (codec) { + case kVideoCodecVP8: + return "Vp8"; + case kVideoCodecVP9: + return "Vp9"; + case kVideoCodecAV1: + return "Av1"; + case kVideoCodecH264: + return "H264"; + case kVideoCodecGeneric: + return "Generic"; + case kVideoCodecMultiplex: + return "Multiplex"; + } +} + +struct TestParams { + VideoCodecType codec_type; + bool is_screenshare; +}; + +} // namespace + +class EncoderOvershootDetectorTest : public TestWithParam { public: static constexpr int kDefaultBitrateBps = 300000; static constexpr double kDefaultFrameRateFps = 15; EncoderOvershootDetectorTest() - : detector_(kWindowSizeMs), + : detector_(kWindowSizeMs, + GetParam().codec_type, + GetParam().is_screenshare), target_bitrate_(DataRate::BitsPerSec(kDefaultBitrateBps)), target_framerate_fps_(kDefaultFrameRateFps) {} protected: + void SetUp() override { metrics::Reset(); } void RunConstantUtilizationTest(double actual_utilization_factor, double expected_utilization_factor, double allowed_error, @@ -70,7 +105,7 @@ class EncoderOvershootDetectorTest : public ::testing::Test { double target_framerate_fps_; }; -TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) { +TEST_P(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) { const int frame_size_bytes = 1000; const int64_t time_interval_ms = 33; detector_.SetTargetRate(target_bitrate_, target_framerate_fps_, @@ -86,26 +121,26 @@ TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) { detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()).has_value()); } -TEST_F(EncoderOvershootDetectorTest, OptimalSize) { +TEST_P(EncoderOvershootDetectorTest, OptimalSize) { // Optimally behaved encoder. // Allow some error margin due to rounding errors, eg due to frame // interval not being an integer. RunConstantUtilizationTest(1.0, 1.0, 0.01, kWindowSizeMs); } -TEST_F(EncoderOvershootDetectorTest, Undershoot) { +TEST_P(EncoderOvershootDetectorTest, Undershoot) { // Undershoot, reported utilization factor should be capped to 1.0 so // that we don't incorrectly boost encoder bitrate during movement. RunConstantUtilizationTest(0.5, 1.0, 0.00, kWindowSizeMs); } -TEST_F(EncoderOvershootDetectorTest, Overshoot) { +TEST_P(EncoderOvershootDetectorTest, Overshoot) { // Overshoot by 20%. // Allow some error margin due to rounding errors. RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs); } -TEST_F(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) { +TEST_P(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) { // Overshoot by 20%, but vary framerate and bitrate. // Allow some error margin due to rounding errors. RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs); @@ -115,7 +150,7 @@ TEST_F(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) { RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2); } -TEST_F(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) { +TEST_P(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) { // Overshoot by 10%, keep framerate and bitrate constant. // Allow some error margin due to rounding errors. RunConstantUtilizationTest(1.1, 1.1, 0.01, kWindowSizeMs); @@ -127,7 +162,7 @@ TEST_F(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) { RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2); } -TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) { +TEST_P(EncoderOvershootDetectorTest, PartialOvershoot) { const int ideal_frame_size_bytes = (target_bitrate_.bps() / target_framerate_fps_) / 8; detector_.SetTargetRate(target_bitrate_, target_framerate_fps_, @@ -163,4 +198,83 @@ TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) { detector_.GetMediaRateUtilizationFactor(rtc::TimeMillis()); EXPECT_NEAR(media_utilization_factor.value_or(-1), 1.00, 0.01); } + +TEST_P(EncoderOvershootDetectorTest, RecordsZeroErrorMetricWithNoOvershoot) { + DataSize ideal_frame_size = + target_bitrate_ / Frequency::Hertz(target_framerate_fps_); + detector_.SetTargetRate(target_bitrate_, target_framerate_fps_, + rtc::TimeMillis()); + detector_.OnEncodedFrame(ideal_frame_size.bytes(), rtc::TimeMillis()); + detector_.Reset(); + + const VideoCodecType codec = GetParam().codec_type; + const bool is_screenshare = GetParam().is_screenshare; + const std::string rmse_histogram_prefix = + is_screenshare ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps." + : "WebRTC.Video.RMSEOfEncodingBitrateInKbps."; + const std::string overshoot_histogram_prefix = + is_screenshare ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot." + : "WebRTC.Video.EncodingBitrateOvershoot."; + // RMSE and overshoot percent = 0, since we used ideal frame size. + EXPECT_METRIC_EQ(1, metrics::NumSamples(rmse_histogram_prefix + + CodecTypeToHistogramSuffix(codec))); + EXPECT_METRIC_EQ( + 1, metrics::NumEvents( + rmse_histogram_prefix + CodecTypeToHistogramSuffix(codec), 0)); + + EXPECT_METRIC_EQ(1, metrics::NumSamples(overshoot_histogram_prefix + + CodecTypeToHistogramSuffix(codec))); + EXPECT_METRIC_EQ(1, metrics::NumEvents(overshoot_histogram_prefix + + CodecTypeToHistogramSuffix(codec), + 0)); +} + +TEST_P(EncoderOvershootDetectorTest, + RecordScreenshareZeroMetricWithNoOvershoot) { + DataSize ideal_frame_size = + target_bitrate_ / Frequency::Hertz(target_framerate_fps_); + // Use target frame size with 50% overshoot. + DataSize target_frame_size = ideal_frame_size * 3 / 2; + detector_.SetTargetRate(target_bitrate_, target_framerate_fps_, + rtc::TimeMillis()); + detector_.OnEncodedFrame(target_frame_size.bytes(), rtc::TimeMillis()); + detector_.Reset(); + + const VideoCodecType codec = GetParam().codec_type; + const bool is_screenshare = GetParam().is_screenshare; + const std::string rmse_histogram_prefix = + is_screenshare ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps." + : "WebRTC.Video.RMSEOfEncodingBitrateInKbps."; + const std::string overshoot_histogram_prefix = + is_screenshare ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot." + : "WebRTC.Video.EncodingBitrateOvershoot."; + // Use ideal_frame_size_kbits to represnt ideal_frame_size.bytes()*8/1000, + // then rmse_in_kbps = ideal_frame_size_kbits/2 + // since we use target frame size with 50% overshoot. + int64_t rmse_in_kbps = ideal_frame_size.bytes() * 8 / 1000 / 2; + EXPECT_METRIC_EQ(1, metrics::NumSamples(rmse_histogram_prefix + + CodecTypeToHistogramSuffix(codec))); + EXPECT_METRIC_EQ(1, metrics::NumEvents(rmse_histogram_prefix + + CodecTypeToHistogramSuffix(codec), + rmse_in_kbps)); + // overshoot percent = 50, since we used ideal_frame_size * 3 / 2; + EXPECT_METRIC_EQ(1, metrics::NumSamples(overshoot_histogram_prefix + + CodecTypeToHistogramSuffix(codec))); + EXPECT_METRIC_EQ(1, metrics::NumEvents(overshoot_histogram_prefix + + CodecTypeToHistogramSuffix(codec), + 50)); +} + +INSTANTIATE_TEST_SUITE_P( + PerCodecType, + EncoderOvershootDetectorTest, + ValuesIn({{VideoCodecType::kVideoCodecVP8, false}, + {VideoCodecType::kVideoCodecVP8, true}, + {VideoCodecType::kVideoCodecVP9, false}, + {VideoCodecType::kVideoCodecVP9, true}, + {VideoCodecType::kVideoCodecAV1, false}, + {VideoCodecType::kVideoCodecAV1, true}, + {VideoCodecType::kVideoCodecH264, false}, + {VideoCodecType::kVideoCodecH264, true}})); + } // namespace webrtc