From 06defc432053601353d7786098d10cf1441d7b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Fri, 10 Sep 2021 15:28:48 +0200 Subject: [PATCH] QualityRampupExperiment: SetMaxBitrate may not be set correctly. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Call SetMaxBitrate when encoder is configured instead of in OnMaybeEncodeFrame (which is called after the initial frame dropping -> max bitrate is not set for dropped frames). Added support for single active stream configuration. Bug: none Change-Id: I33ff96e7feed70b9ea3c9b3da89f117859108347 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/231681 Reviewed-by: Sergey Silkin Commit-Queue: Åsa Persson Cr-Commit-Position: refs/heads/main@{#34973} --- .../experiments/quality_rampup_experiment.cc | 7 ++- .../experiments/quality_rampup_experiment.h | 1 + .../quality_rampup_experiment_helper.cc | 18 +++++--- .../quality_rampup_experiment_helper.h | 7 ++- .../video_stream_encoder_resource_manager.cc | 43 +++++++++++++++++-- video/video_stream_encoder_unittest.cc | 24 ++++++++--- 6 files changed, 82 insertions(+), 18 deletions(-) diff --git a/rtc_base/experiments/quality_rampup_experiment.cc b/rtc_base/experiments/quality_rampup_experiment.cc index ee6675c924..35c83f7011 100644 --- a/rtc_base/experiments/quality_rampup_experiment.cc +++ b/rtc_base/experiments/quality_rampup_experiment.cc @@ -70,8 +70,13 @@ bool QualityRampupExperiment::BwHigh(int64_t now_ms, return (now_ms - *start_ms_) >= min_duration_ms_.Value(); } +void QualityRampupExperiment::Reset() { + start_ms_.reset(); + max_bitrate_kbps_.reset(); +} + bool QualityRampupExperiment::Enabled() const { - return min_pixels_ || min_duration_ms_ || max_bitrate_kbps_; + return min_pixels_ && min_duration_ms_; } } // namespace webrtc diff --git a/rtc_base/experiments/quality_rampup_experiment.h b/rtc_base/experiments/quality_rampup_experiment.h index 78556ebda9..719b1893f6 100644 --- a/rtc_base/experiments/quality_rampup_experiment.h +++ b/rtc_base/experiments/quality_rampup_experiment.h @@ -33,6 +33,7 @@ class QualityRampupExperiment final { // (max_bitrate_factor_) above `max_bitrate_kbps_` for `min_duration_ms_`. bool BwHigh(int64_t now_ms, uint32_t available_bw_kbps); + void Reset(); bool Enabled() const; private: diff --git a/video/adaptation/quality_rampup_experiment_helper.cc b/video/adaptation/quality_rampup_experiment_helper.cc index 6d82503fc6..adcad40c03 100644 --- a/video/adaptation/quality_rampup_experiment_helper.cc +++ b/video/adaptation/quality_rampup_experiment_helper.cc @@ -43,22 +43,30 @@ QualityRampUpExperimentHelper::CreateIfEnabled( return nullptr; } +void QualityRampUpExperimentHelper::ConfigureQualityRampupExperiment( + bool reset, + absl::optional pixels, + absl::optional max_bitrate) { + if (reset) + quality_rampup_experiment_.Reset(); + if (pixels && max_bitrate) + quality_rampup_experiment_.SetMaxBitrate(*pixels, max_bitrate->kbps()); +} + void QualityRampUpExperimentHelper::PerformQualityRampupExperiment( rtc::scoped_refptr quality_scaler_resource, DataRate bandwidth, DataRate encoder_target_bitrate, - DataRate max_bitrate, - int pixels) { - if (!quality_scaler_resource->is_started()) + absl::optional max_bitrate) { + if (!quality_scaler_resource->is_started() || !max_bitrate) return; int64_t now_ms = clock_->TimeInMilliseconds(); - quality_rampup_experiment_.SetMaxBitrate(pixels, max_bitrate.kbps()); bool try_quality_rampup = false; if (quality_rampup_experiment_.BwHigh(now_ms, bandwidth.kbps())) { // Verify that encoder is at max bitrate and the QP is low. - if (encoder_target_bitrate == max_bitrate && + if (encoder_target_bitrate == *max_bitrate && quality_scaler_resource->QpFastFilterLow()) { try_quality_rampup = true; } diff --git a/video/adaptation/quality_rampup_experiment_helper.h b/video/adaptation/quality_rampup_experiment_helper.h index 81be982e7c..4fe1f24876 100644 --- a/video/adaptation/quality_rampup_experiment_helper.h +++ b/video/adaptation/quality_rampup_experiment_helper.h @@ -44,12 +44,15 @@ class QualityRampUpExperimentHelper { void cpu_adapted(bool cpu_adapted); void qp_resolution_adaptations(int qp_adaptations); + void ConfigureQualityRampupExperiment(bool reset, + absl::optional pixels, + absl::optional max_bitrate); + void PerformQualityRampupExperiment( rtc::scoped_refptr quality_scaler_resource, DataRate bandwidth, DataRate encoder_target_bitrate, - DataRate max_bitrate, - int pixels); + absl::optional max_bitrate); private: QualityRampUpExperimentHelper( diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 23a3593625..0b2fa89339 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -88,6 +88,30 @@ bool EqualFlags(const std::vector& a, const std::vector& b) { return std::equal(a.begin(), a.end(), b.begin()); } +absl::optional GetSingleActiveLayerMaxBitrate( + const VideoCodec& codec) { + int num_active = 0; + absl::optional max_bitrate; + if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { + if (codec.spatialLayers[i].active) { + ++num_active; + max_bitrate = + DataRate::KilobitsPerSec(codec.spatialLayers[i].maxBitrate); + } + } + } else { + for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { + if (codec.simulcastStream[i].active) { + ++num_active; + max_bitrate = + DataRate::KilobitsPerSec(codec.simulcastStream[i].maxBitrate); + } + } + } + return (num_active > 1) ? absl::nullopt : max_bitrate; +} + } // namespace class VideoStreamEncoderResourceManager::InitialFrameDropper { @@ -103,7 +127,8 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { use_bandwidth_allocation_(false), bandwidth_allocation_(DataRate::Zero()), last_input_width_(0), - last_input_height_(0) { + last_input_height_(0), + last_stream_configuration_changed_(false) { RTC_DCHECK(quality_scaler_resource_); } @@ -123,6 +148,10 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { : absl::nullopt; } + bool last_stream_configuration_changed() const { + return last_stream_configuration_changed_; + } + // Input signals. void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) { set_start_bitrate_ = start_bitrate; @@ -156,6 +185,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { void OnEncoderSettingsUpdated( const VideoCodec& codec, const VideoAdaptationCounters& adaptation_counters) { + last_stream_configuration_changed_ = false; std::vector active_flags = GetActiveLayersFlags(codec); // Check if the source resolution has changed for the external reasons, // i.e. without any adaptation from WebRTC. @@ -167,6 +197,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { if (!EqualFlags(active_flags, last_active_flags_) || source_resolution_changed) { // Streams configuration has changed. + last_stream_configuration_changed_ = true; // Initial frame drop must be enabled because BWE might be way too low // for the selected resolution. if (quality_scaler_resource_->is_started()) { @@ -226,6 +257,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { VideoAdaptationCounters last_adaptation_counters_; int last_input_width_; int last_input_height_; + bool last_stream_configuration_changed_; }; VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( @@ -394,6 +426,12 @@ void VideoStreamEncoderResourceManager::SetEncoderSettings( initial_frame_dropper_->OnEncoderSettingsUpdated( encoder_settings_->video_codec(), current_adaptation_counters_); MaybeUpdateTargetFrameRate(); + if (quality_rampup_experiment_) { + quality_rampup_experiment_->ConfigureQualityRampupExperiment( + initial_frame_dropper_->last_stream_configuration_changed(), + initial_frame_dropper_->single_active_stream_pixels(), + GetSingleActiveLayerMaxBitrate(encoder_settings_->video_codec())); + } } void VideoStreamEncoderResourceManager::SetStartBitrate( @@ -497,8 +535,7 @@ void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() { quality_rampup_experiment_->PerformQualityRampupExperiment( quality_scaler_resource_, bandwidth, DataRate::BitsPerSec(encoder_target_bitrate_bps_.value_or(0)), - DataRate::KilobitsPerSec(encoder_settings_->video_codec().maxBitrate), - LastFrameSizeOrDefault()); + GetSingleActiveLayerMaxBitrate(encoder_settings_->video_codec())); } } diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b9c6f8f10e..c1dc8a3be8 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -6140,7 +6140,13 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { webrtc::test::ScopedFieldTrials field_trials( - "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/"); + "WebRTC-Video-QualityRampupSettings/" + "min_pixels:921600,min_duration_ms:2000/"); + + const int kWidth = 1280; + const int kHeight = 720; + const int kFps = 10; + max_framerate_ = kFps; // Reset encoder for field trials to take effect. VideoEncoderConfig config = video_encoder_config_.Copy(); @@ -6163,9 +6169,7 @@ TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0); // Expect first frame to be dropped and resolution to be limited. - const int kWidth = 1280; - const int kHeight = 720; - const int64_t kFrameIntervalMs = 100; + const int64_t kFrameIntervalMs = 1000 / kFps; int64_t timestamp_ms = kFrameIntervalMs; source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight)); ExpectDroppedFrame(); @@ -6177,17 +6181,23 @@ TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { max_bitrate, max_bitrate, max_bitrate, 0, 0, 0); // Insert frames and advance `min_duration_ms`. + const int64_t start_bw_high_ms = CurrentTimeMs(); for (size_t i = 1; i <= 10; i++) { timestamp_ms += kFrameIntervalMs; source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight)); WaitForEncodedFrame(timestamp_ms); } + + // Advance to `min_duration_ms` - 1, frame should not trigger high BW. + int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms; + AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1)); + timestamp_ms += kFrameIntervalMs; + source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight)); + WaitForEncodedFrame(timestamp_ms); EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution); EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight); - AdvanceTime(TimeDelta::Millis(2000)); - - // Insert frame should trigger high BW and release quality limitation. + // Frame should trigger high BW and release quality limitation. timestamp_ms += kFrameIntervalMs; source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight)); WaitForEncodedFrame(timestamp_ms);