diff --git a/rtc_base/experiments/quality_rampup_experiment.cc b/rtc_base/experiments/quality_rampup_experiment.cc index caf7e62368..ee6675c924 100644 --- a/rtc_base/experiments/quality_rampup_experiment.cc +++ b/rtc_base/experiments/quality_rampup_experiment.cc @@ -70,4 +70,8 @@ bool QualityRampupExperiment::BwHigh(int64_t now_ms, return (now_ms - *start_ms_) >= min_duration_ms_.Value(); } +bool QualityRampupExperiment::Enabled() const { + return min_pixels_ || min_duration_ms_ || max_bitrate_kbps_; +} + } // namespace webrtc diff --git a/rtc_base/experiments/quality_rampup_experiment.h b/rtc_base/experiments/quality_rampup_experiment.h index ff9d7d38e5..9d46901104 100644 --- a/rtc_base/experiments/quality_rampup_experiment.h +++ b/rtc_base/experiments/quality_rampup_experiment.h @@ -33,6 +33,8 @@ class QualityRampupExperiment final { // (max_bitrate_factor_) above |max_bitrate_kbps_| for |min_duration_ms_|. bool BwHigh(int64_t now_ms, uint32_t available_bw_kbps); + bool Enabled() const; + private: explicit QualityRampupExperiment( const WebRtcKeyValueConfig* const key_value_config); diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index 66187803e6..50c36101ce 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -14,6 +14,8 @@ rtc_library("video_adaptation") { "encode_usage_resource.h", "overuse_frame_detector.cc", "overuse_frame_detector.h", + "quality_rampup_experiment_helper.cc", + "quality_rampup_experiment_helper.h", "quality_scaler_resource.cc", "quality_scaler_resource.h", "video_stream_encoder_resource.cc", diff --git a/video/adaptation/quality_rampup_experiment_helper.cc b/video/adaptation/quality_rampup_experiment_helper.cc new file mode 100644 index 0000000000..1e04e5578e --- /dev/null +++ b/video/adaptation/quality_rampup_experiment_helper.cc @@ -0,0 +1,80 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/quality_rampup_experiment_helper.h" + +#include +#include + +#include "rtc_base/logging.h" + +namespace webrtc { + +QualityRampUpExperimentHelper::QualityRampUpExperimentHelper( + QualityRampUpExperimentListener* experiment_listener, + Clock* clock, + QualityRampupExperiment experiment) + : experiment_listener_(experiment_listener), + clock_(clock), + quality_rampup_experiment_(std::move(experiment)), + cpu_adapted_(false), + qp_resolution_adaptations_(0) { + RTC_DCHECK(experiment_listener_); + RTC_DCHECK(clock_); +} + +std::unique_ptr +QualityRampUpExperimentHelper::CreateIfEnabled( + QualityRampUpExperimentListener* experiment_listener, + Clock* clock) { + QualityRampupExperiment experiment = QualityRampupExperiment::ParseSettings(); + if (experiment.Enabled()) { + return std::unique_ptr( + new QualityRampUpExperimentHelper(experiment_listener, clock, + experiment)); + } + return nullptr; +} + +void QualityRampUpExperimentHelper::PerformQualityRampupExperiment( + rtc::scoped_refptr quality_scaler_resource, + uint32_t bw_kbps, + uint32_t encoder_target_bitrate, + uint32_t max_bitrate_bps, + int pixels) { + if (!quality_scaler_resource->is_started()) + return; + + int64_t now_ms = clock_->TimeInMilliseconds(); + quality_rampup_experiment_.SetMaxBitrate(pixels, max_bitrate_bps); + + bool try_quality_rampup = false; + if (quality_rampup_experiment_.BwHigh(now_ms, bw_kbps)) { + // Verify that encoder is at max bitrate and the QP is low. + if (encoder_target_bitrate == max_bitrate_bps * 1000 && + quality_scaler_resource->QpFastFilterLow()) { + try_quality_rampup = true; + } + } + if (try_quality_rampup && qp_resolution_adaptations_ > 0 && !cpu_adapted_) { + experiment_listener_->OnQualityRampUp(); + } +} + +void QualityRampUpExperimentHelper::cpu_adapted(bool cpu_adapted) { + cpu_adapted_ = cpu_adapted; +} + +void QualityRampUpExperimentHelper::qp_resolution_adaptations( + int qp_resolution_adaptations) { + qp_resolution_adaptations_ = qp_resolution_adaptations; +} + +} // namespace webrtc diff --git a/video/adaptation/quality_rampup_experiment_helper.h b/video/adaptation/quality_rampup_experiment_helper.h new file mode 100644 index 0000000000..8b6a7e0ae3 --- /dev/null +++ b/video/adaptation/quality_rampup_experiment_helper.h @@ -0,0 +1,67 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_QUALITY_RAMPUP_EXPERIMENT_HELPER_H_ +#define VIDEO_ADAPTATION_QUALITY_RAMPUP_EXPERIMENT_HELPER_H_ + +#include + +#include "api/scoped_refptr.h" +#include "rtc_base/experiments/quality_rampup_experiment.h" +#include "system_wrappers/include/clock.h" +#include "video/adaptation/quality_scaler_resource.h" + +namespace webrtc { + +class QualityRampUpExperimentListener { + public: + virtual ~QualityRampUpExperimentListener() = default; + virtual void OnQualityRampUp() = 0; +}; + +// Helper class for orchestrating the WebRTC-Video-QualityRampupSettings +// experiment. +class QualityRampUpExperimentHelper { + public: + // Returns a QualityRampUpExperimentHelper if the experiment is enabled, + // an nullptr otherwise. + static std::unique_ptr CreateIfEnabled( + QualityRampUpExperimentListener* experiment_listener, + Clock* clock); + + QualityRampUpExperimentHelper(const QualityRampUpExperimentHelper&) = delete; + QualityRampUpExperimentHelper& operator=( + const QualityRampUpExperimentHelper&) = delete; + + void cpu_adapted(bool cpu_adapted); + void qp_resolution_adaptations(int qp_adaptations); + + void PerformQualityRampupExperiment( + rtc::scoped_refptr quality_scaler_resource, + uint32_t bw_kbps, + uint32_t encoder_target_bitrate, + uint32_t max_bitrate_bps, + int pixels); + + private: + QualityRampUpExperimentHelper( + QualityRampUpExperimentListener* experiment_listener, + Clock* clock, + QualityRampupExperiment experiment); + QualityRampUpExperimentListener* const experiment_listener_; + Clock* clock_; + QualityRampupExperiment quality_rampup_experiment_; + bool cpu_adapted_; + int qp_resolution_adaptations_; +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_QUALITY_RAMPUP_EXPERIMENT_HELPER_H_ diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 00ad45487a..2dfcc16440 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -285,10 +285,9 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( std::make_unique(quality_scaler_resource_)), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), encoder_target_bitrate_bps_(absl::nullopt), - quality_rampup_done_(false), - quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), - encoder_settings_(absl::nullopt), - active_counts_() { + quality_rampup_experiment_( + QualityRampUpExperimentHelper::CreateIfEnabled(this, clock_)), + encoder_settings_(absl::nullopt) { RTC_DCHECK(encoder_stats_observer_); MapResourceToReason(encode_usage_resource_, VideoAdaptationReason::kCpu); MapResourceToReason(quality_scaler_resource_, @@ -394,10 +393,6 @@ void VideoStreamEncoderResourceManager::SetEncoderSettings( RTC_DCHECK_RUN_ON(encoder_queue_); encoder_settings_ = std::move(encoder_settings); bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_); - - quality_rampup_experiment_.SetMaxBitrate( - LastInputFrameSizeOrDefault(), - encoder_settings_->video_codec().maxBitrate); MaybeUpdateTargetFrameRate(); } @@ -491,7 +486,16 @@ bool VideoStreamEncoderResourceManager::DropInitialFrames() const { void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() { RTC_DCHECK_RUN_ON(encoder_queue_); initial_frame_dropper_->OnMaybeEncodeFrame(); - MaybePerformQualityRampupExperiment(); + if (quality_rampup_experiment_) { + uint32_t bw_kbps = encoder_rates_.has_value() + ? encoder_rates_.value().bandwidth_allocation.kbps() + : 0; + quality_rampup_experiment_->PerformQualityRampupExperiment( + quality_scaler_resource_, bw_kbps, + encoder_target_bitrate_bps_.value_or(0), + encoder_settings_->video_codec().maxBitrate, + LastInputFrameSizeOrDefault()); + } } void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings( @@ -597,7 +601,7 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( // TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback. if (!reason && adaptation_counters.Total() == 0) { // Adaptation was manually reset - clear the per-reason counters too. - ResetActiveCounts(); + encoder_stats_observer_->ClearAdaptationStats(); } // The VideoStreamEncoder makes the manager outlive the encoder queue. This @@ -615,7 +619,7 @@ void VideoStreamEncoderResourceManager::OnResourceLimitationChanged( resource_limitations) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); if (!resource) { - ResetActiveCounts(); + encoder_stats_observer_->ClearAdaptationStats(); return; } @@ -631,13 +635,25 @@ void VideoStreamEncoderResourceManager::OnResourceLimitationChanged( } VideoAdaptationReason adaptation_reason = GetReasonFromResource(resource); - if (active_counts_[adaptation_reason] != limitations[adaptation_reason]) { - active_counts_[adaptation_reason] = limitations[adaptation_reason]; - encoder_stats_observer_->OnAdaptationChanged( - adaptation_reason, active_counts_[VideoAdaptationReason::kCpu], - active_counts_[VideoAdaptationReason::kQuality]); - } - RTC_LOG(LS_INFO) << ActiveCountsToString(); + encoder_stats_observer_->OnAdaptationChanged( + adaptation_reason, limitations[VideoAdaptationReason::kCpu], + limitations[VideoAdaptationReason::kQuality]); + + encoder_queue_->PostTask(ToQueuedTask( + [cpu_limited = limitations.at(VideoAdaptationReason::kCpu).Total() > 0, + qp_resolution_adaptations = + limitations.at(VideoAdaptationReason::kQuality) + .resolution_adaptations, + this]() { + RTC_DCHECK_RUN_ON(encoder_queue_); + if (quality_rampup_experiment_) { + quality_rampup_experiment_->cpu_adapted(cpu_limited); + quality_rampup_experiment_->qp_resolution_adaptations( + qp_resolution_adaptations); + } + })); + + RTC_LOG(LS_INFO) << ActiveCountsToString(limitations); } void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() { @@ -675,77 +691,19 @@ void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const { quality_settings); } -void VideoStreamEncoderResourceManager::MaybePerformQualityRampupExperiment() { - RTC_DCHECK_RUN_ON(encoder_queue_); - if (!quality_scaler_resource_->is_started()) - return; - - if (quality_rampup_done_) - return; - - int64_t now_ms = clock_->TimeInMilliseconds(); - uint32_t bw_kbps = encoder_rates_.has_value() - ? encoder_rates_.value().bandwidth_allocation.kbps() - : 0; - - bool try_quality_rampup = false; - if (quality_rampup_experiment_.BwHigh(now_ms, bw_kbps)) { - // Verify that encoder is at max bitrate and the QP is low. - if (encoder_settings_ && - encoder_target_bitrate_bps_.value_or(0) == - encoder_settings_->video_codec().maxBitrate * 1000 && - quality_scaler_resource_->QpFastFilterLow()) { - try_quality_rampup = true; - } - } - if (try_quality_rampup) { - // The VideoStreamEncoder makes the manager outlive the adaptation queue. - // This means that if the task gets executed, |this| has not been freed yet. - // TODO(https://crbug.com/webrtc/11565): When the manager no longer outlives - // the adaptation queue, add logic to prevent use-after-free on |this|. - resource_adaptation_queue_->PostTask([this] { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - if (!adaptation_processor_) { - // The processor nulled before this task had a chance to execute. This - // happens if the processor is destroyed. No action needed. - return; - } - // TODO(https://crbug.com/webrtc/11392): See if we can rely on the total - // counts or the stats, and not the active counts. - const VideoAdaptationCounters& qp_counts = - active_counts_[VideoAdaptationReason::kQuality]; - const VideoAdaptationCounters& cpu_counts = - active_counts_[VideoAdaptationReason::kCpu]; - if (!quality_rampup_done_ && qp_counts.resolution_adaptations > 0 && - cpu_counts.Total() == 0) { - RTC_LOG(LS_INFO) << "Reset quality limitations."; - adaptation_processor_->ResetVideoSourceRestrictions(); - quality_rampup_done_ = true; - } - }); - } -} - -void VideoStreamEncoderResourceManager::ResetActiveCounts() { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - active_counts_.clear(); - active_counts_[VideoAdaptationReason::kCpu] = VideoAdaptationCounters(); - active_counts_[VideoAdaptationReason::kQuality] = VideoAdaptationCounters(); - encoder_stats_observer_->ClearAdaptationStats(); -} - -std::string VideoStreamEncoderResourceManager::ActiveCountsToString() const { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - RTC_DCHECK_EQ(2, active_counts_.size()); +// static +std::string VideoStreamEncoderResourceManager::ActiveCountsToString( + const std::map& + active_counts) { rtc::StringBuilder ss; ss << "Downgrade counts: fps: {"; - for (auto& reason_count : active_counts_) { + for (auto& reason_count : active_counts) { ss << ToString(reason_count.first) << ":"; ss << reason_count.second.fps_adaptations; } ss << "}, resolution {"; - for (auto& reason_count : active_counts_) { + for (auto& reason_count : active_counts) { ss << ToString(reason_count.first) << ":"; ss << reason_count.second.resolution_adaptations; } @@ -753,4 +711,23 @@ std::string VideoStreamEncoderResourceManager::ActiveCountsToString() const { return ss.Release(); } + +void VideoStreamEncoderResourceManager::OnQualityRampUp() { + RTC_DCHECK_RUN_ON(encoder_queue_); + // The VideoStreamEncoder makes the manager outlive the adaptation queue. + // This means that if the task gets executed, |this| has not been freed yet. + // TODO(https://crbug.com/webrtc/11565): When the manager no longer outlives + // the adaptation queue, add logic to prevent use-after-free on |this|. + resource_adaptation_queue_->PostTask([this] { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + if (!adaptation_processor_) { + // The processor nulled before this task had a chance to execute. This + // happens if the processor is destroyed. No action needed. + return; + } + RTC_LOG(LS_INFO) << "Reset quality limitations."; + adaptation_processor_->ResetVideoSourceRestrictions(); + }); + quality_rampup_experiment_.reset(); +} } // namespace webrtc diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 666e0bbea2..61ae29b6bf 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -36,7 +36,6 @@ #include "call/adaptation/video_stream_adapter.h" #include "call/adaptation/video_stream_input_state_provider.h" #include "rtc_base/critical_section.h" -#include "rtc_base/experiments/quality_rampup_experiment.h" #include "rtc_base/experiments/quality_scaler_settings.h" #include "rtc_base/ref_count.h" #include "rtc_base/strings/string_builder.h" @@ -44,6 +43,7 @@ #include "system_wrappers/include/clock.h" #include "video/adaptation/encode_usage_resource.h" #include "video/adaptation/overuse_frame_detector.h" +#include "video/adaptation/quality_rampup_experiment_helper.h" #include "video/adaptation/quality_scaler_resource.h" #include "video/adaptation/video_stream_encoder_resource.h" @@ -64,7 +64,8 @@ extern const int kDefaultInputPixelsHeight; // The manager is also involved with various mitigations not part of the // ResourceAdaptationProcessor code such as the inital frame dropping. class VideoStreamEncoderResourceManager - : public VideoSourceRestrictionsListener { + : public VideoSourceRestrictionsListener, + public QualityRampUpExperimentListener { public: VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, @@ -112,10 +113,7 @@ class VideoStreamEncoderResourceManager void OnFrameDropped(EncodedImageCallback::DropReason reason); // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order - // to be able to update |active_counts_|, which is used... - // - Legacy getStats() purposes. - // - Preventing adapting up in some circumstances (which may be questionable). - // TODO(hbos): Can we get rid of this? + // to update legacy getStats(). void MapResourceToReason(rtc::scoped_refptr resource, VideoAdaptationReason reason); std::vector> MappedResources() const; @@ -128,7 +126,7 @@ class VideoStreamEncoderResourceManager bool DropInitialFrames() const; // VideoSourceRestrictionsListener implementation. - // Updates |video_source_restrictions_| and |active_counts_|. + // Updates |video_source_restrictions_|. void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, @@ -138,6 +136,9 @@ class VideoStreamEncoderResourceManager const std::map, VideoAdaptationCounters>& resource_limitations) override; + // QualityRampUpExperimentListener implementation. + void OnQualityRampUp() override; + private: class InitialFrameDropper; @@ -157,15 +158,9 @@ class VideoStreamEncoderResourceManager void UpdateStatsAdaptationSettings() const; - // Checks to see if we should execute the quality rampup experiment. The - // experiment resets all video restrictions at the start of the call in the - // case the bandwidth estimate is high enough. - // TODO(https://crbug.com/webrtc/11222) Move experiment details into an inner - // class. - void MaybePerformQualityRampupExperiment(); - - void ResetActiveCounts(); - std::string ActiveCountsToString() const; + static std::string ActiveCountsToString( + const std::map& + active_counts); // TODO(hbos): Add tests for manager's constraints. // Does not trigger adaptations, only prevents adapting up resolution. @@ -260,9 +255,7 @@ class VideoStreamEncoderResourceManager RTC_GUARDED_BY(encoder_queue_); absl::optional encoder_rates_ RTC_GUARDED_BY(encoder_queue_); - // Used on both the encoder queue and resource adaptation queue. - std::atomic quality_rampup_done_; - QualityRampupExperiment quality_rampup_experiment_ + std::unique_ptr quality_rampup_experiment_ RTC_GUARDED_BY(encoder_queue_); absl::optional encoder_settings_ RTC_GUARDED_BY(encoder_queue_); @@ -280,13 +273,6 @@ class VideoStreamEncoderResourceManager }; rtc::CriticalSection resource_lock_; std::vector resources_ RTC_GUARDED_BY(&resource_lock_); - // One AdaptationCounter for each reason, tracking the number of times we have - // adapted for each reason. The sum of active_counts_ MUST always equal the - // total adaptation provided by the VideoSourceRestrictions. - // TODO(bugs.webrtc.org/11553) Remove active counts by computing them on the - // fly. This require changes to MaybePerformQualityRampupExperiment. - std::unordered_map - active_counts_ RTC_GUARDED_BY(resource_adaptation_queue_); }; } // namespace webrtc