diff --git a/video/BUILD.gn b/video/BUILD.gn index 2f7ec1788b..a3e7d07654 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -504,6 +504,7 @@ if (rtc_include_tests) { "end_to_end_tests/stats_tests.cc", "end_to_end_tests/transport_feedback_tests.cc", "frame_encode_metadata_writer_unittest.cc", + "overuse_frame_detector_resource_adaptation_unittest.cc", "overuse_frame_detector_unittest.cc", "picture_id_tests.cc", "quality_limitation_reason_tracker_unittest.cc", diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index 1b88bb35c8..792abc52ef 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -67,8 +67,57 @@ VideoSourceRestrictions ApplyDegradationPreference( return source_restrictions; } +// Returns AdaptationCounters where constraints that don't apply to the +// degradation preference are cleared. This behaviour must reflect that of +// ApplyDegradationPreference for SourceRestrictions. Any changed to that +// method must also change this one. +AdaptationCounters ApplyDegradationPreference( + AdaptationCounters counters, + DegradationPreference degradation_preference) { + switch (degradation_preference) { + case DegradationPreference::BALANCED: + break; + case DegradationPreference::MAINTAIN_FRAMERATE: + counters.fps_adaptations = 0; + break; + case DegradationPreference::MAINTAIN_RESOLUTION: + counters.resolution_adaptations = 0; + break; + case DegradationPreference::DISABLED: + counters.resolution_adaptations = 0; + counters.fps_adaptations = 0; + break; + default: + RTC_NOTREACHED(); + } + return counters; +} + } // namespace +bool AdaptationCounters::operator==(const AdaptationCounters& rhs) const { + return fps_adaptations == rhs.fps_adaptations && + resolution_adaptations == rhs.resolution_adaptations; +} + +bool AdaptationCounters::operator!=(const AdaptationCounters& rhs) const { + return !(rhs == *this); +} + +AdaptationCounters AdaptationCounters::operator+( + const AdaptationCounters& other) const { + return AdaptationCounters( + resolution_adaptations + other.resolution_adaptations, + fps_adaptations + other.fps_adaptations); +} + +AdaptationCounters AdaptationCounters::operator-( + const AdaptationCounters& other) const { + return AdaptationCounters( + resolution_adaptations - other.resolution_adaptations, + fps_adaptations - other.fps_adaptations); +} + // VideoSourceRestrictor is responsible for keeping track of current // VideoSourceRestrictions and how to modify them in response to adapting up or // down. It is not reponsible for determining when we should adapt up or down - @@ -115,8 +164,10 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { VideoSourceRestrictions source_restrictions() { return source_restrictions_; } + const AdaptationCounters& adaptation_counters() const { return adaptations_; } void ClearRestrictions() { source_restrictions_ = VideoSourceRestrictions(); + adaptations_ = AdaptationCounters(); } bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) { @@ -135,6 +186,7 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { ? absl::optional(target_pixels) : absl::nullopt); source_restrictions_.set_target_pixels_per_frame(absl::nullopt); + ++adaptations_.resolution_adaptations; } bool CanIncreaseResolutionTo(int target_pixels) { @@ -157,6 +209,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { max_pixels_wanted != std::numeric_limits::max() ? absl::optional(target_pixels) : absl::nullopt); + --adaptations_.resolution_adaptations; + RTC_DCHECK_GE(adaptations_.resolution_adaptations, 0); } bool CanDecreaseFrameRateTo(int max_frame_rate) { @@ -173,6 +227,7 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { max_frame_rate != std::numeric_limits::max() ? absl::optional(max_frame_rate) : absl::nullopt); + ++adaptations_.fps_adaptations; } bool CanIncreaseFrameRateTo(int max_frame_rate) { @@ -187,6 +242,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { max_frame_rate != std::numeric_limits::max() ? absl::optional(max_frame_rate) : absl::nullopt); + --adaptations_.fps_adaptations; + RTC_DCHECK_GE(adaptations_.fps_adaptations, 0); } private: @@ -207,113 +264,11 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { } VideoSourceRestrictions source_restrictions_; + AdaptationCounters adaptations_; RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor); }; -class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { - public: - AdaptCounter() { - fps_counters_.resize(AdaptationObserverInterface::kScaleReasonSize); - resolution_counters_.resize(AdaptationObserverInterface::kScaleReasonSize); - static_assert(AdaptationObserverInterface::kScaleReasonSize == 2, - "Update MoveCount."); - } - ~AdaptCounter() = default; - - // Get number of adaptation downscales for |reason|. - VideoStreamEncoderObserver::AdaptationSteps Counts(int reason) const { - VideoStreamEncoderObserver::AdaptationSteps counts; - counts.num_framerate_reductions = fps_counters_[reason]; - counts.num_resolution_reductions = resolution_counters_[reason]; - return counts; - } - - std::string ToString() const { - rtc::StringBuilder ss; - ss << "Downgrade counts: fps: {" << ToString(fps_counters_); - ss << "}, resolution: {" << ToString(resolution_counters_) << "}"; - return ss.Release(); - } - - void IncrementFramerate(int reason) { ++(fps_counters_[reason]); } - void IncrementResolution(int reason) { ++(resolution_counters_[reason]); } - void DecrementFramerate(int reason) { - if (fps_counters_[reason] == 0) { - // Balanced mode: Adapt up is in a different order, switch reason. - // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3). - // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0} - // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0} - // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0} - // 4. Up resolution (quality):res={quality:0,cpu:0}, fps={quality:0,cpu:0} - RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; - RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; - MoveCount(&resolution_counters_, reason); - MoveCount(&fps_counters_, - (reason + 1) % AdaptationObserverInterface::kScaleReasonSize); - } - --(fps_counters_[reason]); - RTC_DCHECK_GE(fps_counters_[reason], 0); - } - - void DecrementResolution(int reason) { - if (resolution_counters_[reason] == 0) { - // Balanced mode: Adapt up is in a different order, switch reason. - RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; - RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; - MoveCount(&fps_counters_, reason); - MoveCount(&resolution_counters_, - (reason + 1) % AdaptationObserverInterface::kScaleReasonSize); - } - --(resolution_counters_[reason]); - RTC_DCHECK_GE(resolution_counters_[reason], 0); - } - - void DecrementFramerate(int reason, int cur_fps) { - DecrementFramerate(reason); - // Reset if at max fps (i.e. in case of fewer steps up than down). - if (cur_fps == std::numeric_limits::max()) - absl::c_fill(fps_counters_, 0); - } - - // Gets the total number of downgrades (for all adapt reasons). - int FramerateCount() const { return Count(fps_counters_); } - int ResolutionCount() const { return Count(resolution_counters_); } - - // Gets the total number of downgrades for |reason|. - int FramerateCount(int reason) const { return fps_counters_[reason]; } - int ResolutionCount(int reason) const { return resolution_counters_[reason]; } - int TotalCount(int reason) const { - return FramerateCount(reason) + ResolutionCount(reason); - } - - private: - std::string ToString(const std::vector& counters) const { - rtc::StringBuilder ss; - for (size_t reason = 0; - reason < AdaptationObserverInterface::kScaleReasonSize; ++reason) { - ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; - } - return ss.Release(); - } - - int Count(const std::vector& counters) const { - return absl::c_accumulate(counters, 0); - } - - void MoveCount(std::vector* counters, int from_reason) { - int to_reason = - (from_reason + 1) % AdaptationObserverInterface::kScaleReasonSize; - ++((*counters)[to_reason]); - --((*counters)[from_reason]); - } - - // Degradation counters holding number of framerate/resolution reductions - // per adapt reason. - std::vector fps_counters_; - std::vector resolution_counters_; -}; - class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper { public: explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource) @@ -402,7 +357,6 @@ OveruseFrameDetectorResourceAdaptationModule:: experiment_cpu_load_estimator_(experiment_cpu_load_estimator), has_input_video_(false), degradation_preference_(DegradationPreference::DISABLED), - adapt_counters_(), balanced_settings_(), last_adaptation_request_(absl::nullopt), source_restrictor_(std::make_unique()), @@ -418,10 +372,10 @@ OveruseFrameDetectorResourceAdaptationModule:: quality_rampup_done_(false), quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), encoder_settings_(absl::nullopt), - encoder_stats_observer_(encoder_stats_observer) { + encoder_stats_observer_(encoder_stats_observer), + active_counts_() { RTC_DCHECK(adaptation_listener_); RTC_DCHECK(encoder_stats_observer_); - ClearAdaptCounters(); AddResource(encode_usage_resource_.get(), AdaptationObserverInterface::AdaptReason::kCpu); AddResource(quality_scaler_resource_.get(), @@ -488,10 +442,8 @@ void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( last_adaptation_request_.reset(); if (degradation_preference == DegradationPreference::BALANCED || degradation_preference_ == DegradationPreference::BALANCED) { - // TODO(asapersson): Consider removing |adapt_counters_| map and use one - // AdaptCounter for all modes. source_restrictor_->ClearRestrictions(); - ClearAdaptCounters(); + active_counts_.fill(AdaptationCounters()); } } degradation_preference_ = degradation_preference; @@ -533,7 +485,7 @@ void OveruseFrameDetectorResourceAdaptationModule:: ResetVideoSourceRestrictions() { last_adaptation_request_.reset(); source_restrictor_->ClearRestrictions(); - ClearAdaptCounters(); + active_counts_.fill(AdaptationCounters()); MaybeUpdateVideoSourceRestrictions(); } @@ -543,19 +495,17 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrame( } void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { - int fps_count = GetConstAdaptCounter().FramerateCount( - AdaptationObserverInterface::AdaptReason::kQuality); - int res_count = GetConstAdaptCounter().ResolutionCount( - AdaptationObserverInterface::AdaptReason::kQuality); + AdaptationCounters counters_before = + source_restrictor_->adaptation_counters(); OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); if (degradation_preference() == DegradationPreference::BALANCED && - GetConstAdaptCounter().FramerateCount( - AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) { + source_restrictor_->adaptation_counters().fps_adaptations > + counters_before.fps_adaptations) { // Adapt framerate in same step as resolution. OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); } - if (GetConstAdaptCounter().ResolutionCount( - AdaptationObserverInterface::AdaptReason::kQuality) > res_count) { + if (source_restrictor_->adaptation_counters().resolution_adaptations > + counters_before.resolution_adaptations) { encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); } initial_frame_dropper_->OnFrameDroppedDueToSize(); @@ -690,7 +640,14 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget( if (!has_input_video_) return absl::nullopt; // 1. We can't adapt up if we're already at the highest setting. - int num_downgrades = GetConstAdaptCounter().TotalCount(reason); + // Note that this only includes counts relevant to the current degradation + // preference. e.g. we previously adapted resolution, now prefer adpating fps, + // only count the fps adaptations and not the previous resolution adaptations. + // TODO(https://crbug.com/webrtc/11394): Checking the counts for reason should + // be replaced with checking the overuse state of all resources. + int num_downgrades = ApplyDegradationPreference(active_counts_[reason], + degradation_preference_) + .Total(); RTC_DCHECK_GE(num_downgrades, 0); if (num_downgrades == 0) return absl::nullopt; @@ -745,7 +702,8 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget( } // Attempt to increase pixel count. int target_pixels = input_pixels; - if (GetConstAdaptCounter().ResolutionCount() == 1) { + if (source_restrictor_->adaptation_counters().resolution_adaptations == + 1) { RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; target_pixels = std::numeric_limits::max(); } @@ -759,7 +717,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget( case DegradationPreference::MAINTAIN_RESOLUTION: { // Scale up framerate. int target_fps = input_fps; - if (GetConstAdaptCounter().FramerateCount() == 1) { + if (source_restrictor_->adaptation_counters().fps_adaptations == 1) { RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; target_fps = std::numeric_limits::max(); } @@ -777,8 +735,7 @@ absl::optional OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget( int input_pixels, int input_fps, - int min_pixels_per_frame, - AdaptationObserverInterface::AdaptReason reason) const { + int min_pixels_per_frame) const { // Preconditions for being able to adapt down: if (!has_input_video_) return absl::nullopt; @@ -860,22 +817,19 @@ void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget( switch (target.action) { case AdaptationAction::kIncreaseResolution: source_restrictor_->IncreaseResolutionTo(target.value); - GetAdaptCounter().DecrementResolution(reason); return; case AdaptationAction::kDecreaseResolution: source_restrictor_->DecreaseResolutionTo(target.value, min_pixels_per_frame); - GetAdaptCounter().IncrementResolution(reason); return; case AdaptationAction::kIncreaseFrameRate: source_restrictor_->IncreaseFrameRateTo(target.value); - GetAdaptCounter().DecrementFramerate(reason, target.value); // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps. // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic // in DecrementFramerate() makes it hard to predict whether this will be // the last step. Remove the dependency on GetConstAdaptCounter(). if (EffectiveDegradationPreference() == DegradationPreference::BALANCED && - GetConstAdaptCounter().FramerateCount() == 0 && + source_restrictor_->adaptation_counters().fps_adaptations == 0 && target.value != std::numeric_limits::max()) { RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; source_restrictor_->IncreaseFrameRateTo( @@ -884,7 +838,6 @@ void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget( return; case AdaptationAction::kDecreaseFrameRate: source_restrictor_->DecreaseFrameRateTo(target.value); - GetAdaptCounter().IncrementFramerate(reason); return; } } @@ -908,7 +861,7 @@ void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( MaybeUpdateVideoSourceRestrictions(); // Stats and logging. UpdateAdaptationStats(reason); - RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString(); + RTC_LOG(LS_INFO) << ActiveCountsToString(); } ResourceListenerResponse @@ -921,7 +874,7 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( int min_pixels_per_frame = MinPixelsPerFrame(); // Should we adapt, if so to what target? absl::optional target = - GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame, reason); + GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame); if (!target.has_value()) return ResourceListenerResponse::kNothing; // Apply target. @@ -933,7 +886,7 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( MaybeUpdateVideoSourceRestrictions(); // Stats and logging. UpdateAdaptationStats(reason); - RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString(); + RTC_LOG(INFO) << ActiveCountsToString(); // In BALANCED, if requested FPS is higher or close to input FPS to the target // we tell the QualityScaler to increase its frequency. if (EffectiveDegradationPreference() == DegradationPreference::BALANCED && @@ -1035,9 +988,74 @@ void OveruseFrameDetectorResourceAdaptationModule:: encode_usage_resource_->SetTargetFrameRate(target_frame_rate); } +void OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( + const AdaptationCounters& adaptation_count, + AdaptationCounters* active_count, + AdaptationCounters* other_active) { + RTC_DCHECK(active_count); + RTC_DCHECK(other_active); + const int active_total = active_count->Total(); + const int other_total = other_active->Total(); + const AdaptationCounters prev_total = *active_count + *other_active; + const AdaptationCounters delta = adaptation_count - prev_total; + + RTC_DCHECK_EQ( + std::abs(delta.resolution_adaptations) + std::abs(delta.fps_adaptations), + 1) + << "Adaptation took more than one step!"; + + if (delta.resolution_adaptations > 0) { + ++active_count->resolution_adaptations; + } else if (delta.resolution_adaptations < 0) { + if (active_count->resolution_adaptations == 0) { + RTC_DCHECK_GT(active_count->fps_adaptations, 0) << "No downgrades left"; + RTC_DCHECK_GT(other_active->resolution_adaptations, 0) + << "No resolution adaptation to borrow from"; + // Lend an fps adaptation to other and take one resolution adaptation. + --active_count->fps_adaptations; + ++other_active->fps_adaptations; + --other_active->resolution_adaptations; + } else { + --active_count->resolution_adaptations; + } + } + if (delta.fps_adaptations > 0) { + ++active_count->fps_adaptations; + } else if (delta.fps_adaptations < 0) { + if (active_count->fps_adaptations == 0) { + RTC_DCHECK_GT(active_count->resolution_adaptations, 0) + << "No downgrades left"; + RTC_DCHECK_GT(other_active->fps_adaptations, 0) + << "No fps adaptation to borrow from"; + // Lend a resolution adaptation to other and take one fps adaptation. + --active_count->resolution_adaptations; + ++other_active->resolution_adaptations; + --other_active->fps_adaptations; + } else { + --active_count->fps_adaptations; + } + } + + RTC_DCHECK(*active_count + *other_active == adaptation_count); + RTC_DCHECK_EQ(other_active->Total(), other_total); + RTC_DCHECK_EQ(active_count->Total(), active_total + delta.Total()); + RTC_DCHECK_GE(active_count->resolution_adaptations, 0); + RTC_DCHECK_GE(active_count->fps_adaptations, 0); + RTC_DCHECK_GE(other_active->resolution_adaptations, 0); + RTC_DCHECK_GE(other_active->fps_adaptations, 0); +} + // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( AdaptationObserverInterface::AdaptReason reason) { + // Update active counts + AdaptationCounters& active_count = active_counts_[reason]; + AdaptationCounters& other_active = active_counts_[(reason + 1) % 2]; + const AdaptationCounters total_counts = + source_restrictor_->adaptation_counters(); + + OnAdaptationCountChanged(total_counts, &active_count, &other_active); + switch (reason) { case AdaptationObserverInterface::AdaptReason::kCpu: encoder_stats_observer_->OnAdaptationChanged( @@ -1057,8 +1075,14 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( VideoStreamEncoderObserver::AdaptationSteps OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( AdaptationObserverInterface::AdaptReason reason) { + // TODO(https://crbug.com/webrtc/11392) Ideally this shuold be moved out of + // this class and into the encoder_stats_observer_. + const AdaptationCounters counters = active_counts_[reason]; + VideoStreamEncoderObserver::AdaptationSteps counts = - GetConstAdaptCounter().Counts(reason); + VideoStreamEncoderObserver::AdaptationSteps(); + counts.num_resolution_reductions = counters.resolution_adaptations; + counts.num_framerate_reductions = counters.fps_adaptations; switch (reason) { case AdaptationObserverInterface::AdaptReason::kCpu: if (!IsFramerateScalingEnabled(degradation_preference_)) @@ -1095,30 +1119,6 @@ OveruseFrameDetectorResourceAdaptationModule::EffectiveDegradationPreference() : degradation_preference_; } -OveruseFrameDetectorResourceAdaptationModule::AdaptCounter& -OveruseFrameDetectorResourceAdaptationModule::GetAdaptCounter() { - return adapt_counters_[degradation_preference_]; -} - -void OveruseFrameDetectorResourceAdaptationModule::ClearAdaptCounters() { - adapt_counters_.clear(); - adapt_counters_.insert( - std::make_pair(DegradationPreference::DISABLED, AdaptCounter())); - adapt_counters_.insert(std::make_pair( - DegradationPreference::MAINTAIN_FRAMERATE, AdaptCounter())); - adapt_counters_.insert(std::make_pair( - DegradationPreference::MAINTAIN_RESOLUTION, AdaptCounter())); - adapt_counters_.insert( - std::make_pair(DegradationPreference::BALANCED, AdaptCounter())); -} - -const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter& -OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() const { - auto it = adapt_counters_.find(degradation_preference_); - RTC_DCHECK(it != adapt_counters_.cend()); - return it->second; -} - bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( int pixels, uint32_t bitrate_bps) const { @@ -1158,15 +1158,36 @@ void OveruseFrameDetectorResourceAdaptationModule:: try_quality_rampup = true; } } - if (try_quality_rampup && - GetConstAdaptCounter().ResolutionCount( - AdaptationObserverInterface::AdaptReason::kQuality) > 0 && - GetConstAdaptCounter().TotalCount( - AdaptationObserverInterface::AdaptReason::kCpu) == 0) { + // TODO(https://crbug.com/webrtc/11392): See if we can rely on the total + // counts or the stats, and not the active counts. + const AdaptationCounters& qp_counts = + std::get(active_counts_); + const AdaptationCounters& cpu_counts = + std::get(active_counts_); + if (try_quality_rampup && qp_counts.resolution_adaptations > 0 && + cpu_counts.Total() == 0) { RTC_LOG(LS_INFO) << "Reset quality limitations."; ResetVideoSourceRestrictions(); quality_rampup_done_ = true; } } +std::string OveruseFrameDetectorResourceAdaptationModule::ActiveCountsToString() + const { + rtc::StringBuilder ss; + + ss << "Downgrade counts: fps: {"; + for (size_t reason = 0; reason < active_counts_.size(); ++reason) { + ss << (reason ? " cpu" : "quality") << ":"; + ss << active_counts_[reason].fps_adaptations; + } + ss << "}, resolution {"; + for (size_t reason = 0; reason < active_counts_.size(); ++reason) { + ss << (reason ? " cpu" : "quality") << ":"; + ss << active_counts_[reason].resolution_adaptations; + } + ss << "}"; + + return ss.Release(); +} } // namespace webrtc diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 6e846d75f0..30f0aa3432 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -30,6 +30,7 @@ #include "rtc_base/experiments/balanced_degradation_settings.h" #include "rtc_base/experiments/quality_rampup_experiment.h" #include "rtc_base/experiments/quality_scaler_settings.h" +#include "rtc_base/strings/string_builder.h" #include "system_wrappers/include/clock.h" #include "video/encode_usage_resource.h" #include "video/overuse_frame_detector.h" @@ -37,6 +38,26 @@ namespace webrtc { +// Counts the number of adaptations have resulted due to resource overuse. +// Today we can adapt resolution and fps. +struct AdaptationCounters { + AdaptationCounters() : resolution_adaptations(0), fps_adaptations(0) {} + AdaptationCounters(int resolution_adaptations, int fps_adaptations) + : resolution_adaptations(resolution_adaptations), + fps_adaptations(fps_adaptations) {} + + int Total() const { return fps_adaptations + resolution_adaptations; } + + bool operator==(const AdaptationCounters& rhs) const; + bool operator!=(const AdaptationCounters& rhs) const; + + AdaptationCounters operator+(const AdaptationCounters& other) const; + AdaptationCounters operator-(const AdaptationCounters& other) const; + + int resolution_adaptations; + int fps_adaptations; +}; + class VideoStreamEncoder; // This class is used by the VideoStreamEncoder and is responsible for adapting @@ -114,9 +135,20 @@ class OveruseFrameDetectorResourceAdaptationModule ResourceListenerResponse OnResourceUsageStateMeasured( const Resource& resource) override; + // For reasons of adaptation and statistics, we not only count the total + // number of adaptations, but we also count the number of adaptations per + // reason. + // This method takes the new total number of adaptations and allocates that to + // the "active" count - number of adaptations for the current reason. + // The "other" count is the number of adaptations for the other reason. + // This must be called for each adaptation step made. + static void OnAdaptationCountChanged( + const AdaptationCounters& adaptation_count, + AdaptationCounters* active_count, + AdaptationCounters* other_active); + private: class VideoSourceRestrictor; - class AdaptCounter; class InitialFrameDropper; enum class State { kStopped, kStarted }; @@ -160,8 +192,7 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional GetAdaptDownTarget( int input_pixels, int input_fps, - int min_pixels_per_frame, - AdaptationObserverInterface::AdaptReason reason) const; + int min_pixels_per_frame) const; // Applies the |target| to |source_restrictor_|. void ApplyAdaptationTarget(const AdaptationTarget& target, int min_pixels_per_frame, @@ -179,8 +210,6 @@ class OveruseFrameDetectorResourceAdaptationModule int MinPixelsPerFrame() const; VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( AdaptationObserverInterface::AdaptReason reason); - void ClearAdaptCounters(); - const AdaptCounter& GetConstAdaptCounter() const; // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener @@ -196,7 +225,6 @@ class OveruseFrameDetectorResourceAdaptationModule void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason); DegradationPreference EffectiveDegradationPreference() const; - AdaptCounter& GetAdaptCounter(); bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const; // Checks to see if we should execute the quality rampup experiment. The @@ -207,6 +235,8 @@ class OveruseFrameDetectorResourceAdaptationModule void MaybePerformQualityRampupExperiment(); void ResetVideoSourceRestrictions(); + std::string ActiveCountsToString() const; + ResourceAdaptationModuleListener* const adaptation_listener_; Clock* clock_; State state_; @@ -215,12 +245,6 @@ class OveruseFrameDetectorResourceAdaptationModule VideoSourceRestrictions video_source_restrictions_; bool has_input_video_; DegradationPreference degradation_preference_; - // Counters used for deciding if the video resolution or framerate is - // currently restricted, and if so, why, on a per degradation preference - // basis. - // TODO(sprang): Replace this with a state holding a relative overuse measure - // instead, that can be translated into suitable down-scale or fps limit. - std::map adapt_counters_; const BalancedDegradationSettings balanced_settings_; // Stores a snapshot of the last adaptation request triggered by an AdaptUp // or AdaptDown signal. @@ -253,6 +277,15 @@ class OveruseFrameDetectorResourceAdaptationModule const AdaptationObserverInterface::AdaptReason reason; }; std::vector resources_; + // One AdaptationCounter for each reason, tracking the number of times we have + // adapted for each reason. The sum of active_counts_ MUST always equal the + // total adaptation provided by the VideoSourceRestrictions. + // TODO(https://crbug.com/webrtc/11392): Move all active count logic to + // encoder_stats_observer_; Counters used for deciding if the video resolution + // or framerate is currently restricted, and if so, why, on a per degradation + // preference basis. + std::array + active_counts_; }; } // namespace webrtc diff --git a/video/overuse_frame_detector_resource_adaptation_unittest.cc b/video/overuse_frame_detector_resource_adaptation_unittest.cc new file mode 100644 index 0000000000..428618bc40 --- /dev/null +++ b/video/overuse_frame_detector_resource_adaptation_unittest.cc @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/overuse_frame_detector_resource_adaptation_module.h" + +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(AdaptationCountersTest, Addition) { + AdaptationCounters a; + AdaptationCounters b(1, 2); + AdaptationCounters total = a + b; + EXPECT_EQ(1, total.resolution_adaptations); + EXPECT_EQ(2, total.fps_adaptations); +} + +TEST(AdaptationCountersTest, Subtraction) { + AdaptationCounters a(0, 1); + AdaptationCounters b(2, 1); + AdaptationCounters diff = a - b; + EXPECT_EQ(-2, diff.resolution_adaptations); + EXPECT_EQ(0, diff.fps_adaptations); +} + +TEST(AdaptationCountersTest, Equality) { + AdaptationCounters a(1, 2); + AdaptationCounters b(2, 1); + EXPECT_EQ(a, a); + EXPECT_NE(a, b); +} + +TEST(AdaptationCountersTest, SelfAdditionSubtraction) { + AdaptationCounters a(1, 0); + AdaptationCounters b(0, 1); + + EXPECT_EQ(a, a + b - b); + EXPECT_EQ(a, b + a - b); + EXPECT_EQ(a, a - b + b); + EXPECT_EQ(a, b - b + a); +} + +TEST(OveruseFrameDetectorResourceAdaptationModuleTest, + FirstAdaptationDown_Fps) { + AdaptationCounters cpu; + AdaptationCounters qp; + AdaptationCounters total(0, 1); + + OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( + total, &cpu, &qp); + AdaptationCounters expected_cpu(0, 1); + AdaptationCounters expected_qp; + EXPECT_EQ(expected_cpu, cpu); + EXPECT_EQ(expected_qp, qp); +} + +TEST(OveruseFrameDetectorResourceAdaptationModuleTest, + FirstAdaptationDown_Resolution) { + AdaptationCounters cpu; + AdaptationCounters qp; + AdaptationCounters total(1, 0); + + OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( + total, &cpu, &qp); + AdaptationCounters expected_cpu(1, 0); + AdaptationCounters expected_qp; + EXPECT_EQ(expected_cpu, cpu); + EXPECT_EQ(expected_qp, qp); +} + +TEST(OveruseFrameDetectorResourceAdaptationModuleTest, LastAdaptUp_Fps) { + AdaptationCounters cpu(0, 1); + AdaptationCounters qp; + AdaptationCounters total; + + OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( + total, &cpu, &qp); + AdaptationCounters expected_cpu; + AdaptationCounters expected_qp; + EXPECT_EQ(expected_cpu, cpu); + EXPECT_EQ(expected_qp, qp); +} + +TEST(OveruseFrameDetectorResourceAdaptationModuleTest, LastAdaptUp_Resolution) { + AdaptationCounters cpu(1, 0); + AdaptationCounters qp; + AdaptationCounters total; + + OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( + total, &cpu, &qp); + AdaptationCounters expected_cpu; + AdaptationCounters expected_qp; + EXPECT_EQ(expected_cpu, cpu); + EXPECT_EQ(expected_qp, qp); +} + +TEST(OveruseFrameDetectorResourceAdaptationModuleTest, + AdaptUpWithBorrow_Resolution) { + AdaptationCounters cpu(0, 1); + AdaptationCounters qp(1, 0); + AdaptationCounters total(0, 1); + + // CPU adaptation for resolution, but no + // resolution adaptation left from CPU. + // We then borrow the resolution + // adaptation from qp, and give qp the + // fps adaptation from CPU. + OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( + total, &cpu, &qp); + + AdaptationCounters expected_cpu(0, 0); + AdaptationCounters expected_qp(0, 1); + EXPECT_EQ(expected_cpu, cpu); + EXPECT_EQ(expected_qp, qp); +} + +TEST(OveruseFrameDetectorResourceAdaptationModuleTest, AdaptUpWithBorrow_Fps) { + AdaptationCounters cpu(1, 0); + AdaptationCounters qp(0, 1); + AdaptationCounters total(1, 0); + + // CPU adaptation for fps, but no + // fps adaptation left from CPU. We + // then borrow the fps adaptation + // from qp, and give qp the + // resolution adaptation from CPU. + OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( + total, &cpu, &qp); + + AdaptationCounters expected_cpu(0, 0); + AdaptationCounters expected_qp(1, 0); + EXPECT_EQ(expected_cpu, cpu); + EXPECT_EQ(expected_qp, qp); +} + +} // namespace webrtc diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index e48ccd806a..9d6925e9c7 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -2235,6 +2235,73 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) { + video_stream_encoder_->OnBitrateUpdated( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + const int kWidth = 1280; + const int kHeight = 720; + int sequence = 1; + + // Enable BALANCED preference, no initial limitation. + test::FrameForwarder source; + video_stream_encoder_->SetSource(&source, + webrtc::DegradationPreference::BALANCED); + source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight)); + WaitForEncodedFrame(sequence++); + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); + EXPECT_EQ(0, stats.number_of_cpu_adapt_changes); + + // Trigger CPU overuse, should now adapt down. + video_stream_encoder_->TriggerCpuOveruse(); + source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight)); + WaitForEncodedFrame(sequence++); + stats = stats_proxy_->GetStats(); + EXPECT_EQ(1, stats.number_of_cpu_adapt_changes); + + // Set new degradation preference should clear restrictions since we changed + // from BALANCED. + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight)); + WaitForEncodedFrame(sequence++); + stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); + EXPECT_EQ(1, stats.number_of_cpu_adapt_changes); + + // Force an input frame rate to be available, or the adaptation call won't + // know what framerate to adapt from. + VideoSendStream::Stats mock_stats = stats_proxy_->GetStats(); + mock_stats.input_frame_rate = 30; + stats_proxy_->SetMockStats(mock_stats); + video_stream_encoder_->TriggerCpuOveruse(); + stats_proxy_->ResetMockStats(); + source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight)); + WaitForEncodedFrame(sequence++); + + // We have now adapted once. + stats = stats_proxy_->GetStats(); + EXPECT_EQ(2, stats.number_of_cpu_adapt_changes); + + // Back to BALANCED, should clear the restrictions again. + video_stream_encoder_->SetSource(&source, + webrtc::DegradationPreference::BALANCED); + source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight)); + WaitForEncodedFrame(sequence++); + stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); + EXPECT_EQ(2, stats.number_of_cpu_adapt_changes); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { video_stream_encoder_->OnBitrateUpdated(