From 7875c99e82201258dbfb4f38f9f5d865a78dfd8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Thu, 6 Feb 2020 10:35:00 +0100 Subject: [PATCH] [Overuse] Add EncodeUsageResource and QualityScalerResource. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This refactors the usage of OveruseFrameDetector in OveruseFrameDetectorResourceAdaptationModule into an inner class of the module, making the interaction between the detector and the module the responsibility of this helper class instead. Similarly, QualityScaler usage is moved into QualityScalerResource. This takes us one step closer to separate the act of detecting overuse/underuse of a resource and the logic of what to do when overuse/underuse happens. Follow-up CLs should build on this in order to materialize the concept of having resources, streams and a central decision-maker deciding how to reconfigure the streams based on resource usage state. Bug: webrtc:11222 Change-Id: I99a08a42218a871db8f477f31447a6379433ad05 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168057 Commit-Queue: Henrik Boström Reviewed-by: Evan Shrubsole Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#30468} --- ...ame_detector_resource_adaptation_module.cc | 306 ++++++++++++------ ...rame_detector_resource_adaptation_module.h | 46 +-- video/video_stream_encoder.cc | 8 +- video/video_stream_encoder.h | 13 +- video/video_stream_encoder_unittest.cc | 4 +- 5 files changed, 252 insertions(+), 125 deletions(-) diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index edced6dd9a..5671607739 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -72,12 +72,154 @@ const int kMaxInitialFramedrop = 4; } // namespace +// Handles interaction with the OveruseDetector. +class OveruseFrameDetectorResourceAdaptationModule::EncodeUsageResource + : public AdaptationObserverInterface { + public: + EncodeUsageResource(OveruseFrameDetectorResourceAdaptationModule* module, + std::unique_ptr overuse_detector) + : module_(module), + overuse_detector_(std::move(overuse_detector)), + is_started_(false), + target_frame_rate_(absl::nullopt) { + RTC_DCHECK(module_); + RTC_DCHECK(overuse_detector_); + } + + void StartCheckForOveruse(CpuOveruseOptions options) { + RTC_DCHECK(!is_started_); + overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(), + std::move(options), this); + is_started_ = true; + overuse_detector_->OnTargetFramerateUpdated(TargetFrameRateAsInt()); + } + + void StopCheckForOveruse() { + overuse_detector_->StopCheckForOveruse(); + is_started_ = false; + } + + void SetTargetFrameRate(absl::optional target_frame_rate) { + if (target_frame_rate == target_frame_rate_) + return; + target_frame_rate_ = target_frame_rate; + if (is_started_) + overuse_detector_->OnTargetFramerateUpdated(TargetFrameRateAsInt()); + } + + void OnEncodeStarted(const VideoFrame& cropped_frame, + int64_t time_when_first_seen_us) { + // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g. + // "OnEncodeStarted"?) or revise usage. + overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us); + } + + void OnEncodeCompleted(uint32_t timestamp, + int64_t time_sent_in_us, + int64_t capture_time_us, + absl::optional encode_duration_us) { + // TODO(hbos): Rename FrameSent() to something more appropriate (e.g. + // "OnEncodeCompleted"?). + overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, + encode_duration_us); + } + + // AdaptationObserverInterface implementation. + void AdaptUp(AdaptReason reason) override { + RTC_DCHECK_EQ(reason, AdaptReason::kCpu); + module_->OnResourceUnderuse(reason); + } + bool AdaptDown(AdaptReason reason) override { + RTC_DCHECK_EQ(reason, AdaptReason::kCpu); + return module_->OnResourceOveruse(reason); + } + + private: + int TargetFrameRateAsInt() { + return target_frame_rate_.has_value() + ? static_cast(target_frame_rate_.value()) + : std::numeric_limits::max(); + } + + OveruseFrameDetectorResourceAdaptationModule* const module_; + const std::unique_ptr overuse_detector_; + bool is_started_; + absl::optional target_frame_rate_; +}; + +// Handles interaction with the QualityScaler. +class OveruseFrameDetectorResourceAdaptationModule::QualityScalerResource + : public AdaptationObserverInterface { + public: + explicit QualityScalerResource( + OveruseFrameDetectorResourceAdaptationModule* module) + : module_(module), quality_scaler_(nullptr) { + RTC_DCHECK(module_); + } + + bool is_started() const { return quality_scaler_.get(); } + // TODO(https://crbug.com/webrtc/11222): Don't expose the quality scaler. + QualityScaler* quality_scaler() const { return quality_scaler_.get(); } + + void StartCheckForOveruse(VideoEncoder::QpThresholds qp_thresholds) { + RTC_DCHECK(!is_started()); + quality_scaler_ = + std::make_unique(this, std::move(qp_thresholds)); + } + + void StopCheckForOveruse() { quality_scaler_.reset(); } + + void SetQpThresholds(VideoEncoder::QpThresholds qp_thresholds) { + RTC_DCHECK(is_started()); + quality_scaler_->SetQpThresholds(std::move(qp_thresholds)); + } + + bool QpFastFilterLow() { + RTC_DCHECK(is_started()); + return quality_scaler_->QpFastFilterLow(); + } + + void OnEncodeCompleted(const EncodedImage& encoded_image, + int64_t time_sent_in_us) { + if (quality_scaler_ && encoded_image.qp_ >= 0) + quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us); + } + + void OnFrameDropped(EncodedImageCallback::DropReason reason) { + if (!quality_scaler_) + return; + switch (reason) { + case EncodedImageCallback::DropReason::kDroppedByMediaOptimizations: + quality_scaler_->ReportDroppedFrameByMediaOpt(); + break; + case EncodedImageCallback::DropReason::kDroppedByEncoder: + quality_scaler_->ReportDroppedFrameByEncoder(); + break; + } + } + + // AdaptationObserverInterface implementation. + void AdaptUp(AdaptReason reason) override { + RTC_DCHECK_EQ(reason, AdaptReason::kQuality); + module_->OnResourceUnderuse(reason); + } + bool AdaptDown(AdaptReason reason) override { + RTC_DCHECK_EQ(reason, AdaptReason::kQuality); + return module_->OnResourceOveruse(reason); + } + + private: + OveruseFrameDetectorResourceAdaptationModule* const module_; + std::unique_ptr quality_scaler_; +}; + // VideoSourceRestrictor is responsible for keeping track of current // VideoSourceRestrictions and how to modify them in response to adapting up or // down. It is not reponsible for determining when we should adapt up or down - -// for that, see OveruseFrameDetectorResourceAdaptationModule::AdaptUp() and -// AdaptDown() - only how to modify the source/sink restrictions when this -// happens. Note that it is also not responsible for reconfigruring the +// for that, see +// OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse() and +// OnResourceOveruse() - only how to modify the source/sink restrictions when +// this happens. Note that it is also not responsible for reconfigruring the // source/sink, it is only a keeper of desired restrictions. class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { public: @@ -224,9 +366,10 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { public: AdaptCounter() { - fps_counters_.resize(kScaleReasonSize); - resolution_counters_.resize(kScaleReasonSize); - static_assert(kScaleReasonSize == 2, "Update MoveCount."); + fps_counters_.resize(AdaptationObserverInterface::kScaleReasonSize); + resolution_counters_.resize(AdaptationObserverInterface::kScaleReasonSize); + static_assert(AdaptationObserverInterface::kScaleReasonSize == 2, + "Update MoveCount."); } ~AdaptCounter() = default; @@ -258,7 +401,8 @@ class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; MoveCount(&resolution_counters_, reason); - MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize); + MoveCount(&fps_counters_, + (reason + 1) % AdaptationObserverInterface::kScaleReasonSize); } --(fps_counters_[reason]); RTC_DCHECK_GE(fps_counters_[reason], 0); @@ -270,7 +414,8 @@ class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; MoveCount(&fps_counters_, reason); - MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize); + MoveCount(&resolution_counters_, + (reason + 1) % AdaptationObserverInterface::kScaleReasonSize); } --(resolution_counters_[reason]); RTC_DCHECK_GE(resolution_counters_[reason], 0); @@ -297,7 +442,8 @@ class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { private: std::string ToString(const std::vector& counters) const { rtc::StringBuilder ss; - for (size_t reason = 0; reason < kScaleReasonSize; ++reason) { + for (size_t reason = 0; + reason < AdaptationObserverInterface::kScaleReasonSize; ++reason) { ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; } return ss.Release(); @@ -308,7 +454,8 @@ class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { } void MoveCount(std::vector* counters, int from_reason) { - int to_reason = (from_reason + 1) % kScaleReasonSize; + int to_reason = + (from_reason + 1) % AdaptationObserverInterface::kScaleReasonSize; ++((*counters)[to_reason]); --((*counters)[from_reason]); } @@ -335,13 +482,14 @@ OveruseFrameDetectorResourceAdaptationModule:: balanced_settings_(), last_adaptation_request_(absl::nullopt), source_restrictor_(std::make_unique()), - overuse_detector_(std::move(overuse_detector)), - overuse_detector_is_started_(false), + encode_usage_resource_( + std::make_unique(this, + std::move(overuse_detector))), + quality_scaler_resource_(std::make_unique(this)), + quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), last_input_frame_size_(absl::nullopt), target_frame_rate_(absl::nullopt), encoder_target_bitrate_bps_(absl::nullopt), - quality_scaler_(nullptr), - quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), quality_rampup_done_(false), quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), @@ -349,7 +497,6 @@ OveruseFrameDetectorResourceAdaptationModule:: encoder_stats_observer_(encoder_stats_observer), initial_framedrop_(0) { RTC_DCHECK(adaptation_listener_); - RTC_DCHECK(overuse_detector_); RTC_DCHECK(encoder_stats_observer_); } @@ -359,32 +506,21 @@ OveruseFrameDetectorResourceAdaptationModule:: void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( ResourceAdaptationModuleListener* adaptation_listener) { RTC_DCHECK(encoder_settings_.has_value()); - RTC_DCHECK(!overuse_detector_is_started_); - // TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside - // the interval between StartCheckForOveruse() and StopCheckForOveruse(), - // support configuring which |adaptation_listener_| to use on the fly. It is - // currently hardcoded for the entire lifetime of the module in order to - // support adaptation caused by VideoStreamEncoder or QualityScaler invoking - // AdaptUp() and AdaptDown() even when the OveruseDetector is inactive. + // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener + // should be passed in and why. If resources are separated from modules then + // those resources may be started or stopped separately from the module. RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); - overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(), - GetCpuOveruseOptions(), this); - overuse_detector_is_started_ = true; - overuse_detector_->OnTargetFramerateUpdated( - target_frame_rate_.has_value() - ? static_cast(target_frame_rate_.value()) - : std::numeric_limits::max()); + encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions()); } void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { - overuse_detector_->StopCheckForOveruse(); - overuse_detector_is_started_ = false; - quality_scaler_.reset(); + encode_usage_resource_->StopCheckForOveruse(); + quality_scaler_resource_->StopCheckForOveruse(); } void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( bool has_input_video) { - // While false, AdaptUp() and AdaptDown() are NO-OPS. + // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS. has_input_video_ = has_input_video; } @@ -430,8 +566,11 @@ void OveruseFrameDetectorResourceAdaptationModule::SetTargetBitrate( encoder_target_bitrate_bps_ = target_bitrate.bps(); // Check for bwe drop experiment + // TODO(https://crbug.com/webrtc/11222): Should this move to + // QualityScalerResource? if (start_bitrate_.set_start_bitrate_ > DataRate::Zero() && - !start_bitrate_.has_seen_first_bwe_drop_ && quality_scaler_ && + !start_bitrate_.has_seen_first_bwe_drop_ && + quality_scaler_resource_->is_started() && quality_scaler_settings_.InitialBitrateIntervalMs() && quality_scaler_settings_.InitialBitrateFactor()) { int64_t diff_ms = clock_->TimeInMilliseconds() - @@ -472,12 +611,12 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { AdaptationObserverInterface::AdaptReason::kQuality); int res_count = GetConstAdaptCounter().ResolutionCount( AdaptationObserverInterface::AdaptReason::kQuality); - AdaptDown(AdaptationObserverInterface::AdaptReason::kQuality); + OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); if (degradation_preference() == DegradationPreference::BALANCED && GetConstAdaptCounter().FramerateCount( AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) { // Adapt framerate in same step as resolution. - AdaptDown(AdaptationObserverInterface::AdaptReason::kQuality); + OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); } if (GetConstAdaptCounter().ResolutionCount( AdaptationObserverInterface::AdaptReason::kQuality) > res_count) { @@ -489,39 +628,27 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted( const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) { - // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g. - // "OnEncodeStarted"?) or revise usage. - overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us); + encode_usage_resource_->OnEncodeStarted(cropped_frame, + time_when_first_seen_us); } void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted( const EncodedImage& encoded_image, int64_t time_sent_in_us, absl::optional encode_duration_us) { - // TODO(hbos): Rename FrameSent() to something more appropriate (e.g. - // "OnEncodeCompleted"?). + // Inform |encode_usage_resource_| of the encode completed event. uint32_t timestamp = encoded_image.Timestamp(); int64_t capture_time_us = encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; - overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, - encode_duration_us); - if (quality_scaler_ && encoded_image.qp_ >= 0) - quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us); + encode_usage_resource_->OnEncodeCompleted( + timestamp, time_sent_in_us, capture_time_us, encode_duration_us); + // Inform |quality_scaler_resource_| of the encode completed event. + quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); } void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped( EncodedImageCallback::DropReason reason) { - if (!quality_scaler_) { - return; - } - switch (reason) { - case EncodedImageCallback::DropReason::kDroppedByMediaOptimizations: - quality_scaler_->ReportDroppedFrameByMediaOpt(); - break; - case EncodedImageCallback::DropReason::kDroppedByEncoder: - quality_scaler_->ReportDroppedFrameByEncoder(); - break; - } + quality_scaler_resource_->OnFrameDropped(reason); } void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() { @@ -536,12 +663,12 @@ bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const { void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( absl::optional qp_thresholds) { if (qp_thresholds.has_value()) { - quality_scaler_ = - std::make_unique(this, qp_thresholds.value()); + quality_scaler_resource_->StopCheckForOveruse(); + quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value()); // Restart frame drops due to size. initial_framedrop_ = 0; } else { - quality_scaler_ = nullptr; + quality_scaler_resource_->StopCheckForOveruse(); // Quality scaling disabled so we shouldn't drop initial frames. initial_framedrop_ = kMaxInitialFramedrop; } @@ -554,8 +681,10 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( IsResolutionScalingEnabled(degradation_preference_) && scaling_settings.thresholds; + // TODO(https://crbug.com/webrtc/11222): Should this move to + // QualityScalerResource? if (quality_scaling_allowed) { - if (quality_scaler_ == nullptr) { + if (!quality_scaler_resource_->is_started()) { // Quality scaler has not already been configured. // Use experimental thresholds if available. @@ -574,12 +703,12 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( // Set the qp-thresholds to the balanced settings if balanced mode. if (degradation_preference_ == DegradationPreference::BALANCED && - quality_scaler_) { + quality_scaler_resource_->is_started()) { absl::optional thresholds = balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault()); if (thresholds) { - quality_scaler_->SetQpThresholds(*thresholds); + quality_scaler_resource_->SetQpThresholds(*thresholds); } } @@ -589,7 +718,8 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); } -void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { +void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( + AdaptationObserverInterface::AdaptReason reason) { if (!has_input_video_) return; const AdaptCounter& adapt_counter = GetConstAdaptCounter(); @@ -621,7 +751,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { switch (EffectiveDegradataionPreference()) { case DegradationPreference::BALANCED: { // Check if quality should be increased based on bitrate. - if (reason == kQuality && + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && !balanced_settings_.CanAdaptUp( GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault(), encoder_target_bitrate_bps_.value_or(0))) { @@ -642,7 +772,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { break; } // Check if resolution should be increased based on bitrate. - if (reason == kQuality && + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && !balanced_settings_.CanAdaptUpResolution( GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault(), encoder_target_bitrate_bps_.value_or(0))) { @@ -654,7 +784,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { case DegradationPreference::MAINTAIN_FRAMERATE: { // Check if resolution should be increased based on bitrate and // limits specified by encoder capabilities. - if (reason == kQuality && + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && !CanAdaptUpResolution(LastInputFrameSizeOrDefault(), encoder_target_bitrate_bps_.value_or(0))) { return; @@ -702,8 +832,8 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { RTC_LOG(LS_INFO) << adapt_counter.ToString(); } -bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( - AdaptReason reason) { +bool OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( + AdaptationObserverInterface::AdaptReason reason) { if (!has_input_video_) return false; AdaptationRequest adaptation_request = { @@ -847,8 +977,8 @@ int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault() // yet. // TODO(hbos): Can we simply DCHECK has_value() before usage instead? Having a // DCHECK passed all the tests but adding it does change the requirements of - // this class (= not being allowed to call AdaptUp() or AdaptDown() before - // OnFrame()) and deserves a standalone CL. + // this class (= not being allowed to call OnResourceUnderuse() or + // OnResourceOveruse() before OnFrame()) and deserves a standalone CL. return last_input_frame_size_.value_or( VideoStreamEncoder::kDefaultLastFrameInfoWidth * VideoStreamEncoder::kDefaultLastFrameInfoHeight); @@ -886,53 +1016,47 @@ void OveruseFrameDetectorResourceAdaptationModule:: codec_max_frame_rate.value() < target_frame_rate.value())) { target_frame_rate = codec_max_frame_rate; } - if (target_frame_rate != target_frame_rate_) { - target_frame_rate_ = target_frame_rate; - if (overuse_detector_is_started_) { - overuse_detector_->OnTargetFramerateUpdated( - target_frame_rate_.has_value() - ? static_cast(target_frame_rate_.value()) - : std::numeric_limits::max()); - } - } + encode_usage_resource_->SetTargetFrameRate(target_frame_rate); } // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( - AdaptReason reason) { + AdaptationObserverInterface::AdaptReason reason) { switch (reason) { - case kCpu: + case AdaptationObserverInterface::AdaptReason::kCpu: encoder_stats_observer_->OnAdaptationChanged( VideoStreamEncoderObserver::AdaptationReason::kCpu, - GetActiveCounts(kCpu), GetActiveCounts(kQuality)); + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); break; - case kQuality: + case AdaptationObserverInterface::AdaptReason::kQuality: encoder_stats_observer_->OnAdaptationChanged( VideoStreamEncoderObserver::AdaptationReason::kQuality, - GetActiveCounts(kCpu), GetActiveCounts(kQuality)); + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); break; } } VideoStreamEncoderObserver::AdaptationSteps OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( - AdaptReason reason) { + AdaptationObserverInterface::AdaptReason reason) { VideoStreamEncoderObserver::AdaptationSteps counts = GetConstAdaptCounter().Counts(reason); switch (reason) { - case kCpu: + case AdaptationObserverInterface::AdaptReason::kCpu: if (!IsFramerateScalingEnabled(degradation_preference_)) counts.num_framerate_reductions = absl::nullopt; if (!IsResolutionScalingEnabled(degradation_preference_)) counts.num_resolution_reductions = absl::nullopt; break; - case kQuality: + case AdaptationObserverInterface::AdaptReason::kQuality: if (!IsFramerateScalingEnabled(degradation_preference_) || - !quality_scaler_) { + !quality_scaler_resource_->is_started()) { counts.num_framerate_reductions = absl::nullopt; } if (!IsResolutionScalingEnabled(degradation_preference_) || - !quality_scaler_) { + !quality_scaler_resource_->is_started()) { counts.num_resolution_reductions = absl::nullopt; } break; @@ -983,7 +1107,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( void OveruseFrameDetectorResourceAdaptationModule:: MaybePerformQualityRampupExperiment() { - if (!quality_scaler_) + if (!quality_scaler_resource_->is_started()) return; if (quality_rampup_done_) @@ -1000,7 +1124,7 @@ void OveruseFrameDetectorResourceAdaptationModule:: if (encoder_settings_ && encoder_target_bitrate_bps_.value_or(0) == encoder_settings_->video_codec().maxBitrate * 1000 && - quality_scaler_->QpFastFilterLow()) { + quality_scaler_resource_->QpFastFilterLow()) { try_quality_rampup = true; } } diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 6c841dc7a7..437510be2c 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -51,8 +51,7 @@ class VideoStreamEncoder; // generic interface in VideoStreamEncoder, unblocking other modules from being // implemented and used. class OveruseFrameDetectorResourceAdaptationModule - : public ResourceAdaptationModuleInterface, - public AdaptationObserverInterface { + : public ResourceAdaptationModuleInterface { public: // The module can be constructed on any sequence, but must be initialized and // used on a single sequence, e.g. the encoder queue. @@ -91,6 +90,11 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional encode_duration_us) override; void OnFrameDropped(EncodedImageCallback::DropReason reason) override; + // TODO(hbos): Is dropping initial frames really just a special case of "don't + // encode frames right now"? Can this be part of VideoSourceRestrictions, + // which handles the output of the rest of the encoder settings? This is + // something we'll need to support for "disable video due to overuse", not + // initial frames. bool DropInitialFrames() const; // TODO(eshr): This can be made private if we configure on @@ -98,20 +102,20 @@ class OveruseFrameDetectorResourceAdaptationModule // (https://crbug.com/webrtc/11338) void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); - // AdaptationObserverInterface implementation. Used both "internally" as - // feedback from |overuse_detector_|, and externally from VideoStreamEncoder: - // - It is wired to the VideoStreamEncoder::quality_scaler_. - // - It is invoked by VideoStreamEncoder::MaybeEncodeVideoFrame(). - // TODO(hbos): Decouple quality scaling and resource adaptation, or find an - // interface for reconfiguring externally. - // TODO(hbos): VideoStreamEncoder should not be responsible for any part of - // the adaptation. - void AdaptUp(AdaptReason reason) override; - bool AdaptDown(AdaptReason reason) override; + // Signal that a resource (kCpu or kQuality) is overused or underused. This is + // currently used by EncodeUsageResource, QualityScalerResource and testing. + // TODO(https://crbug.com/webrtc/11222): Make use of ResourceUsageState and + // implement resources per call/adaptation/resource.h. When adaptation happens + // because a resource is in specific usage state, get rid of these explicit + // triggers. + void OnResourceUnderuse(AdaptationObserverInterface::AdaptReason reason); + bool OnResourceOveruse(AdaptationObserverInterface::AdaptReason reason); private: - class AdaptCounter; + class EncodeUsageResource; + class QualityScalerResource; class VideoSourceRestrictor; + class AdaptCounter; struct AdaptationRequest { // The pixel count produced by the source at the time of the adaptation. @@ -132,23 +136,22 @@ class OveruseFrameDetectorResourceAdaptationModule VideoCodecType GetVideoCodecTypeOrGeneric() const; int LastInputFrameSizeOrDefault() const; VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( - AdaptReason reason); + AdaptationObserverInterface::AdaptReason reason); const AdaptCounter& GetConstAdaptCounter(); // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener // to reconfigure the source accordingly. void MaybeUpdateVideoSourceRestrictions(); - // Calculates an up-to-date value of |target_frame_rate_| and informs the - // |overuse_detector_| of the new value if it changed and the detector is - // started. + // Calculates an up-to-date value of the target frame rate and informs the + // |encode_usage_resource_| of the new value. void MaybeUpdateTargetFrameRate(); // Use nullopt to disable quality scaling. void UpdateQualityScalerSettings( absl::optional qp_thresholds); - void UpdateAdaptationStats(AdaptReason reason); + void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason); DegradationPreference EffectiveDegradataionPreference(); AdaptCounter& GetAdaptCounter(); bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const; @@ -180,15 +183,14 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional last_adaptation_request_; // Keeps track of source restrictions that this adaptation module outputs. const std::unique_ptr source_restrictor_; - const std::unique_ptr overuse_detector_; - bool overuse_detector_is_started_; + const std::unique_ptr encode_usage_resource_; + const std::unique_ptr quality_scaler_resource_; + const bool quality_scaling_experiment_enabled_; absl::optional last_input_frame_size_; absl::optional target_frame_rate_; // This is the last non-zero target bitrate for the encoder. absl::optional encoder_target_bitrate_bps_; absl::optional encoder_rates_; - std::unique_ptr quality_scaler_; - const bool quality_scaling_experiment_enabled_; const QualityScalerSettings quality_scaler_settings_; bool quality_rampup_done_; QualityRampupExperiment quality_rampup_experiment_; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index e90dd0cc0d..a8fbac9de9 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1577,16 +1577,16 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { return false; } -bool VideoStreamEncoder::TriggerAdaptDown( +void VideoStreamEncoder::OnResourceUnderuseForTesting( AdaptationObserverInterface::AdaptReason reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); - return resource_adaptation_module_->AdaptDown(reason); + resource_adaptation_module_->OnResourceUnderuse(reason); } -void VideoStreamEncoder::TriggerAdaptUp( +bool VideoStreamEncoder::OnResourceOveruseForTesting( AdaptationObserverInterface::AdaptReason reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); - resource_adaptation_module_->AdaptUp(reason); + return resource_adaptation_module_->OnResourceOveruse(reason); } void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 465d611eee..fee106703c 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -117,12 +117,13 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // be called on |encoder_queue_|. rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } - // These methods are protected for easier testing. - // TODO(hbos): When "DropDueToSize" no longer causes TriggerAdaptDown(), these - // methods are only used for testing and can be removed in favor of the test - // invoking AdaptUp() or AdaptDown() on a test-injected adaptation module. - void TriggerAdaptUp(AdaptationObserverInterface::AdaptReason reason); - bool TriggerAdaptDown(AdaptationObserverInterface::AdaptReason reason); + // TODO(https://crbug.com/webrtc/11222): When the concept of "resources" that + // can be overused or underused has materialized, trigger overuse/underuse by + // injecting a fake Resource instead and remove these methods. + void OnResourceUnderuseForTesting( + AdaptationObserverInterface::AdaptReason reason); + bool OnResourceOveruseForTesting( + AdaptationObserverInterface::AdaptReason reason); void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions) override; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index fcdac2e513..472027b478 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -166,9 +166,9 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { rtc::Event event; encoder_queue()->PostTask([this, &event, reason, down, expected_results] { if (down) - EXPECT_EQ(expected_results, TriggerAdaptDown(reason)); + EXPECT_EQ(expected_results, OnResourceOveruseForTesting(reason)); else - TriggerAdaptUp(reason); + OnResourceUnderuseForTesting(reason); event.Set(); }); ASSERT_TRUE(event.Wait(5000));