diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index edced6dd9a..5671607739 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -72,12 +72,154 @@ const int kMaxInitialFramedrop = 4; } // namespace +// Handles interaction with the OveruseDetector. +class OveruseFrameDetectorResourceAdaptationModule::EncodeUsageResource + : public AdaptationObserverInterface { + public: + EncodeUsageResource(OveruseFrameDetectorResourceAdaptationModule* module, + std::unique_ptr overuse_detector) + : module_(module), + overuse_detector_(std::move(overuse_detector)), + is_started_(false), + target_frame_rate_(absl::nullopt) { + RTC_DCHECK(module_); + RTC_DCHECK(overuse_detector_); + } + + void StartCheckForOveruse(CpuOveruseOptions options) { + RTC_DCHECK(!is_started_); + overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(), + std::move(options), this); + is_started_ = true; + overuse_detector_->OnTargetFramerateUpdated(TargetFrameRateAsInt()); + } + + void StopCheckForOveruse() { + overuse_detector_->StopCheckForOveruse(); + is_started_ = false; + } + + void SetTargetFrameRate(absl::optional target_frame_rate) { + if (target_frame_rate == target_frame_rate_) + return; + target_frame_rate_ = target_frame_rate; + if (is_started_) + overuse_detector_->OnTargetFramerateUpdated(TargetFrameRateAsInt()); + } + + void OnEncodeStarted(const VideoFrame& cropped_frame, + int64_t time_when_first_seen_us) { + // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g. + // "OnEncodeStarted"?) or revise usage. + overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us); + } + + void OnEncodeCompleted(uint32_t timestamp, + int64_t time_sent_in_us, + int64_t capture_time_us, + absl::optional encode_duration_us) { + // TODO(hbos): Rename FrameSent() to something more appropriate (e.g. + // "OnEncodeCompleted"?). + overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, + encode_duration_us); + } + + // AdaptationObserverInterface implementation. + void AdaptUp(AdaptReason reason) override { + RTC_DCHECK_EQ(reason, AdaptReason::kCpu); + module_->OnResourceUnderuse(reason); + } + bool AdaptDown(AdaptReason reason) override { + RTC_DCHECK_EQ(reason, AdaptReason::kCpu); + return module_->OnResourceOveruse(reason); + } + + private: + int TargetFrameRateAsInt() { + return target_frame_rate_.has_value() + ? static_cast(target_frame_rate_.value()) + : std::numeric_limits::max(); + } + + OveruseFrameDetectorResourceAdaptationModule* const module_; + const std::unique_ptr overuse_detector_; + bool is_started_; + absl::optional target_frame_rate_; +}; + +// Handles interaction with the QualityScaler. +class OveruseFrameDetectorResourceAdaptationModule::QualityScalerResource + : public AdaptationObserverInterface { + public: + explicit QualityScalerResource( + OveruseFrameDetectorResourceAdaptationModule* module) + : module_(module), quality_scaler_(nullptr) { + RTC_DCHECK(module_); + } + + bool is_started() const { return quality_scaler_.get(); } + // TODO(https://crbug.com/webrtc/11222): Don't expose the quality scaler. + QualityScaler* quality_scaler() const { return quality_scaler_.get(); } + + void StartCheckForOveruse(VideoEncoder::QpThresholds qp_thresholds) { + RTC_DCHECK(!is_started()); + quality_scaler_ = + std::make_unique(this, std::move(qp_thresholds)); + } + + void StopCheckForOveruse() { quality_scaler_.reset(); } + + void SetQpThresholds(VideoEncoder::QpThresholds qp_thresholds) { + RTC_DCHECK(is_started()); + quality_scaler_->SetQpThresholds(std::move(qp_thresholds)); + } + + bool QpFastFilterLow() { + RTC_DCHECK(is_started()); + return quality_scaler_->QpFastFilterLow(); + } + + void OnEncodeCompleted(const EncodedImage& encoded_image, + int64_t time_sent_in_us) { + if (quality_scaler_ && encoded_image.qp_ >= 0) + quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us); + } + + void OnFrameDropped(EncodedImageCallback::DropReason reason) { + if (!quality_scaler_) + return; + switch (reason) { + case EncodedImageCallback::DropReason::kDroppedByMediaOptimizations: + quality_scaler_->ReportDroppedFrameByMediaOpt(); + break; + case EncodedImageCallback::DropReason::kDroppedByEncoder: + quality_scaler_->ReportDroppedFrameByEncoder(); + break; + } + } + + // AdaptationObserverInterface implementation. + void AdaptUp(AdaptReason reason) override { + RTC_DCHECK_EQ(reason, AdaptReason::kQuality); + module_->OnResourceUnderuse(reason); + } + bool AdaptDown(AdaptReason reason) override { + RTC_DCHECK_EQ(reason, AdaptReason::kQuality); + return module_->OnResourceOveruse(reason); + } + + private: + OveruseFrameDetectorResourceAdaptationModule* const module_; + std::unique_ptr quality_scaler_; +}; + // VideoSourceRestrictor is responsible for keeping track of current // VideoSourceRestrictions and how to modify them in response to adapting up or // down. It is not reponsible for determining when we should adapt up or down - -// for that, see OveruseFrameDetectorResourceAdaptationModule::AdaptUp() and -// AdaptDown() - only how to modify the source/sink restrictions when this -// happens. Note that it is also not responsible for reconfigruring the +// for that, see +// OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse() and +// OnResourceOveruse() - only how to modify the source/sink restrictions when +// this happens. Note that it is also not responsible for reconfigruring the // source/sink, it is only a keeper of desired restrictions. class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { public: @@ -224,9 +366,10 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { public: AdaptCounter() { - fps_counters_.resize(kScaleReasonSize); - resolution_counters_.resize(kScaleReasonSize); - static_assert(kScaleReasonSize == 2, "Update MoveCount."); + fps_counters_.resize(AdaptationObserverInterface::kScaleReasonSize); + resolution_counters_.resize(AdaptationObserverInterface::kScaleReasonSize); + static_assert(AdaptationObserverInterface::kScaleReasonSize == 2, + "Update MoveCount."); } ~AdaptCounter() = default; @@ -258,7 +401,8 @@ class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; MoveCount(&resolution_counters_, reason); - MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize); + MoveCount(&fps_counters_, + (reason + 1) % AdaptationObserverInterface::kScaleReasonSize); } --(fps_counters_[reason]); RTC_DCHECK_GE(fps_counters_[reason], 0); @@ -270,7 +414,8 @@ class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; MoveCount(&fps_counters_, reason); - MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize); + MoveCount(&resolution_counters_, + (reason + 1) % AdaptationObserverInterface::kScaleReasonSize); } --(resolution_counters_[reason]); RTC_DCHECK_GE(resolution_counters_[reason], 0); @@ -297,7 +442,8 @@ class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { private: std::string ToString(const std::vector& counters) const { rtc::StringBuilder ss; - for (size_t reason = 0; reason < kScaleReasonSize; ++reason) { + for (size_t reason = 0; + reason < AdaptationObserverInterface::kScaleReasonSize; ++reason) { ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; } return ss.Release(); @@ -308,7 +454,8 @@ class OveruseFrameDetectorResourceAdaptationModule::AdaptCounter final { } void MoveCount(std::vector* counters, int from_reason) { - int to_reason = (from_reason + 1) % kScaleReasonSize; + int to_reason = + (from_reason + 1) % AdaptationObserverInterface::kScaleReasonSize; ++((*counters)[to_reason]); --((*counters)[from_reason]); } @@ -335,13 +482,14 @@ OveruseFrameDetectorResourceAdaptationModule:: balanced_settings_(), last_adaptation_request_(absl::nullopt), source_restrictor_(std::make_unique()), - overuse_detector_(std::move(overuse_detector)), - overuse_detector_is_started_(false), + encode_usage_resource_( + std::make_unique(this, + std::move(overuse_detector))), + quality_scaler_resource_(std::make_unique(this)), + quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), last_input_frame_size_(absl::nullopt), target_frame_rate_(absl::nullopt), encoder_target_bitrate_bps_(absl::nullopt), - quality_scaler_(nullptr), - quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), quality_rampup_done_(false), quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), @@ -349,7 +497,6 @@ OveruseFrameDetectorResourceAdaptationModule:: encoder_stats_observer_(encoder_stats_observer), initial_framedrop_(0) { RTC_DCHECK(adaptation_listener_); - RTC_DCHECK(overuse_detector_); RTC_DCHECK(encoder_stats_observer_); } @@ -359,32 +506,21 @@ OveruseFrameDetectorResourceAdaptationModule:: void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( ResourceAdaptationModuleListener* adaptation_listener) { RTC_DCHECK(encoder_settings_.has_value()); - RTC_DCHECK(!overuse_detector_is_started_); - // TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside - // the interval between StartCheckForOveruse() and StopCheckForOveruse(), - // support configuring which |adaptation_listener_| to use on the fly. It is - // currently hardcoded for the entire lifetime of the module in order to - // support adaptation caused by VideoStreamEncoder or QualityScaler invoking - // AdaptUp() and AdaptDown() even when the OveruseDetector is inactive. + // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener + // should be passed in and why. If resources are separated from modules then + // those resources may be started or stopped separately from the module. RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); - overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(), - GetCpuOveruseOptions(), this); - overuse_detector_is_started_ = true; - overuse_detector_->OnTargetFramerateUpdated( - target_frame_rate_.has_value() - ? static_cast(target_frame_rate_.value()) - : std::numeric_limits::max()); + encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions()); } void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { - overuse_detector_->StopCheckForOveruse(); - overuse_detector_is_started_ = false; - quality_scaler_.reset(); + encode_usage_resource_->StopCheckForOveruse(); + quality_scaler_resource_->StopCheckForOveruse(); } void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( bool has_input_video) { - // While false, AdaptUp() and AdaptDown() are NO-OPS. + // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS. has_input_video_ = has_input_video; } @@ -430,8 +566,11 @@ void OveruseFrameDetectorResourceAdaptationModule::SetTargetBitrate( encoder_target_bitrate_bps_ = target_bitrate.bps(); // Check for bwe drop experiment + // TODO(https://crbug.com/webrtc/11222): Should this move to + // QualityScalerResource? if (start_bitrate_.set_start_bitrate_ > DataRate::Zero() && - !start_bitrate_.has_seen_first_bwe_drop_ && quality_scaler_ && + !start_bitrate_.has_seen_first_bwe_drop_ && + quality_scaler_resource_->is_started() && quality_scaler_settings_.InitialBitrateIntervalMs() && quality_scaler_settings_.InitialBitrateFactor()) { int64_t diff_ms = clock_->TimeInMilliseconds() - @@ -472,12 +611,12 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { AdaptationObserverInterface::AdaptReason::kQuality); int res_count = GetConstAdaptCounter().ResolutionCount( AdaptationObserverInterface::AdaptReason::kQuality); - AdaptDown(AdaptationObserverInterface::AdaptReason::kQuality); + OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); if (degradation_preference() == DegradationPreference::BALANCED && GetConstAdaptCounter().FramerateCount( AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) { // Adapt framerate in same step as resolution. - AdaptDown(AdaptationObserverInterface::AdaptReason::kQuality); + OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); } if (GetConstAdaptCounter().ResolutionCount( AdaptationObserverInterface::AdaptReason::kQuality) > res_count) { @@ -489,39 +628,27 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted( const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) { - // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g. - // "OnEncodeStarted"?) or revise usage. - overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us); + encode_usage_resource_->OnEncodeStarted(cropped_frame, + time_when_first_seen_us); } void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted( const EncodedImage& encoded_image, int64_t time_sent_in_us, absl::optional encode_duration_us) { - // TODO(hbos): Rename FrameSent() to something more appropriate (e.g. - // "OnEncodeCompleted"?). + // Inform |encode_usage_resource_| of the encode completed event. uint32_t timestamp = encoded_image.Timestamp(); int64_t capture_time_us = encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; - overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, - encode_duration_us); - if (quality_scaler_ && encoded_image.qp_ >= 0) - quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us); + encode_usage_resource_->OnEncodeCompleted( + timestamp, time_sent_in_us, capture_time_us, encode_duration_us); + // Inform |quality_scaler_resource_| of the encode completed event. + quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); } void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped( EncodedImageCallback::DropReason reason) { - if (!quality_scaler_) { - return; - } - switch (reason) { - case EncodedImageCallback::DropReason::kDroppedByMediaOptimizations: - quality_scaler_->ReportDroppedFrameByMediaOpt(); - break; - case EncodedImageCallback::DropReason::kDroppedByEncoder: - quality_scaler_->ReportDroppedFrameByEncoder(); - break; - } + quality_scaler_resource_->OnFrameDropped(reason); } void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() { @@ -536,12 +663,12 @@ bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const { void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( absl::optional qp_thresholds) { if (qp_thresholds.has_value()) { - quality_scaler_ = - std::make_unique(this, qp_thresholds.value()); + quality_scaler_resource_->StopCheckForOveruse(); + quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value()); // Restart frame drops due to size. initial_framedrop_ = 0; } else { - quality_scaler_ = nullptr; + quality_scaler_resource_->StopCheckForOveruse(); // Quality scaling disabled so we shouldn't drop initial frames. initial_framedrop_ = kMaxInitialFramedrop; } @@ -554,8 +681,10 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( IsResolutionScalingEnabled(degradation_preference_) && scaling_settings.thresholds; + // TODO(https://crbug.com/webrtc/11222): Should this move to + // QualityScalerResource? if (quality_scaling_allowed) { - if (quality_scaler_ == nullptr) { + if (!quality_scaler_resource_->is_started()) { // Quality scaler has not already been configured. // Use experimental thresholds if available. @@ -574,12 +703,12 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( // Set the qp-thresholds to the balanced settings if balanced mode. if (degradation_preference_ == DegradationPreference::BALANCED && - quality_scaler_) { + quality_scaler_resource_->is_started()) { absl::optional thresholds = balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault()); if (thresholds) { - quality_scaler_->SetQpThresholds(*thresholds); + quality_scaler_resource_->SetQpThresholds(*thresholds); } } @@ -589,7 +718,8 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); } -void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { +void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( + AdaptationObserverInterface::AdaptReason reason) { if (!has_input_video_) return; const AdaptCounter& adapt_counter = GetConstAdaptCounter(); @@ -621,7 +751,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { switch (EffectiveDegradataionPreference()) { case DegradationPreference::BALANCED: { // Check if quality should be increased based on bitrate. - if (reason == kQuality && + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && !balanced_settings_.CanAdaptUp( GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault(), encoder_target_bitrate_bps_.value_or(0))) { @@ -642,7 +772,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { break; } // Check if resolution should be increased based on bitrate. - if (reason == kQuality && + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && !balanced_settings_.CanAdaptUpResolution( GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault(), encoder_target_bitrate_bps_.value_or(0))) { @@ -654,7 +784,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { case DegradationPreference::MAINTAIN_FRAMERATE: { // Check if resolution should be increased based on bitrate and // limits specified by encoder capabilities. - if (reason == kQuality && + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && !CanAdaptUpResolution(LastInputFrameSizeOrDefault(), encoder_target_bitrate_bps_.value_or(0))) { return; @@ -702,8 +832,8 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { RTC_LOG(LS_INFO) << adapt_counter.ToString(); } -bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( - AdaptReason reason) { +bool OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( + AdaptationObserverInterface::AdaptReason reason) { if (!has_input_video_) return false; AdaptationRequest adaptation_request = { @@ -847,8 +977,8 @@ int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault() // yet. // TODO(hbos): Can we simply DCHECK has_value() before usage instead? Having a // DCHECK passed all the tests but adding it does change the requirements of - // this class (= not being allowed to call AdaptUp() or AdaptDown() before - // OnFrame()) and deserves a standalone CL. + // this class (= not being allowed to call OnResourceUnderuse() or + // OnResourceOveruse() before OnFrame()) and deserves a standalone CL. return last_input_frame_size_.value_or( VideoStreamEncoder::kDefaultLastFrameInfoWidth * VideoStreamEncoder::kDefaultLastFrameInfoHeight); @@ -886,53 +1016,47 @@ void OveruseFrameDetectorResourceAdaptationModule:: codec_max_frame_rate.value() < target_frame_rate.value())) { target_frame_rate = codec_max_frame_rate; } - if (target_frame_rate != target_frame_rate_) { - target_frame_rate_ = target_frame_rate; - if (overuse_detector_is_started_) { - overuse_detector_->OnTargetFramerateUpdated( - target_frame_rate_.has_value() - ? static_cast(target_frame_rate_.value()) - : std::numeric_limits::max()); - } - } + encode_usage_resource_->SetTargetFrameRate(target_frame_rate); } // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( - AdaptReason reason) { + AdaptationObserverInterface::AdaptReason reason) { switch (reason) { - case kCpu: + case AdaptationObserverInterface::AdaptReason::kCpu: encoder_stats_observer_->OnAdaptationChanged( VideoStreamEncoderObserver::AdaptationReason::kCpu, - GetActiveCounts(kCpu), GetActiveCounts(kQuality)); + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); break; - case kQuality: + case AdaptationObserverInterface::AdaptReason::kQuality: encoder_stats_observer_->OnAdaptationChanged( VideoStreamEncoderObserver::AdaptationReason::kQuality, - GetActiveCounts(kCpu), GetActiveCounts(kQuality)); + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); break; } } VideoStreamEncoderObserver::AdaptationSteps OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( - AdaptReason reason) { + AdaptationObserverInterface::AdaptReason reason) { VideoStreamEncoderObserver::AdaptationSteps counts = GetConstAdaptCounter().Counts(reason); switch (reason) { - case kCpu: + case AdaptationObserverInterface::AdaptReason::kCpu: if (!IsFramerateScalingEnabled(degradation_preference_)) counts.num_framerate_reductions = absl::nullopt; if (!IsResolutionScalingEnabled(degradation_preference_)) counts.num_resolution_reductions = absl::nullopt; break; - case kQuality: + case AdaptationObserverInterface::AdaptReason::kQuality: if (!IsFramerateScalingEnabled(degradation_preference_) || - !quality_scaler_) { + !quality_scaler_resource_->is_started()) { counts.num_framerate_reductions = absl::nullopt; } if (!IsResolutionScalingEnabled(degradation_preference_) || - !quality_scaler_) { + !quality_scaler_resource_->is_started()) { counts.num_resolution_reductions = absl::nullopt; } break; @@ -983,7 +1107,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( void OveruseFrameDetectorResourceAdaptationModule:: MaybePerformQualityRampupExperiment() { - if (!quality_scaler_) + if (!quality_scaler_resource_->is_started()) return; if (quality_rampup_done_) @@ -1000,7 +1124,7 @@ void OveruseFrameDetectorResourceAdaptationModule:: if (encoder_settings_ && encoder_target_bitrate_bps_.value_or(0) == encoder_settings_->video_codec().maxBitrate * 1000 && - quality_scaler_->QpFastFilterLow()) { + quality_scaler_resource_->QpFastFilterLow()) { try_quality_rampup = true; } } diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 6c841dc7a7..437510be2c 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -51,8 +51,7 @@ class VideoStreamEncoder; // generic interface in VideoStreamEncoder, unblocking other modules from being // implemented and used. class OveruseFrameDetectorResourceAdaptationModule - : public ResourceAdaptationModuleInterface, - public AdaptationObserverInterface { + : public ResourceAdaptationModuleInterface { public: // The module can be constructed on any sequence, but must be initialized and // used on a single sequence, e.g. the encoder queue. @@ -91,6 +90,11 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional encode_duration_us) override; void OnFrameDropped(EncodedImageCallback::DropReason reason) override; + // TODO(hbos): Is dropping initial frames really just a special case of "don't + // encode frames right now"? Can this be part of VideoSourceRestrictions, + // which handles the output of the rest of the encoder settings? This is + // something we'll need to support for "disable video due to overuse", not + // initial frames. bool DropInitialFrames() const; // TODO(eshr): This can be made private if we configure on @@ -98,20 +102,20 @@ class OveruseFrameDetectorResourceAdaptationModule // (https://crbug.com/webrtc/11338) void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); - // AdaptationObserverInterface implementation. Used both "internally" as - // feedback from |overuse_detector_|, and externally from VideoStreamEncoder: - // - It is wired to the VideoStreamEncoder::quality_scaler_. - // - It is invoked by VideoStreamEncoder::MaybeEncodeVideoFrame(). - // TODO(hbos): Decouple quality scaling and resource adaptation, or find an - // interface for reconfiguring externally. - // TODO(hbos): VideoStreamEncoder should not be responsible for any part of - // the adaptation. - void AdaptUp(AdaptReason reason) override; - bool AdaptDown(AdaptReason reason) override; + // Signal that a resource (kCpu or kQuality) is overused or underused. This is + // currently used by EncodeUsageResource, QualityScalerResource and testing. + // TODO(https://crbug.com/webrtc/11222): Make use of ResourceUsageState and + // implement resources per call/adaptation/resource.h. When adaptation happens + // because a resource is in specific usage state, get rid of these explicit + // triggers. + void OnResourceUnderuse(AdaptationObserverInterface::AdaptReason reason); + bool OnResourceOveruse(AdaptationObserverInterface::AdaptReason reason); private: - class AdaptCounter; + class EncodeUsageResource; + class QualityScalerResource; class VideoSourceRestrictor; + class AdaptCounter; struct AdaptationRequest { // The pixel count produced by the source at the time of the adaptation. @@ -132,23 +136,22 @@ class OveruseFrameDetectorResourceAdaptationModule VideoCodecType GetVideoCodecTypeOrGeneric() const; int LastInputFrameSizeOrDefault() const; VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( - AdaptReason reason); + AdaptationObserverInterface::AdaptReason reason); const AdaptCounter& GetConstAdaptCounter(); // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener // to reconfigure the source accordingly. void MaybeUpdateVideoSourceRestrictions(); - // Calculates an up-to-date value of |target_frame_rate_| and informs the - // |overuse_detector_| of the new value if it changed and the detector is - // started. + // Calculates an up-to-date value of the target frame rate and informs the + // |encode_usage_resource_| of the new value. void MaybeUpdateTargetFrameRate(); // Use nullopt to disable quality scaling. void UpdateQualityScalerSettings( absl::optional qp_thresholds); - void UpdateAdaptationStats(AdaptReason reason); + void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason); DegradationPreference EffectiveDegradataionPreference(); AdaptCounter& GetAdaptCounter(); bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const; @@ -180,15 +183,14 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional last_adaptation_request_; // Keeps track of source restrictions that this adaptation module outputs. const std::unique_ptr source_restrictor_; - const std::unique_ptr overuse_detector_; - bool overuse_detector_is_started_; + const std::unique_ptr encode_usage_resource_; + const std::unique_ptr quality_scaler_resource_; + const bool quality_scaling_experiment_enabled_; absl::optional last_input_frame_size_; absl::optional target_frame_rate_; // This is the last non-zero target bitrate for the encoder. absl::optional encoder_target_bitrate_bps_; absl::optional encoder_rates_; - std::unique_ptr quality_scaler_; - const bool quality_scaling_experiment_enabled_; const QualityScalerSettings quality_scaler_settings_; bool quality_rampup_done_; QualityRampupExperiment quality_rampup_experiment_; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index e90dd0cc0d..a8fbac9de9 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1577,16 +1577,16 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { return false; } -bool VideoStreamEncoder::TriggerAdaptDown( +void VideoStreamEncoder::OnResourceUnderuseForTesting( AdaptationObserverInterface::AdaptReason reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); - return resource_adaptation_module_->AdaptDown(reason); + resource_adaptation_module_->OnResourceUnderuse(reason); } -void VideoStreamEncoder::TriggerAdaptUp( +bool VideoStreamEncoder::OnResourceOveruseForTesting( AdaptationObserverInterface::AdaptReason reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); - resource_adaptation_module_->AdaptUp(reason); + return resource_adaptation_module_->OnResourceOveruse(reason); } void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 465d611eee..fee106703c 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -117,12 +117,13 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // be called on |encoder_queue_|. rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } - // These methods are protected for easier testing. - // TODO(hbos): When "DropDueToSize" no longer causes TriggerAdaptDown(), these - // methods are only used for testing and can be removed in favor of the test - // invoking AdaptUp() or AdaptDown() on a test-injected adaptation module. - void TriggerAdaptUp(AdaptationObserverInterface::AdaptReason reason); - bool TriggerAdaptDown(AdaptationObserverInterface::AdaptReason reason); + // TODO(https://crbug.com/webrtc/11222): When the concept of "resources" that + // can be overused or underused has materialized, trigger overuse/underuse by + // injecting a fake Resource instead and remove these methods. + void OnResourceUnderuseForTesting( + AdaptationObserverInterface::AdaptReason reason); + bool OnResourceOveruseForTesting( + AdaptationObserverInterface::AdaptReason reason); void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions) override; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index fcdac2e513..472027b478 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -166,9 +166,9 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { rtc::Event event; encoder_queue()->PostTask([this, &event, reason, down, expected_results] { if (down) - EXPECT_EQ(expected_results, TriggerAdaptDown(reason)); + EXPECT_EQ(expected_results, OnResourceOveruseForTesting(reason)); else - TriggerAdaptUp(reason); + OnResourceUnderuseForTesting(reason); event.Set(); }); ASSERT_TRUE(event.Wait(5000));