From c81798b0c44d0b0098528e547874e7f24ab0f3d8 Mon Sep 17 00:00:00 2001 From: Evan Shrubsole Date: Mon, 3 Feb 2020 15:46:08 +0100 Subject: [PATCH] Configure QP scaler in adaptation module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:11222 Change-Id: Ia50ba3d024d0cbbaeddf8bf67ee652be602c5df9 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168052 Reviewed-by: Erik Språng Reviewed-by: Henrik Boström Commit-Queue: Evan Shrubsole Cr-Commit-Position: refs/heads/master@{#30453} --- .../resource_adaptation_module_interface.h | 2 + ...ame_detector_resource_adaptation_module.cc | 49 ++++++++++++++-- ...rame_detector_resource_adaptation_module.h | 26 ++++----- video/video_stream_encoder.cc | 57 +------------------ video/video_stream_encoder.h | 2 - 5 files changed, 60 insertions(+), 76 deletions(-) diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h index 4f4a302fb2..94968426f9 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -118,6 +118,8 @@ class ResourceAdaptationModuleInterface { // or ever (for example if encoding is paused). // TODO(eshr): Try replace OnMaybeEncodeFrame and merge behaviour into // EncodeStarted. + // TODO(eshr): Try to merge OnFrame, OnFrameDroppedDueToSize, and + // OnMaybeEncode frame into one method. virtual void OnMaybeEncodeFrame() = 0; // 2.iii) An input frame is about to be encoded. It may have been cropped and // have different dimensions than what was observed at OnFrame(). Next diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index c67e47e262..cf93309e75 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -364,6 +364,7 @@ OveruseFrameDetectorResourceAdaptationModule:: target_frame_rate_(absl::nullopt), target_bitrate_bps_(absl::nullopt), quality_scaler_(nullptr), + quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), encoder_settings_(absl::nullopt), encoder_stats_observer_(encoder_stats_observer), initial_framedrop_(0) { @@ -398,6 +399,7 @@ void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { overuse_detector_->StopCheckForOveruse(); overuse_detector_is_started_ = false; + quality_scaler_.reset(); } void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( @@ -531,6 +533,48 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( } } +void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info) { + const auto scaling_settings = encoder_info.scaling_settings; + const bool quality_scaling_allowed = + IsResolutionScalingEnabled(degradation_preference_) && + scaling_settings.thresholds; + + if (quality_scaling_allowed) { + if (quality_scaler_ == nullptr) { + // Quality scaler has not already been configured. + + // Use experimental thresholds if available. + absl::optional experimental_thresholds; + if (quality_scaling_experiment_enabled_) { + experimental_thresholds = QualityScalingExperiment::GetQpThresholds( + GetVideoCodecTypeOrGeneric()); + } + UpdateQualityScalerSettings(experimental_thresholds + ? *experimental_thresholds + : *(scaling_settings.thresholds)); + } + } else { + UpdateQualityScalerSettings(absl::nullopt); + } + + // Set the qp-thresholds to the balanced settings if balanced mode. + if (degradation_preference_ == DegradationPreference::BALANCED && + quality_scaler_) { + absl::optional thresholds = + balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(), + LastInputFrameSizeOrDefault()); + if (thresholds) { + quality_scaler_->SetQpThresholds(*thresholds); + } + } + + encoder_stats_observer_->OnAdaptationChanged( + VideoStreamEncoderObserver::AdaptationReason::kNone, + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), + GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); +} + void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { if (!has_input_video_) return; @@ -906,11 +950,6 @@ OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() { return adapt_counters_[degradation_preference_]; } -absl::optional -OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const { - return balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(), - LastInputFrameSizeOrDefault()); -} bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( int pixels, uint32_t bitrate_bps) const { diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 379e1ad723..12471c7b36 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -90,9 +90,10 @@ class OveruseFrameDetectorResourceAdaptationModule // TODO(eshr): Remove once all qp-scaling is in this class. void ResetInitialFrameDropping(); - // Use nullopt to disable quality scaling. - void UpdateQualityScalerSettings( - absl::optional qp_thresholds); + // TODO(eshr): This can be made private if we configure on + // SetDegredationPreference and SetEncoderSettings. + // (https://crbug.com/webrtc/11338) + void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); class AdaptCounter final { public: @@ -141,23 +142,11 @@ class OveruseFrameDetectorResourceAdaptationModule void AdaptUp(AdaptReason reason) override; bool AdaptDown(AdaptReason reason) override; - // Used by VideoStreamEncoder when ConfigureQualityScaler() occurs and the - // |encoder_stats_observer_| is called outside of this class. - // TODO(hbos): Decouple quality scaling and resource adaptation logic and make - // this method private. - VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( - AdaptReason reason); - // Used by VideoStreamEncoder::MaybeEncodeVideoFrame(). // TODO(hbos): VideoStreamEncoder should not be responsible for any part of // the adaptation. Move this logic to this module? const AdaptCounter& GetConstAdaptCounter(); - // Used by VideoStreamEncoder::ConfigureQualityScaler(). - // TODO(hbos): Decouple quality scaling and resource adaptation logic and - // delete this method. - absl::optional GetQpThresholds() const; - private: class VideoSourceRestrictor; @@ -173,6 +162,8 @@ class OveruseFrameDetectorResourceAdaptationModule CpuOveruseOptions GetCpuOveruseOptions() const; VideoCodecType GetVideoCodecTypeOrGeneric() const; int LastInputFrameSizeOrDefault() const; + VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( + AdaptReason reason); // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener @@ -183,6 +174,10 @@ class OveruseFrameDetectorResourceAdaptationModule // started. void MaybeUpdateTargetFrameRate(); + // Use nullopt to disable quality scaling. + void UpdateQualityScalerSettings( + absl::optional qp_thresholds); + void UpdateAdaptationStats(AdaptReason reason); DegradationPreference EffectiveDegradataionPreference(); AdaptCounter& GetAdaptCounter(); @@ -212,6 +207,7 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional target_frame_rate_; absl::optional target_bitrate_bps_; std::unique_ptr quality_scaler_; + const bool quality_scaling_experiment_enabled_; absl::optional encoder_settings_; VideoStreamEncoderObserver* const encoder_stats_observer_; // Counts how many frames we've dropped in the initial framedrop phase. diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index e1fdb8c2c6..cef407a42c 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -63,11 +63,6 @@ const int64_t kParameterUpdateIntervalMs = 1000; // Animation is capped to 720p. constexpr int kMaxAnimationPixels = 1280 * 720; -bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { - return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE || - degradation_preference == DegradationPreference::BALANCED; -} - bool RequiresEncoderReset(const VideoCodec& prev_send_codec, const VideoCodec& new_send_codec, bool was_encode_called_since_last_initialization) { @@ -337,7 +332,6 @@ void VideoStreamEncoder::Stop() { rate_allocator_ = nullptr; bitrate_observer_ = nullptr; ReleaseEncoder(); - resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt); shutdown_event_.Set(); }); @@ -377,7 +371,8 @@ void VideoStreamEncoder::SetSource( resource_adaptation_module_->SetDegradationPreference( degradation_preference); if (encoder_) - ConfigureQualityScaler(encoder_->GetEncoderInfo()); + resource_adaptation_module_->ConfigureQualityScaler( + encoder_->GetEncoderInfo()); }); } @@ -768,53 +763,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { std::move(streams), encoder_config_.content_type, encoder_config_.min_transmit_bitrate_bps); - ConfigureQualityScaler(info); -} - -void VideoStreamEncoder::ConfigureQualityScaler( - const VideoEncoder::EncoderInfo& encoder_info) { - RTC_DCHECK_RUN_ON(&encoder_queue_); - const auto scaling_settings = encoder_info.scaling_settings; - const bool quality_scaling_allowed = - IsResolutionScalingEnabled( - resource_adaptation_module_->degradation_preference()) && - scaling_settings.thresholds; - - if (quality_scaling_allowed) { - if (resource_adaptation_module_->quality_scaler() == nullptr) { - // Quality scaler has not already been configured. - - // Use experimental thresholds if available. - absl::optional experimental_thresholds; - if (quality_scaling_experiment_enabled_) { - experimental_thresholds = QualityScalingExperiment::GetQpThresholds( - encoder_config_.codec_type); - } - resource_adaptation_module_->UpdateQualityScalerSettings( - experimental_thresholds ? *experimental_thresholds - : *(scaling_settings.thresholds)); - } - } else { - resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt); - } - - QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler(); - if (resource_adaptation_module_->degradation_preference() == - DegradationPreference::BALANCED && - quality_scaler && last_frame_info_) { - absl::optional thresholds = - resource_adaptation_module_->GetQpThresholds(); - if (thresholds) { - quality_scaler->SetQpThresholds(*thresholds); - } - } - - encoder_stats_observer_->OnAdaptationChanged( - VideoStreamEncoderObserver::AdaptationReason::kNone, - resource_adaptation_module_->GetActiveCounts( - AdaptationObserverInterface::AdaptReason::kCpu), - resource_adaptation_module_->GetActiveCounts( - AdaptationObserverInterface::AdaptReason::kQuality)); + resource_adaptation_module_->ConfigureQualityScaler(info); } void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index d0b037df1d..1258054be8 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -161,8 +161,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_); - void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); - // Implements VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; void OnDiscardedFrame() override;