Configure QP scaler in adaptation module

Bug: webrtc:11222
Change-Id: Ia50ba3d024d0cbbaeddf8bf67ee652be602c5df9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168052
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Commit-Queue: Evan Shrubsole <eshr@google.com>
Cr-Commit-Position: refs/heads/master@{#30453}

call/adaptation/resource_adaptation_module_interface.h

@@ -118,6 +118,8 @@ class ResourceAdaptationModuleInterface {
   // or ever (for example if encoding is paused).
   // TODO(eshr): Try replace OnMaybeEncodeFrame and merge behaviour into
   // EncodeStarted.
+  // TODO(eshr): Try to merge OnFrame, OnFrameDroppedDueToSize, and
+  // OnMaybeEncode frame into one method.
   virtual void OnMaybeEncodeFrame() = 0;
   // 2.iii) An input frame is about to be encoded. It may have been cropped and
   // have different dimensions than what was observed at OnFrame(). Next

video/overuse_frame_detector_resource_adaptation_module.cc

@@ -364,6 +364,7 @@ OveruseFrameDetectorResourceAdaptationModule::
       target_frame_rate_(absl::nullopt),
       target_bitrate_bps_(absl::nullopt),
       quality_scaler_(nullptr),
+      quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
       encoder_settings_(absl::nullopt),
       encoder_stats_observer_(encoder_stats_observer),
       initial_framedrop_(0) {
@@ -398,6 +399,7 @@ void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation(
 void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() {
   overuse_detector_->StopCheckForOveruse();
   overuse_detector_is_started_ = false;
+  quality_scaler_.reset();
 }
 
 void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo(
@@ -531,6 +533,48 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings(
   }
 }
 
+void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler(
+    const VideoEncoder::EncoderInfo& encoder_info) {
+  const auto scaling_settings = encoder_info.scaling_settings;
+  const bool quality_scaling_allowed =
+      IsResolutionScalingEnabled(degradation_preference_) &&
+      scaling_settings.thresholds;
+
+  if (quality_scaling_allowed) {
+    if (quality_scaler_ == nullptr) {
+      // Quality scaler has not already been configured.
+
+      // Use experimental thresholds if available.
+      absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
+      if (quality_scaling_experiment_enabled_) {
+        experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
+            GetVideoCodecTypeOrGeneric());
+      }
+      UpdateQualityScalerSettings(experimental_thresholds
+                                      ? *experimental_thresholds
+                                      : *(scaling_settings.thresholds));
+    }
+  } else {
+    UpdateQualityScalerSettings(absl::nullopt);
+  }
+
+  // Set the qp-thresholds to the balanced settings if balanced mode.
+  if (degradation_preference_ == DegradationPreference::BALANCED &&
+      quality_scaler_) {
+    absl::optional<VideoEncoder::QpThresholds> thresholds =
+        balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
+                                           LastInputFrameSizeOrDefault());
+    if (thresholds) {
+      quality_scaler_->SetQpThresholds(*thresholds);
+    }
+  }
+
+  encoder_stats_observer_->OnAdaptationChanged(
+      VideoStreamEncoderObserver::AdaptationReason::kNone,
+      GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu),
+      GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality));
+}
+
 void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
   if (!has_input_video_)
     return;
@@ -906,11 +950,6 @@ OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() {
   return adapt_counters_[degradation_preference_];
 }
 
-absl::optional<VideoEncoder::QpThresholds>
-OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const {
-  return balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
-                                            LastInputFrameSizeOrDefault());
-}
 bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
     int pixels,
     uint32_t bitrate_bps) const {

video/overuse_frame_detector_resource_adaptation_module.h

@@ -90,9 +90,10 @@ class OveruseFrameDetectorResourceAdaptationModule
   // TODO(eshr): Remove once all qp-scaling is in this class.
   void ResetInitialFrameDropping();
 
-  // Use nullopt to disable quality scaling.
-  void UpdateQualityScalerSettings(
-      absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
+  // TODO(eshr): This can be made private if we configure on
+  // SetDegredationPreference and SetEncoderSettings.
+  // (https://crbug.com/webrtc/11338)
+  void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
 
   class AdaptCounter final {
    public:
@@ -141,23 +142,11 @@ class OveruseFrameDetectorResourceAdaptationModule
   void AdaptUp(AdaptReason reason) override;
   bool AdaptDown(AdaptReason reason) override;
 
-  // Used by VideoStreamEncoder when ConfigureQualityScaler() occurs and the
-  // |encoder_stats_observer_| is called outside of this class.
-  // TODO(hbos): Decouple quality scaling and resource adaptation logic and make
-  // this method private.
-  VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
-      AdaptReason reason);
-
   // Used by VideoStreamEncoder::MaybeEncodeVideoFrame().
   // TODO(hbos): VideoStreamEncoder should not be responsible for any part of
   // the adaptation. Move this logic to this module?
   const AdaptCounter& GetConstAdaptCounter();
 
-  // Used by VideoStreamEncoder::ConfigureQualityScaler().
-  // TODO(hbos): Decouple quality scaling and resource adaptation logic and
-  // delete this method.
-  absl::optional<VideoEncoder::QpThresholds> GetQpThresholds() const;
-
  private:
   class VideoSourceRestrictor;
 
@@ -173,6 +162,8 @@ class OveruseFrameDetectorResourceAdaptationModule
   CpuOveruseOptions GetCpuOveruseOptions() const;
   VideoCodecType GetVideoCodecTypeOrGeneric() const;
   int LastInputFrameSizeOrDefault() const;
+  VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
+      AdaptReason reason);
 
   // Makes |video_source_restrictions_| up-to-date and informs the
   // |adaptation_listener_| if restrictions are changed, allowing the listener
@@ -183,6 +174,10 @@ class OveruseFrameDetectorResourceAdaptationModule
   // started.
   void MaybeUpdateTargetFrameRate();
 
+  // Use nullopt to disable quality scaling.
+  void UpdateQualityScalerSettings(
+      absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
+
   void UpdateAdaptationStats(AdaptReason reason);
   DegradationPreference EffectiveDegradataionPreference();
   AdaptCounter& GetAdaptCounter();
@@ -212,6 +207,7 @@ class OveruseFrameDetectorResourceAdaptationModule
   absl::optional<double> target_frame_rate_;
   absl::optional<uint32_t> target_bitrate_bps_;
   std::unique_ptr<QualityScaler> quality_scaler_;
+  const bool quality_scaling_experiment_enabled_;
   absl::optional<EncoderSettings> encoder_settings_;
   VideoStreamEncoderObserver* const encoder_stats_observer_;
   // Counts how many frames we've dropped in the initial framedrop phase.

video/video_stream_encoder.cc

@@ -63,11 +63,6 @@ const int64_t kParameterUpdateIntervalMs = 1000;
 // Animation is capped to 720p.
 constexpr int kMaxAnimationPixels = 1280 * 720;
 
-bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
-  return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
-         degradation_preference == DegradationPreference::BALANCED;
-}
-
 bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
                           const VideoCodec& new_send_codec,
                           bool was_encode_called_since_last_initialization) {
@@ -337,7 +332,6 @@ void VideoStreamEncoder::Stop() {
     rate_allocator_ = nullptr;
     bitrate_observer_ = nullptr;
     ReleaseEncoder();
-    resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt);
     shutdown_event_.Set();
   });
 
@@ -377,7 +371,8 @@ void VideoStreamEncoder::SetSource(
     resource_adaptation_module_->SetDegradationPreference(
         degradation_preference);
     if (encoder_)
-      ConfigureQualityScaler(encoder_->GetEncoderInfo());
+      resource_adaptation_module_->ConfigureQualityScaler(
+          encoder_->GetEncoderInfo());
   });
 }
 
@@ -768,53 +763,7 @@ void VideoStreamEncoder::ReconfigureEncoder() {
       std::move(streams), encoder_config_.content_type,
       encoder_config_.min_transmit_bitrate_bps);
 
-  ConfigureQualityScaler(info);
-}
-
-void VideoStreamEncoder::ConfigureQualityScaler(
-    const VideoEncoder::EncoderInfo& encoder_info) {
-  RTC_DCHECK_RUN_ON(&encoder_queue_);
-  const auto scaling_settings = encoder_info.scaling_settings;
-  const bool quality_scaling_allowed =
-      IsResolutionScalingEnabled(
-          resource_adaptation_module_->degradation_preference()) &&
-      scaling_settings.thresholds;
-
-  if (quality_scaling_allowed) {
-    if (resource_adaptation_module_->quality_scaler() == nullptr) {
-      // Quality scaler has not already been configured.
-
-      // Use experimental thresholds if available.
-      absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
-      if (quality_scaling_experiment_enabled_) {
-        experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
-            encoder_config_.codec_type);
-      }
-      resource_adaptation_module_->UpdateQualityScalerSettings(
-          experimental_thresholds ? *experimental_thresholds
-                                  : *(scaling_settings.thresholds));
-    }
-  } else {
-    resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt);
-  }
-
-  QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler();
-  if (resource_adaptation_module_->degradation_preference() ==
-          DegradationPreference::BALANCED &&
-      quality_scaler && last_frame_info_) {
-    absl::optional<VideoEncoder::QpThresholds> thresholds =
-        resource_adaptation_module_->GetQpThresholds();
-    if (thresholds) {
-      quality_scaler->SetQpThresholds(*thresholds);
-    }
-  }
-
-  encoder_stats_observer_->OnAdaptationChanged(
-      VideoStreamEncoderObserver::AdaptationReason::kNone,
-      resource_adaptation_module_->GetActiveCounts(
-          AdaptationObserverInterface::AdaptReason::kCpu),
-      resource_adaptation_module_->GetActiveCounts(
-          AdaptationObserverInterface::AdaptReason::kQuality));
+  resource_adaptation_module_->ConfigureQualityScaler(info);
 }
 
 void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) {

video/video_stream_encoder.h

@@ -161,8 +161,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
 
   void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_);
 
-  void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
-
   // Implements VideoSinkInterface.
   void OnFrame(const VideoFrame& video_frame) override;
   void OnDiscardedFrame() override;