Add incomplete ResourceAdaptationModuleInterface.

This interface will be improved upon iteratively to aid reviewability.
The initial version only handles starting and stopping the module; input
and output of the module is still implementation-specific.

TBR=sprang@webrtc.org

Bug: webrtc:11222
Change-Id: Ie307cfe3d3211c84346c035f2c0e9a632f58221b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/162580
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Cr-Commit-Position: refs/heads/master@{#30167}

call/adaptation/BUILD.gn

@@ -12,6 +12,8 @@ rtc_library("resource_adaptation") {
   sources = [
     "resource.cc",
     "resource.h",
+    "resource_adaptation_module_interface.cc",
+    "resource_adaptation_module_interface.h",
     "resource_adaptation_processor.cc",
     "resource_adaptation_processor.h",
     "resource_consumer.cc",

call/adaptation/resource_adaptation_module_interface.cc

@@ -0,0 +1,17 @@
+/*
+ *  Copyright 2019 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "call/adaptation/resource_adaptation_module_interface.h"
+
+namespace webrtc {
+
+ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {}
+
+}  // namespace webrtc

call/adaptation/resource_adaptation_module_interface.h

@@ -0,0 +1,44 @@
+/*
+ *  Copyright 2019 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_
+#define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_
+
+namespace webrtc {
+
+// Responsible for reconfiguring encoded streams based on resource consumption,
+// such as scaling down resolution or frame rate when CPU is overused. This
+// interface is meant to be injectable into VideoStreamEncoder.
+//
+// [UNDER CONSTRUCTION] This interface is work-in-progress. In the future it
+// needs to be able to handle all the necessary input and output for resource
+// adaptation decision making.
+//
+// TODO(https://crbug.com/webrtc/11222): Make this interface feature-complete so
+// that a module (such as OveruseFrameDetectorResourceAdaptationModule) is fully
+// operational through this abstract interface.
+class ResourceAdaptationModuleInterface {
+ public:
+  virtual ~ResourceAdaptationModuleInterface();
+
+  // TODO(hbos): When input/output of the module is adequetly handled by this
+  // interface, these methods need to say which stream to start/stop, enabling
+  // multi-stream aware implementations of ResourceAdaptationModuleInterface. We
+  // don't want to do this before we have the right interfaces (e.g. if we pass
+  // in a VideoStreamEncoder here directly then have a dependency on a different
+  // build target). For the multi-stream use case we may consider making
+  // ResourceAdaptationModuleInterface reference counted.
+  virtual void StartCheckForOveruse() = 0;
+  virtual void StopCheckForOveruse() = 0;
+};
+
+}  // namespace webrtc
+
+#endif  // CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_

video/BUILD.gn

@@ -203,6 +203,7 @@ rtc_library("video_stream_encoder_impl") {
     "../api/video:video_rtp_headers",
     "../api/video:video_stream_encoder",
     "../api/video_codecs:video_codecs_api",
+    "../call/adaptation:resource_adaptation",
     "../common_video",
     "../modules:module_api_public",
     "../modules/video_coding",

video/overuse_frame_detector_resource_adaptation_module.cc

@@ -423,10 +423,12 @@ OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ToString(
 
 OveruseFrameDetectorResourceAdaptationModule::
     OveruseFrameDetectorResourceAdaptationModule(
+        VideoStreamEncoder* video_stream_encoder,
         rtc::VideoSinkInterface<VideoFrame>* sink,
         std::unique_ptr<OveruseFrameDetector> overuse_detector,
         VideoStreamEncoderObserver* encoder_stats_observer)
     : encoder_queue_(nullptr),
+      video_stream_encoder_(video_stream_encoder),
       degradation_preference_(DegradationPreference::DISABLED),
       adapt_counters_(),
       balanced_settings_(),
@@ -440,6 +442,7 @@ OveruseFrameDetectorResourceAdaptationModule::
       encoder_config_(),
       encoder_(nullptr),
       encoder_stats_observer_(encoder_stats_observer) {
+  RTC_DCHECK(video_stream_encoder_);
   RTC_DCHECK(overuse_detector_);
   RTC_DCHECK(encoder_stats_observer_);
 }
@@ -461,12 +464,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoder(
   encoder_ = encoder;
 }
 
-void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse(
-    const CpuOveruseOptions& options) {
+void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse() {
   RTC_DCHECK(encoder_queue_);
   RTC_DCHECK_RUN_ON(encoder_queue_);
   RTC_DCHECK(encoder_);
-  overuse_detector_->StartCheckForOveruse(encoder_queue_, options, this);
+  overuse_detector_->StartCheckForOveruse(
+      encoder_queue_, video_stream_encoder_->GetCpuOveruseOptions(), this);
 }
 
 void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() {

video/overuse_frame_detector_resource_adaptation_module.h

@@ -25,11 +25,14 @@
 #include "api/video/video_stream_encoder_observer.h"
 #include "api/video_codecs/video_encoder.h"
 #include "api/video_codecs/video_encoder_config.h"
+#include "call/adaptation/resource_adaptation_module_interface.h"
 #include "rtc_base/experiments/balanced_degradation_settings.h"
 #include "video/overuse_frame_detector.h"
 
 namespace webrtc {
 
+class VideoStreamEncoder;
+
 // This class is used by the VideoStreamEncoder and is responsible for adapting
 // resolution up or down based on encode usage percent. It keeps track of video
 // source settings, adaptation counters and may get influenced by
@@ -43,9 +46,11 @@ namespace webrtc {
 // generic interface in VideoStreamEncoder, unblocking other modules from being
 // implemented and used.
 class OveruseFrameDetectorResourceAdaptationModule
-    : public AdaptationObserverInterface {
+    : public ResourceAdaptationModuleInterface,
+      public AdaptationObserverInterface {
  public:
   OveruseFrameDetectorResourceAdaptationModule(
+      VideoStreamEncoder* video_stream_encoder,
       rtc::VideoSinkInterface<VideoFrame>* sink,
       std::unique_ptr<OveruseFrameDetector> overuse_detector,
       VideoStreamEncoderObserver* encoder_stats_observer);
@@ -64,13 +69,15 @@ class OveruseFrameDetectorResourceAdaptationModule
     return degradation_preference_;
   }
 
+  // ResourceAdaptationModuleInterface implementation.
+  void StartCheckForOveruse() override;
+  void StopCheckForOveruse() override;
+
   // Input to the OveruseFrameDetector, which are required for this module to
   // function. These map to OveruseFrameDetector methods.
   // TODO(hbos): Define virtual methods in ResourceAdaptationModuleInterface
   // for input that are more generic so that this class can be used without
   // assumptions about underlying implementation.
-  void StartCheckForOveruse(const CpuOveruseOptions& options);
-  void StopCheckForOveruse();
   void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us);
   void FrameSent(uint32_t timestamp,
                  int64_t time_sent_in_us,
@@ -187,7 +194,11 @@ class OveruseFrameDetectorResourceAdaptationModule
   bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const
       RTC_RUN_ON(encoder_queue_);
 
+  // TODO(hbos): Can we move the |source_proxy_| to the |encoder_queue_| and
+  // replace |encoder_queue_| with a sequence checker instead?
   rtc::TaskQueue* encoder_queue_;
+  // Used to query CpuOveruseOptions at StartCheckForOveruse().
+  VideoStreamEncoder* video_stream_encoder_ RTC_GUARDED_BY(encoder_queue_);
   DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_);
   // Counters used for deciding if the video resolution or framerate is
   // currently restricted, and if so, why, on a per degradation preference

video/video_stream_encoder.cc

@@ -84,26 +84,6 @@ bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
          degradation_preference == DegradationPreference::BALANCED;
 }
 
-// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
-// pipelining encoders better (multiple input frames before something comes
-// out). This should effectively turn off CPU adaptations for systems that
-// remotely cope with the load right now.
-CpuOveruseOptions GetCpuOveruseOptions(
-    const VideoStreamEncoderSettings& settings,
-    bool full_overuse_time) {
-  CpuOveruseOptions options;
-
-  if (full_overuse_time) {
-    options.low_encode_usage_threshold_percent = 150;
-    options.high_encode_usage_threshold_percent = 200;
-  }
-  if (settings.experiment_cpu_load_estimator) {
-    options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec;
-  }
-
-  return options;
-}
-
 bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
                           const VideoCodec& new_send_codec,
                           bool was_encode_called_since_last_initialization) {
@@ -340,7 +320,8 @@ VideoStreamEncoder::VideoStreamEncoder(
       encoder_switch_requested_(false),
       resource_adaptation_module_(
           std::make_unique<OveruseFrameDetectorResourceAdaptationModule>(
-              this,
+              /*video_stream_encoder=*/this,
+              /*sink=*/this,
               std::move(overuse_detector),
               encoder_stats_observer)),
       encoder_queue_(task_queue_factory->CreateTaskQueue(
@@ -703,8 +684,7 @@ void VideoStreamEncoder::ReconfigureEncoder() {
 
   if (pending_encoder_creation_) {
     resource_adaptation_module_->StopCheckForOveruse();
-    resource_adaptation_module_->StartCheckForOveruse(GetCpuOveruseOptions(
-        settings_, encoder_->GetEncoderInfo().is_hardware_accelerated));
+    resource_adaptation_module_->StartCheckForOveruse();
     pending_encoder_creation_ = false;
   }
 
@@ -1718,6 +1698,25 @@ bool VideoStreamEncoder::TryQualityRampup(int64_t now_ms) {
   return false;
 }
 
+// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
+// pipelining encoders better (multiple input frames before something comes
+// out). This should effectively turn off CPU adaptations for systems that
+// remotely cope with the load right now.
+CpuOveruseOptions VideoStreamEncoder::GetCpuOveruseOptions() const {
+  RTC_DCHECK_RUN_ON(&encoder_queue_);
+  CpuOveruseOptions options;
+  // Hardware accelerated encoders are assumed to be pipelined; give them
+  // additional overuse time.
+  if (encoder_->GetEncoderInfo().is_hardware_accelerated) {
+    options.low_encode_usage_threshold_percent = 150;
+    options.high_encode_usage_threshold_percent = 200;
+  }
+  if (settings_.experiment_cpu_load_estimator) {
+    options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec;
+  }
+  return options;
+}
+
 bool VideoStreamEncoder::TriggerAdaptDown(
     AdaptationObserverInterface::AdaptReason reason) {
   return resource_adaptation_module_->AdaptDown(reason);

video/video_stream_encoder.h

@@ -100,6 +100,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
                         uint8_t fraction_lost,
                         int64_t round_trip_time_ms) override;
 
+  // If an OveruseFrameDetectorResourceAdaptationModule is used, this method is
+  // used by the module to configure its OveruseFrameDetector.
+  CpuOveruseOptions GetCpuOveruseOptions() const;
+
  protected:
   // Used for testing. For example the |ScalingObserverInterface| methods must
   // be called on |encoder_queue_|.