[Overuse] Make VideoStreamAdapter responsible for executing adaptation.

This CL moves GetAdaptUpTarget(), GetAdaptDownTarget() and
ApplyAdaptationTarget() - and related code - to the VideoStreamAdapter.

This includes pieces related to calculating how to adapt, including:
- DegradationPreference
- BalancedDegradationPreference
- AdaptationRequest and last_adaptation_request_
- CanAdaptUpResolution()

The VideoStreamAdapter's interface has changed: VideoSourceRestrictor
methods are now hidden in favor of methods exposing AdaptationTarget.

This CL also does some misc moves:
- GetEncoderBitrateLimits is moved and renamed to
  VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution.
- EncoderSettings moved to a separate file.

// For api/video_codecs/video_encoder.[cc/h] changes, which is the
// moving of a function.
TBR=sprang@webrtc.org

Bug: webrtc:11393
Change-Id: Ie6bd8ef644ce927d7eca6ab90a0a7bcace682f3c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169842
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30708}

api/video_codecs/video_encoder.cc

@@ -11,6 +11,7 @@
 #include "api/video_codecs/video_encoder.h"
 
 #include <string.h>
+#include <algorithm>
 
 #include "rtc_base/checks.h"
 #include "rtc_base/strings/string_builder.h"
@@ -208,6 +209,42 @@ bool VideoEncoder::EncoderInfo::operator==(const EncoderInfo& rhs) const {
   return true;
 }
 
+absl::optional<VideoEncoder::ResolutionBitrateLimits>
+VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution(
+    int frame_size_pixels) const {
+  std::vector<ResolutionBitrateLimits> bitrate_limits =
+      resolution_bitrate_limits;
+
+  // Sort the list of bitrate limits by resolution.
+  sort(bitrate_limits.begin(), bitrate_limits.end(),
+       [](const ResolutionBitrateLimits& lhs,
+          const ResolutionBitrateLimits& rhs) {
+         return lhs.frame_size_pixels < rhs.frame_size_pixels;
+       });
+
+  for (size_t i = 0; i < bitrate_limits.size(); ++i) {
+    RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0);
+    RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0);
+    RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
+                  bitrate_limits[i].min_bitrate_bps);
+    if (i > 0) {
+      // The bitrate limits aren't expected to decrease with resolution.
+      RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps,
+                    bitrate_limits[i - 1].min_bitrate_bps);
+      RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps,
+                    bitrate_limits[i - 1].min_start_bitrate_bps);
+      RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
+                    bitrate_limits[i - 1].max_bitrate_bps);
+    }
+
+    if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) {
+      return absl::optional<ResolutionBitrateLimits>(bitrate_limits[i]);
+    }
+  }
+
+  return absl::nullopt;
+}
+
 VideoEncoder::RateControlParameters::RateControlParameters()
     : bitrate(VideoBitrateAllocation()),
       framerate_fps(0.0),

api/video_codecs/video_encoder.h

@@ -236,6 +236,11 @@ class RTC_EXPORT VideoEncoder {
     // Recommended bitrate limits for different resolutions.
     std::vector<ResolutionBitrateLimits> resolution_bitrate_limits;
 
+    // Obtains the limits from |resolution_bitrate_limits| that best matches the
+    // |frame_size_pixels|.
+    absl::optional<ResolutionBitrateLimits>
+    GetEncoderBitrateLimitsForResolution(int frame_size_pixels) const;
+
     // If true, this encoder has internal support for generating simulcast
     // streams. Otherwise, an adapter class will be needed.
     // Even if true, the config provided to InitEncode() might not be supported,

call/adaptation/BUILD.gn

@@ -10,6 +10,8 @@ import("../../webrtc.gni")
 
 rtc_library("resource_adaptation") {
   sources = [
+    "encoder_settings.cc",
+    "encoder_settings.h",
     "resource.cc",
     "resource.h",
     "resource_adaptation_module_interface.cc",

call/adaptation/encoder_settings.cc

@@ -0,0 +1,42 @@
+/*
+ *  Copyright 2020 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "call/adaptation/encoder_settings.h"
+
+#include <utility>
+
+namespace webrtc {
+
+EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
+                                 VideoEncoderConfig encoder_config,
+                                 VideoCodec video_codec)
+    : encoder_info_(std::move(encoder_info)),
+      encoder_config_(std::move(encoder_config)),
+      video_codec_(std::move(video_codec)) {}
+
+const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const {
+  return encoder_info_;
+}
+
+const VideoEncoderConfig& EncoderSettings::encoder_config() const {
+  return encoder_config_;
+}
+
+const VideoCodec& EncoderSettings::video_codec() const {
+  return video_codec_;
+}
+
+VideoCodecType GetVideoCodecTypeOrGeneric(
+    const absl::optional<EncoderSettings>& settings) {
+  return settings.has_value() ? settings->encoder_config().codec_type
+                              : kVideoCodecGeneric;
+}
+
+}  // namespace webrtc

call/adaptation/encoder_settings.h

@@ -0,0 +1,46 @@
+/*
+ *  Copyright 2020 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef CALL_ADAPTATION_ENCODER_SETTINGS_H_
+#define CALL_ADAPTATION_ENCODER_SETTINGS_H_
+
+#include "absl/types/optional.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/video_encoder_config.h"
+
+namespace webrtc {
+
+// Information about an encoder available when reconfiguring the encoder.
+class EncoderSettings {
+ public:
+  EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
+                  VideoEncoderConfig encoder_config,
+                  VideoCodec video_codec);
+
+  // Encoder capabilities, implementation info, etc.
+  const VideoEncoder::EncoderInfo& encoder_info() const;
+  // Configuration parameters, ultimately coming from the API and negotiation.
+  const VideoEncoderConfig& encoder_config() const;
+  // Lower level config, heavily based on the VideoEncoderConfig.
+  const VideoCodec& video_codec() const;
+
+ private:
+  VideoEncoder::EncoderInfo encoder_info_;
+  VideoEncoderConfig encoder_config_;
+  VideoCodec video_codec_;
+};
+
+VideoCodecType GetVideoCodecTypeOrGeneric(
+    const absl::optional<EncoderSettings>& settings);
+
+}  // namespace webrtc
+
+#endif  // CALL_ADAPTATION_ENCODER_SETTINGS_H_

call/adaptation/resource_adaptation_module_interface.cc

@@ -10,29 +10,8 @@
 
 #include "call/adaptation/resource_adaptation_module_interface.h"
 
-#include <utility>
-
 namespace webrtc {
 
-EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
-                                 VideoEncoderConfig encoder_config,
-                                 VideoCodec video_codec)
-    : encoder_info_(std::move(encoder_info)),
-      encoder_config_(std::move(encoder_config)),
-      video_codec_(std::move(video_codec)) {}
-
-const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const {
-  return encoder_info_;
-}
-
-const VideoEncoderConfig& EncoderSettings::encoder_config() const {
-  return encoder_config_;
-}
-
-const VideoCodec& EncoderSettings::video_codec() const {
-  return video_codec_;
-}
-
 ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {}
 
 ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {}

call/adaptation/resource_adaptation_module_interface.h

@@ -14,33 +14,12 @@
 #include "absl/types/optional.h"
 #include "api/rtp_parameters.h"
 #include "api/video/video_frame.h"
-#include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/encoder_settings.h"
-#include "api/video_codecs/video_encoder_config.h"
 #include "call/adaptation/resource.h"
 #include "call/adaptation/video_source_restrictions.h"
 
 namespace webrtc {
 
-// Information about an encoder available when reconfiguring the encoder.
-class EncoderSettings {
- public:
-  EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
-                  VideoEncoderConfig encoder_config,
-                  VideoCodec video_codec);
-
-  // Encoder capabilities, implementation info, etc.
-  const VideoEncoder::EncoderInfo& encoder_info() const;
-  // Configuration parameters, ultimately coming from the API and negotiation.
-  const VideoEncoderConfig& encoder_config() const;
-  // Lower level config, heavily based on the VideoEncoderConfig.
-  const VideoCodec& video_codec() const;
-
- private:
-  VideoEncoder::EncoderInfo encoder_info_;
-  VideoEncoderConfig encoder_config_;
-  VideoCodec video_codec_;
-};
-
 // The listener is responsible for carrying out the reconfiguration of the video
 // source such that the VideoSourceRestrictions are fulfilled.
 class ResourceAdaptationModuleListener {

video/adaptation/BUILD.gn

@@ -17,12 +17,17 @@ rtc_library("video_adaptation") {
   ]
 
   deps = [
+    "../../api:rtp_parameters",
+    "../../api/video:video_stream_encoder",
+    "../../api/video_codecs:video_codecs_api",
     "../../call/adaptation:resource_adaptation",
+    "../../modules/video_coding:video_coding_utility",
     "../../rtc_base:checks",
     "../../rtc_base:logging",
     "../../rtc_base:rtc_base_approved",
     "../../rtc_base:rtc_event",
     "../../rtc_base:rtc_numerics",
+    "../../rtc_base/experiments:balanced_degradation_settings",
     "//third_party/abseil-cpp/absl/types:optional",
   ]
 }

video/adaptation/video_stream_adapter.cc

@@ -14,50 +14,87 @@
 #include <limits>
 
 #include "absl/types/optional.h"
+#include "api/video_codecs/video_encoder.h"
 #include "rtc_base/constructor_magic.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/numerics/safe_conversions.h"
 
 namespace webrtc {
 
-// VideoSourceRestrictor is responsible for keeping track of current
+namespace {
-// VideoSourceRestrictions. It suggests higher and lower frame rates and
+
-// resolutions (used by "maintain-resolution" and "maintain-framerate"), but is
+const int kMinFramerateFps = 2;
-// ultimately not reponsible for determining when or how we should adapt up or
+
-// down (e.g. "balanced" mode also uses BalancedDegradationPreference).
+// Generate suggested higher and lower frame rates and resolutions, to be
-class VideoStreamAdapter::VideoSourceRestrictor {
+// applied to the VideoSourceRestrictor. These are used in "maintain-resolution"
- public:
+// and "maintain-framerate". The "balanced" degradation preference also makes
-  // For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
+// use of BalancedDegradationPreference when generating suggestions. The
-  static int GetLowerFrameRateThan(int fps) {
+// VideoSourceRestrictor decidedes whether or not a proposed adaptation is
+// valid.
+
+// For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
+int GetLowerFrameRateThan(int fps) {
   RTC_DCHECK(fps != std::numeric_limits<int>::max());
   return (fps * 2) / 3;
-  }
+}
-  // TODO(hbos): Use absl::optional<> instead?
+// TODO(hbos): Use absl::optional<> instead?
-  static int GetHigherFrameRateThan(int fps) {
+int GetHigherFrameRateThan(int fps) {
   return fps != std::numeric_limits<int>::max()
              ? (fps * 3) / 2
              : std::numeric_limits<int>::max();
-  }
+}
 
-  // For resolution, the steps we take are 3/5 (down) and 5/3 (up).
+// For resolution, the steps we take are 3/5 (down) and 5/3 (up).
-  // Notice the asymmetry of which restriction property is set depending on if
+// Notice the asymmetry of which restriction property is set depending on if
-  // we are adapting up or down:
+// we are adapting up or down:
-  // - DecreaseResolution() sets the max_pixels_per_frame() to the desired
+// - VideoSourceRestrictor::DecreaseResolution() sets the max_pixels_per_frame()
-  //   target and target_pixels_per_frame() to null.
+//   to the desired target and target_pixels_per_frame() to null.
-  // - IncreaseResolutionTo() sets the target_pixels_per_frame() to the desired
+// - VideoSourceRestrictor::IncreaseResolutionTo() sets the
-  //   target, and max_pixels_per_frame() is set according to
+//   target_pixels_per_frame() to the desired target, and max_pixels_per_frame()
-  //   GetIncreasedMaxPixelsWanted().
+//   is set according to VideoSourceRestrictor::GetIncreasedMaxPixelsWanted().
-  static int GetLowerResolutionThan(int pixel_count) {
+int GetLowerResolutionThan(int pixel_count) {
   RTC_DCHECK(pixel_count != std::numeric_limits<int>::max());
   return (pixel_count * 3) / 5;
-  }
+}
-  // TODO(hbos): Use absl::optional<> instead?
+// TODO(hbos): Use absl::optional<> instead?
-  static int GetHigherResolutionThan(int pixel_count) {
+int GetHigherResolutionThan(int pixel_count) {
   return pixel_count != std::numeric_limits<int>::max()
              ? (pixel_count * 5) / 3
              : std::numeric_limits<int>::max();
-  }
+}
 
+// One of the conditions used in VideoStreamAdapter::GetAdaptUpTarget().
+// TODO(hbos): Whether or not we can adapt up due to encoder settings and
+// bitrate should be expressed as a bandwidth-related Resource.
+bool CanAdaptUpResolution(
+    const absl::optional<EncoderSettings>& encoder_settings,
+    absl::optional<uint32_t> encoder_target_bitrate_bps,
+    int input_pixels) {
+  uint32_t bitrate_bps = encoder_target_bitrate_bps.value_or(0);
+  absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
+      encoder_settings.has_value()
+          ? encoder_settings->encoder_info()
+                .GetEncoderBitrateLimitsForResolution(
+                    GetHigherResolutionThan(input_pixels))
+          : absl::nullopt;
+  if (!bitrate_limits.has_value() || bitrate_bps == 0) {
+    return true;  // No limit configured or bitrate provided.
+  }
+  RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, input_pixels);
+  return bitrate_bps >=
+         static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
+}
+
+}  // namespace
+
+VideoStreamAdapter::AdaptationTarget::AdaptationTarget(AdaptationAction action,
+                                                       int value)
+    : action(action), value(value) {}
+
+// VideoSourceRestrictor is responsible for keeping track of current
+// VideoSourceRestrictions.
+class VideoStreamAdapter::VideoSourceRestrictor {
+ public:
   VideoSourceRestrictor() {}
 
   VideoSourceRestrictions source_restrictions() const {
@@ -168,30 +205,27 @@ class VideoStreamAdapter::VideoSourceRestrictor {
   RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
 };
 
-const int VideoStreamAdapter::kMinFramerateFps = 2;
-
 // static
-int VideoStreamAdapter::GetLowerFrameRateThan(int fps) {
+VideoStreamAdapter::AdaptationRequest::Mode
-  return VideoSourceRestrictor::GetLowerFrameRateThan(fps);
+VideoStreamAdapter::AdaptationRequest::GetModeFromAdaptationAction(
-}
+    VideoStreamAdapter::AdaptationAction action) {
-
+  switch (action) {
-// static
+    case AdaptationAction::kIncreaseResolution:
-int VideoStreamAdapter::GetHigherFrameRateThan(int fps) {
+      return AdaptationRequest::Mode::kAdaptUp;
-  return VideoSourceRestrictor::GetHigherFrameRateThan(fps);
+    case AdaptationAction::kDecreaseResolution:
-}
+      return AdaptationRequest::Mode::kAdaptDown;
-
+    case AdaptationAction::kIncreaseFrameRate:
-// static
+      return AdaptationRequest::Mode::kAdaptUp;
-int VideoStreamAdapter::GetLowerResolutionThan(int pixel_count) {
+    case AdaptationAction::kDecreaseFrameRate:
-  return VideoSourceRestrictor::GetLowerResolutionThan(pixel_count);
+      return AdaptationRequest::Mode::kAdaptDown;
-}
+  }
-
-// static
-int VideoStreamAdapter::GetHigherResolutionThan(int pixel_count) {
-  return VideoSourceRestrictor::GetHigherResolutionThan(pixel_count);
 }
 
 VideoStreamAdapter::VideoStreamAdapter()
-    : source_restrictor_(std::make_unique<VideoSourceRestrictor>()) {}
+    : source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
+      balanced_settings_(),
+      degradation_preference_(DegradationPreference::DISABLED),
+      last_adaptation_request_(absl::nullopt) {}
 
 VideoStreamAdapter::~VideoStreamAdapter() {}
 
@@ -203,43 +237,257 @@ const AdaptationCounters& VideoStreamAdapter::adaptation_counters() const {
   return source_restrictor_->adaptation_counters();
 }
 
+const BalancedDegradationSettings& VideoStreamAdapter::balanced_settings()
+    const {
+  return balanced_settings_;
+}
+
 void VideoStreamAdapter::ClearRestrictions() {
   source_restrictor_->ClearRestrictions();
+  last_adaptation_request_.reset();
 }
 
-bool VideoStreamAdapter::CanDecreaseResolutionTo(int target_pixels,
+VideoStreamAdapter::SetDegradationPreferenceResult
+VideoStreamAdapter::SetDegradationPreference(
+    DegradationPreference degradation_preference) {
+  bool did_clear = false;
+  if (degradation_preference_ != degradation_preference) {
+    if (degradation_preference == DegradationPreference::BALANCED ||
+        degradation_preference_ == DegradationPreference::BALANCED) {
+      ClearRestrictions();
+      did_clear = true;
+    }
+  }
+  degradation_preference_ = degradation_preference;
+  return did_clear ? SetDegradationPreferenceResult::kRestrictionsCleared
+                   : SetDegradationPreferenceResult::kRestrictionsNotCleared;
+}
+
+DegradationPreference VideoStreamAdapter::EffectiveDegradationPreference(
+    VideoInputMode input_mode) const {
+  // Balanced mode for screenshare works via automatic animation detection:
+  // Resolution is capped for fullscreen animated content.
+  // Adapatation is done only via framerate downgrade.
+  // Thus effective degradation preference is MAINTAIN_RESOLUTION.
+  // TODO(hbos): Don't do this. This is not what "balanced" means. If the
+  // application wants to maintain resolution it should set that degradation
+  // preference rather than depend on non-standard behaviors.
+  return (input_mode == VideoInputMode::kScreenshareVideo &&
+          degradation_preference_ == DegradationPreference::BALANCED)
+             ? DegradationPreference::MAINTAIN_RESOLUTION
+             : degradation_preference_;
+}
+
+absl::optional<VideoStreamAdapter::AdaptationTarget>
+VideoStreamAdapter::GetAdaptUpTarget(
+    const absl::optional<EncoderSettings>& encoder_settings,
+    absl::optional<uint32_t> encoder_target_bitrate_bps,
+    VideoInputMode input_mode,
+    int input_pixels,
+    int input_fps,
+    AdaptationObserverInterface::AdaptReason reason) const {
+  // Preconditions for being able to adapt up:
+  if (input_mode == VideoInputMode::kNoVideo)
+    return absl::nullopt;
+  // 1. We shouldn't adapt up if we're currently waiting for a previous upgrade
+  // to have an effect.
+  // TODO(hbos): What about in the case of other degradation preferences?
+  bool last_adaptation_was_up =
+      last_adaptation_request_ &&
+      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
+  if (last_adaptation_was_up &&
+      degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
+      input_pixels <= last_adaptation_request_->input_pixel_count_) {
+    return absl::nullopt;
+  }
+  // 2. We shouldn't adapt up if BalancedSettings doesn't allow it, which is
+  // only applicable if reason is kQuality and preference is BALANCED.
+  if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
+      EffectiveDegradationPreference(input_mode) ==
+          DegradationPreference::BALANCED &&
+      !balanced_settings_.CanAdaptUp(
+          GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels,
+          encoder_target_bitrate_bps.value_or(0))) {
+    return absl::nullopt;
+  }
+
+  // Attempt to find an allowed adaptation target.
+  switch (EffectiveDegradationPreference(input_mode)) {
+    case DegradationPreference::BALANCED: {
+      // Attempt to increase target frame rate.
+      int target_fps = balanced_settings_.MaxFps(
+          GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels);
+      if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) {
+        return AdaptationTarget(AdaptationAction::kIncreaseFrameRate,
+                                target_fps);
+      }
+      // Fall-through to maybe-adapting resolution, unless |balanced_settings_|
+      // forbids it based on bitrate.
+      if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
+          !balanced_settings_.CanAdaptUpResolution(
+              GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels,
+              encoder_target_bitrate_bps.value_or(0))) {
+        return absl::nullopt;
+      }
+      // Scale up resolution.
+      ABSL_FALLTHROUGH_INTENDED;
+    }
+    case DegradationPreference::MAINTAIN_FRAMERATE: {
+      // Don't adapt resolution if CanAdaptUpResolution() forbids it based on
+      // bitrate and limits specified by encoder capabilities.
+      if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
+          !CanAdaptUpResolution(encoder_settings, encoder_target_bitrate_bps,
+                                input_pixels)) {
+        return absl::nullopt;
+      }
+      // Attempt to increase pixel count.
+      int target_pixels = input_pixels;
+      if (source_restrictor_->adaptation_counters().resolution_adaptations ==
+          1) {
+        RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
+        target_pixels = std::numeric_limits<int>::max();
+      }
+      target_pixels = GetHigherResolutionThan(target_pixels);
+      if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels))
+        return absl::nullopt;
+      return AdaptationTarget(AdaptationAction::kIncreaseResolution,
+                              target_pixels);
+    }
+    case DegradationPreference::MAINTAIN_RESOLUTION: {
+      // Scale up framerate.
+      int target_fps = input_fps;
+      if (source_restrictor_->adaptation_counters().fps_adaptations == 1) {
+        RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
+        target_fps = std::numeric_limits<int>::max();
+      }
+      target_fps = GetHigherFrameRateThan(target_fps);
+      if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps))
+        return absl::nullopt;
+      return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
+    }
+    case DegradationPreference::DISABLED:
+      return absl::nullopt;
+  }
+}
+
+absl::optional<VideoStreamAdapter::AdaptationTarget>
+VideoStreamAdapter::GetAdaptDownTarget(
+    const absl::optional<EncoderSettings>& encoder_settings,
+    VideoInputMode input_mode,
+    int input_pixels,
+    int input_fps,
+    int min_pixels_per_frame,
+    VideoStreamEncoderObserver* encoder_stats_observer) const {
+  // Preconditions for being able to adapt down:
+  if (input_mode == VideoInputMode::kNoVideo)
+    return absl::nullopt;
+  // 1. We are not disabled.
+  // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
+  // causes scaling due to bandwidth constraints (QualityScalerResource) to be
+  // ignored, not just CPU signals. This is not a use case we want to support
+  // long-term; remove this enum value.
+  if (degradation_preference_ == DegradationPreference::DISABLED)
+    return absl::nullopt;
+  bool last_adaptation_was_down =
+      last_adaptation_request_ &&
+      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
+  // 2. We shouldn't adapt down if our frame rate is below the minimum or if its
+  // currently unknown.
+  if (EffectiveDegradationPreference(input_mode) ==
+      DegradationPreference::MAINTAIN_RESOLUTION) {
+    // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
+    // - delete it.
+    if (input_fps <= 0 ||
+        (last_adaptation_was_down && input_fps < kMinFramerateFps)) {
+      return absl::nullopt;
+    }
+  }
+  // 3. We shouldn't adapt down if we're currently waiting for a previous
+  // downgrade to have an effect.
+  // TODO(hbos): What about in the case of other degradation preferences?
+  if (last_adaptation_was_down &&
+      degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
+      input_pixels >= last_adaptation_request_->input_pixel_count_) {
+    return absl::nullopt;
+  }
+
+  // Attempt to find an allowed adaptation target.
+  switch (EffectiveDegradationPreference(input_mode)) {
+    case DegradationPreference::BALANCED: {
+      // Try scale down framerate, if lower.
+      int target_fps = balanced_settings_.MinFps(
+          GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels);
+      if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) {
+        return AdaptationTarget(AdaptationAction::kDecreaseFrameRate,
+                                target_fps);
+      }
+      // Scale down resolution.
+      ABSL_FALLTHROUGH_INTENDED;
+    }
+    case DegradationPreference::MAINTAIN_FRAMERATE: {
+      // Scale down resolution.
+      int target_pixels = GetLowerResolutionThan(input_pixels);
+      // TODO(https://crbug.com/webrtc/11393): Move this logic to
+      // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
+      // target is available should not have side-effects.
+      if (target_pixels < min_pixels_per_frame)
+        encoder_stats_observer->OnMinPixelLimitReached();
+      if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels,
+                                                       min_pixels_per_frame)) {
+        return absl::nullopt;
+      }
+      return AdaptationTarget(AdaptationAction::kDecreaseResolution,
+                              target_pixels);
+    }
+    case DegradationPreference::MAINTAIN_RESOLUTION: {
+      int target_fps = GetLowerFrameRateThan(input_fps);
+      if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps))
+        return absl::nullopt;
+      return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
+    }
+    case DegradationPreference::DISABLED:
+      RTC_NOTREACHED();
+      return absl::nullopt;
+  }
+}
+
+void VideoStreamAdapter::ApplyAdaptationTarget(const AdaptationTarget& target,
+                                               VideoInputMode input_mode,
+                                               int input_pixels,
+                                               int input_fps,
                                                int min_pixels_per_frame) {
-  return source_restrictor_->CanDecreaseResolutionTo(target_pixels,
+  // Remember the input pixels and fps of this adaptation. Used to avoid
+  // adapting again before this adaptation has had an effect.
+  last_adaptation_request_.emplace(AdaptationRequest{
+      input_pixels, input_fps,
+      AdaptationRequest::GetModeFromAdaptationAction(target.action)});
+  switch (target.action) {
+    case AdaptationAction::kIncreaseResolution:
+      source_restrictor_->IncreaseResolutionTo(target.value);
+      return;
+    case AdaptationAction::kDecreaseResolution:
+      source_restrictor_->DecreaseResolutionTo(target.value,
                                                min_pixels_per_frame);
-}
+      return;
-
+    case AdaptationAction::kIncreaseFrameRate:
-void VideoStreamAdapter::DecreaseResolutionTo(int target_pixels,
+      source_restrictor_->IncreaseFrameRateTo(target.value);
-                                              int min_pixels_per_frame) {
+      // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
-  source_restrictor_->DecreaseResolutionTo(target_pixels, min_pixels_per_frame);
+      // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
-}
+      // in DecrementFramerate() makes it hard to predict whether this will be
-
+      // the last step. Remove the dependency on GetConstAdaptCounter().
-bool VideoStreamAdapter::CanIncreaseResolutionTo(int target_pixels) {
+      if (EffectiveDegradationPreference(input_mode) ==
-  return source_restrictor_->CanIncreaseResolutionTo(target_pixels);
+              DegradationPreference::BALANCED &&
-}
+          source_restrictor_->adaptation_counters().fps_adaptations == 0 &&
-
+          target.value != std::numeric_limits<int>::max()) {
-void VideoStreamAdapter::IncreaseResolutionTo(int target_pixels) {
+        RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
-  source_restrictor_->IncreaseResolutionTo(target_pixels);
+        source_restrictor_->IncreaseFrameRateTo(
-}
+            std::numeric_limits<int>::max());
-
+      }
-bool VideoStreamAdapter::CanDecreaseFrameRateTo(int max_frame_rate) {
+      return;
-  return source_restrictor_->CanDecreaseFrameRateTo(max_frame_rate);
+    case AdaptationAction::kDecreaseFrameRate:
-}
+      source_restrictor_->DecreaseFrameRateTo(target.value);
-
+      return;
-void VideoStreamAdapter::DecreaseFrameRateTo(int max_frame_rate) {
+  }
-  source_restrictor_->DecreaseFrameRateTo(max_frame_rate);
-}
-
-bool VideoStreamAdapter::CanIncreaseFrameRateTo(int max_frame_rate) {
-  return source_restrictor_->CanIncreaseFrameRateTo(max_frame_rate);
-}
-
-void VideoStreamAdapter::IncreaseFrameRateTo(int max_frame_rate) {
-  source_restrictor_->IncreaseFrameRateTo(max_frame_rate);
 }
 
 }  // namespace webrtc

video/adaptation/video_stream_adapter.h

@@ -13,7 +13,13 @@
 
 #include <memory>
 
+#include "absl/types/optional.h"
+#include "api/rtp_parameters.h"
+#include "api/video/video_stream_encoder_observer.h"
+#include "call/adaptation/encoder_settings.h"
 #include "call/adaptation/video_source_restrictions.h"
+#include "modules/video_coding/utility/quality_scaler.h"
+#include "rtc_base/experiments/balanced_degradation_settings.h"
 #include "video/adaptation/adaptation_counters.h"
 
 namespace webrtc {
@@ -26,39 +32,128 @@ namespace webrtc {
 // 3. Modify the stream's restrictions in one of the valid ways.
 class VideoStreamAdapter {
  public:
-  static const int kMinFramerateFps;
+  enum class SetDegradationPreferenceResult {
+    kRestrictionsNotCleared,
+    kRestrictionsCleared,
+  };
 
-  static int GetLowerFrameRateThan(int fps);
+  enum class VideoInputMode {
-  static int GetHigherFrameRateThan(int fps);
+    kNoVideo,
-  static int GetLowerResolutionThan(int pixel_count);
+    kNormalVideo,
-  static int GetHigherResolutionThan(int pixel_count);
+    kScreenshareVideo,
+  };
+
+  enum class AdaptationAction {
+    kIncreaseResolution,
+    kDecreaseResolution,
+    kIncreaseFrameRate,
+    kDecreaseFrameRate,
+  };
+
+  // Describes an adaptation step: increasing or decreasing resolution or frame
+  // rate to a given value.
+  // TODO(https://crbug.com/webrtc/11393): Make these private implementation
+  // details, and expose something that allows you to inspect the
+  // VideoSourceRestrictions instead. The adaptation steps could be expressed as
+  // a graph, for instance.
+  struct AdaptationTarget {
+    AdaptationTarget(AdaptationAction action, int value);
+    // Which action the VideoSourceRestrictor needs to take.
+    const AdaptationAction action;
+    // Target pixel count or frame rate depending on |action|.
+    const int value;
+
+    // Allow this struct to be instantiated as an optional, even though it's in
+    // a private namespace.
+    friend class absl::optional<AdaptationTarget>;
+  };
 
   VideoStreamAdapter();
   ~VideoStreamAdapter();
 
-  // TODO(hbos): Why isn't this const?
   VideoSourceRestrictions source_restrictions() const;
   const AdaptationCounters& adaptation_counters() const;
+  // TODO(hbos): Can we get rid of any external dependencies on
+  // BalancedDegradationPreference? How the adaptor generates possible next
+  // steps for adaptation should be an implementation detail. Can the relevant
+  // information be inferred from GetAdaptUpTarget()/GetAdaptDownTarget()?
+  const BalancedDegradationSettings& balanced_settings() const;
   void ClearRestrictions();
 
-  // "Can adapt?" and "do adapt!" methods.
+  // TODO(hbos): Setting the degradation preference should not clear
-  // TODO(https://crbug.com/webrtc/11393): Make the adapter responsible for
+  // restrictions! This is not defined in the spec and is unexpected, there is a
-  // deciding what the next step are, i.e. taking on degradation preference
+  // tiny risk that people would discover and rely on this behavior.
-  // logic. Then, these can be expressed either as CanAdaptUp() and DoAdaptUp()
+  SetDegradationPreferenceResult SetDegradationPreference(
-  // or as GetNextRestrictionsUp() and ApplyRestrictions().
+      DegradationPreference degradation_preference);
-  bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame);
+  // TODO(hbos): This is only used in one place externally by
-  void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame);
+  // OveruseFrameDetectorResourceAdaptationModule - can we get rid of that
-  bool CanIncreaseResolutionTo(int target_pixels);
+  // usage? This is exposing an implementation detail.
-  void IncreaseResolutionTo(int target_pixels);
+  DegradationPreference EffectiveDegradationPreference(
-  bool CanDecreaseFrameRateTo(int max_frame_rate);
+      VideoInputMode input_mode) const;
-  void DecreaseFrameRateTo(int max_frame_rate);
+
-  bool CanIncreaseFrameRateTo(int max_frame_rate);
+  // Returns a target that we are guaranteed to be able to adapt to, or null if
-  void IncreaseFrameRateTo(int max_frame_rate);
+  // adaptation is not desired or not possible.
+  absl::optional<AdaptationTarget> GetAdaptUpTarget(
+      const absl::optional<EncoderSettings>& encoder_settings,
+      absl::optional<uint32_t> encoder_target_bitrate_bps,
+      VideoInputMode input_mode,
+      int input_pixels,
+      int input_fps,
+      AdaptationObserverInterface::AdaptReason reason) const;
+  // TODO(https://crbug.com/webrtc/11393): Remove the dependency on
+  // |encoder_stats_observer| - simply checking which adaptation target is
+  // available should not have side-effects.
+  absl::optional<AdaptationTarget> GetAdaptDownTarget(
+      const absl::optional<EncoderSettings>& encoder_settings,
+      VideoInputMode input_mode,
+      int input_pixels,
+      int input_fps,
+      int min_pixels_per_frame,
+      VideoStreamEncoderObserver* encoder_stats_observer) const;
+  // Applies the |target| to |source_restrictor_|.
+  void ApplyAdaptationTarget(const AdaptationTarget& target,
+                             VideoInputMode input_mode,
+                             int input_pixels,
+                             int input_fps,
+                             int min_pixels_per_frame);
 
  private:
   class VideoSourceRestrictor;
 
+  // The input frame rate and resolution at the time of an adaptation in the
+  // direction described by |mode_| (up or down).
+  // TODO(https://crbug.com/webrtc/11393): Can this be renamed? Can this be
+  // merged with AdaptationTarget?
+  struct AdaptationRequest {
+    // The pixel count produced by the source at the time of the adaptation.
+    int input_pixel_count_;
+    // Framerate received from the source at the time of the adaptation.
+    int framerate_fps_;
+    // Indicates if request was to adapt up or down.
+    enum class Mode { kAdaptUp, kAdaptDown } mode_;
+
+    // This is a static method rather than an anonymous namespace function due
+    // to namespace visiblity.
+    static Mode GetModeFromAdaptationAction(AdaptationAction action);
+  };
+
+  // Owner and modifier of the VideoSourceRestriction of this stream adaptor.
   const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
+  // Decides the next adaptation target in DegradationPreference::BALANCED.
+  const BalancedDegradationSettings balanced_settings_;
+  // When deciding the next target up or down, different strategies are used
+  // depending on the DegradationPreference.
+  // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference
+  DegradationPreference degradation_preference_;
+
+  // The input frame rate, resolution and adaptation direction of the last
+  // ApplyAdaptationTarget(). Used to avoid adapting twice if a recent
+  // adaptation has not had an effect on the input frame rate or resolution yet.
+  // TODO(hbos): Can we implement a more general "cooldown" mechanism of
+  // resources intead? If we already have adapted it seems like we should wait
+  // a while before adapting again, so that we are not acting on usage
+  // measurements that are made obsolete/unreliable by an "ongoing" adaptation.
+  absl::optional<AdaptationRequest> last_adaptation_request_;
 };
 
 }  // namespace webrtc

video/overuse_frame_detector_resource_adaptation_module.cc

@@ -164,10 +164,6 @@ class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper {
   int initial_framedrop_;
 };
 
-OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget::
-    AdaptationTarget(AdaptationAction action, int value)
-    : action(action), value(value) {}
-
 OveruseFrameDetectorResourceAdaptationModule::
     OveruseFrameDetectorResourceAdaptationModule(
         Clock* clock,
@@ -181,8 +177,6 @@ OveruseFrameDetectorResourceAdaptationModule::
       experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
       has_input_video_(false),
       degradation_preference_(DegradationPreference::DISABLED),
-      balanced_settings_(),
-      last_adaptation_request_(absl::nullopt),
       stream_adapter_(std::make_unique<VideoStreamAdapter>()),
       encode_usage_resource_(
           std::make_unique<EncodeUsageResource>(std::move(overuse_detector))),
@@ -260,17 +254,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo(
 
 void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference(
     DegradationPreference degradation_preference) {
-  if (degradation_preference_ != degradation_preference) {
+  degradation_preference_ = degradation_preference;
-    // Reset adaptation state, so that we're not tricked into thinking there's
+  if (stream_adapter_->SetDegradationPreference(degradation_preference) ==
-    // an already pending request of the same type.
+      VideoStreamAdapter::SetDegradationPreferenceResult::
-    last_adaptation_request_.reset();
+          kRestrictionsCleared) {
-    if (degradation_preference == DegradationPreference::BALANCED ||
-        degradation_preference_ == DegradationPreference::BALANCED) {
-      stream_adapter_->ClearRestrictions();
     active_counts_.fill(AdaptationCounters());
   }
-  }
-  degradation_preference_ = degradation_preference;
   MaybeUpdateVideoSourceRestrictions();
 }
 
@@ -307,7 +296,6 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates(
 
 void OveruseFrameDetectorResourceAdaptationModule::
     ResetVideoSourceRestrictions() {
-  last_adaptation_request_.reset();
   stream_adapter_->ClearRestrictions();
   active_counts_.fill(AdaptationCounters());
   MaybeUpdateVideoSourceRestrictions();
@@ -397,7 +385,7 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler(
       absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
       if (quality_scaling_experiment_enabled_) {
         experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
-            GetVideoCodecTypeOrGeneric());
+            GetVideoCodecTypeOrGeneric(encoder_settings_));
       }
       UpdateQualityScalerSettings(experimental_thresholds
                                       ? *experimental_thresholds
@@ -411,7 +399,8 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler(
   if (degradation_preference_ == DegradationPreference::BALANCED &&
       quality_scaler_resource_->is_started()) {
     absl::optional<VideoEncoder::QpThresholds> thresholds =
-        balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
+        stream_adapter_->balanced_settings().GetQpThresholds(
+            GetVideoCodecTypeOrGeneric(encoder_settings_),
             LastInputFrameSizeOrDefault());
     if (thresholds) {
       quality_scaler_resource_->SetQpThresholds(*thresholds);
@@ -454,212 +443,51 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured(
   }
 }
 
-absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget>
+absl::optional<VideoStreamAdapter::AdaptationTarget>
 OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget(
     int input_pixels,
     int input_fps,
     AdaptationObserverInterface::AdaptReason reason) const {
-  // Preconditions for being able to adapt up:
+  // We can't adapt up if we're already at the highest setting.
-  if (!has_input_video_)
-    return absl::nullopt;
-  // 1. We can't adapt up if we're already at the highest setting.
   // Note that this only includes counts relevant to the current degradation
   // preference. e.g. we previously adapted resolution, now prefer adpating fps,
   // only count the fps adaptations and not the previous resolution adaptations.
+  //
   // TODO(https://crbug.com/webrtc/11394): Checking the counts for reason should
-  // be replaced with checking the overuse state of all resources.
+  // be replaced with checking the overuse state of all resources. This is
+  // effectively trying to infer if the the Resource specified by |reason| is OK
+  // with adapting up by looking at active counters. If the relevant Resources
+  // simply told us this directly we wouldn't have to depend on stats counters
+  // to abort GetAdaptUpTarget().
   int num_downgrades = ApplyDegradationPreference(active_counts_[reason],
                                                   degradation_preference_)
                            .Total();
   RTC_DCHECK_GE(num_downgrades, 0);
   if (num_downgrades == 0)
     return absl::nullopt;
-  // 2. We shouldn't adapt up if we're currently waiting for a previous upgrade
+  return stream_adapter_->GetAdaptUpTarget(
-  // to have an effect.
+      encoder_settings_, encoder_target_bitrate_bps_, GetVideoInputMode(),
-  // TODO(hbos): What about in the case of other degradation preferences?
+      input_pixels, input_fps, reason);
-  bool last_adaptation_was_up =
-      last_adaptation_request_ &&
-      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
-  if (last_adaptation_was_up &&
-      degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
-      input_pixels <= last_adaptation_request_->input_pixel_count_) {
-    return absl::nullopt;
-  }
-  // 3. We shouldn't adapt up if BalancedSettings doesn't allow it, which is
-  // only applicable if reason is kQuality and preference is BALANCED.
-  if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
-      EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
-      !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(), input_pixels,
-                                     encoder_target_bitrate_bps_.value_or(0))) {
-    return absl::nullopt;
-  }
-
-  // Attempt to find an allowed adaptation target.
-  switch (EffectiveDegradationPreference()) {
-    case DegradationPreference::BALANCED: {
-      // Attempt to increase target frame rate.
-      int target_fps =
-          balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(), input_pixels);
-      if (stream_adapter_->CanIncreaseFrameRateTo(target_fps)) {
-        return AdaptationTarget(AdaptationAction::kIncreaseFrameRate,
-                                target_fps);
-      }
-      // Fall-through to maybe-adapting resolution, unless |balanced_settings_|
-      // forbids it based on bitrate.
-      if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
-          !balanced_settings_.CanAdaptUpResolution(
-              GetVideoCodecTypeOrGeneric(), input_pixels,
-              encoder_target_bitrate_bps_.value_or(0))) {
-        return absl::nullopt;
-      }
-      // Scale up resolution.
-      ABSL_FALLTHROUGH_INTENDED;
-    }
-    case DegradationPreference::MAINTAIN_FRAMERATE: {
-      // Don't adapt resolution if CanAdaptUpResolution() forbids it based on
-      // bitrate and limits specified by encoder capabilities.
-      if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
-          !CanAdaptUpResolution(input_pixels,
-                                encoder_target_bitrate_bps_.value_or(0))) {
-        return absl::nullopt;
-      }
-      // Attempt to increase pixel count.
-      int target_pixels = input_pixels;
-      if (stream_adapter_->adaptation_counters().resolution_adaptations == 1) {
-        RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
-        target_pixels = std::numeric_limits<int>::max();
-      }
-      target_pixels =
-          VideoStreamAdapter::GetHigherResolutionThan(target_pixels);
-      if (!stream_adapter_->CanIncreaseResolutionTo(target_pixels))
-        return absl::nullopt;
-      return AdaptationTarget(AdaptationAction::kIncreaseResolution,
-                              target_pixels);
-    }
-    case DegradationPreference::MAINTAIN_RESOLUTION: {
-      // Scale up framerate.
-      int target_fps = input_fps;
-      if (stream_adapter_->adaptation_counters().fps_adaptations == 1) {
-        RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
-        target_fps = std::numeric_limits<int>::max();
-      }
-      target_fps = VideoStreamAdapter::GetHigherFrameRateThan(target_fps);
-      if (!stream_adapter_->CanIncreaseFrameRateTo(target_fps))
-        return absl::nullopt;
-      return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
-    }
-    case DegradationPreference::DISABLED:
-      return absl::nullopt;
-  }
 }
 
-absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget>
+absl::optional<VideoStreamAdapter::AdaptationTarget>
 OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget(
     int input_pixels,
     int input_fps,
     int min_pixels_per_frame) const {
-  // Preconditions for being able to adapt down:
+  return stream_adapter_->GetAdaptDownTarget(
-  if (!has_input_video_)
+      encoder_settings_, GetVideoInputMode(), input_pixels, input_fps,
-    return absl::nullopt;
+      min_pixels_per_frame, encoder_stats_observer_);
-  // 1. We are not disabled.
-  // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
-  // causes scaling due to bandwidth constraints (QualityScalerResource) to be
-  // ignored, not just CPU signals. This is not a use case we want to support;
-  // remove the enum value.
-  if (degradation_preference_ == DegradationPreference::DISABLED)
-    return absl::nullopt;
-  bool last_adaptation_was_down =
-      last_adaptation_request_ &&
-      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
-  // 2. We shouldn't adapt down if our frame rate is below the minimum or if its
-  // currently unknown.
-  if (EffectiveDegradationPreference() ==
-      DegradationPreference::MAINTAIN_RESOLUTION) {
-    // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
-    // - delete it.
-    if (input_fps <= 0 || (last_adaptation_was_down &&
-                           input_fps < VideoStreamAdapter::kMinFramerateFps)) {
-      return absl::nullopt;
-    }
-  }
-  // 3. We shouldn't adapt down if we're currently waiting for a previous
-  // downgrade to have an effect.
-  // TODO(hbos): What about in the case of other degradation preferences?
-  if (last_adaptation_was_down &&
-      degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
-      input_pixels >= last_adaptation_request_->input_pixel_count_) {
-    return absl::nullopt;
-  }
-
-  // Attempt to find an allowed adaptation target.
-  switch (EffectiveDegradationPreference()) {
-    case DegradationPreference::BALANCED: {
-      // Try scale down framerate, if lower.
-      int target_fps =
-          balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(), input_pixels);
-      if (stream_adapter_->CanDecreaseFrameRateTo(target_fps)) {
-        return AdaptationTarget(AdaptationAction::kDecreaseFrameRate,
-                                target_fps);
-      }
-      // Scale down resolution.
-      ABSL_FALLTHROUGH_INTENDED;
-    }
-    case DegradationPreference::MAINTAIN_FRAMERATE: {
-      // Scale down resolution.
-      int target_pixels =
-          VideoStreamAdapter::GetLowerResolutionThan(input_pixels);
-      // TODO(https://crbug.com/webrtc/11222): Move this logic to
-      // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
-      // target is available should not have side-effects.
-      if (target_pixels < min_pixels_per_frame)
-        encoder_stats_observer_->OnMinPixelLimitReached();
-      if (!stream_adapter_->CanDecreaseResolutionTo(target_pixels,
-                                                    min_pixels_per_frame)) {
-        return absl::nullopt;
-      }
-      return AdaptationTarget(AdaptationAction::kDecreaseResolution,
-                              target_pixels);
-    }
-    case DegradationPreference::MAINTAIN_RESOLUTION: {
-      int target_fps = VideoStreamAdapter::GetLowerFrameRateThan(input_fps);
-      if (!stream_adapter_->CanDecreaseFrameRateTo(target_fps))
-        return absl::nullopt;
-      return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
-    }
-    case DegradationPreference::DISABLED:
-      RTC_NOTREACHED();
-      return absl::nullopt;
-  }
 }
 
 void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget(
-    const AdaptationTarget& target,
+    const VideoStreamAdapter::AdaptationTarget& target,
-    int min_pixels_per_frame,
+    int input_pixels,
-    AdaptationObserverInterface::AdaptReason reason) {
+    int input_fps,
-  switch (target.action) {
+    int min_pixels_per_frame) {
-    case AdaptationAction::kIncreaseResolution:
+  stream_adapter_->ApplyAdaptationTarget(target, GetVideoInputMode(),
-      stream_adapter_->IncreaseResolutionTo(target.value);
+                                         input_pixels, input_fps,
-      return;
+                                         min_pixels_per_frame);
-    case AdaptationAction::kDecreaseResolution:
-      stream_adapter_->DecreaseResolutionTo(target.value, min_pixels_per_frame);
-      return;
-    case AdaptationAction::kIncreaseFrameRate:
-      stream_adapter_->IncreaseFrameRateTo(target.value);
-      // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
-      // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
-      // in DecrementFramerate() makes it hard to predict whether this will be
-      // the last step. Remove the dependency on GetConstAdaptCounter().
-      if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
-          stream_adapter_->adaptation_counters().fps_adaptations == 0 &&
-          target.value != std::numeric_limits<int>::max()) {
-        RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
-        stream_adapter_->IncreaseFrameRateTo(std::numeric_limits<int>::max());
-      }
-      return;
-    case AdaptationAction::kDecreaseFrameRate:
-      stream_adapter_->DecreaseFrameRateTo(target.value);
-      return;
-  }
 }
 
 void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse(
@@ -668,14 +496,13 @@ void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse(
   int input_fps = encoder_stats_observer_->GetInputFrameRate();
   int min_pixels_per_frame = MinPixelsPerFrame();
   // Should we adapt, if so to what target?
-  absl::optional<AdaptationTarget> target =
+  absl::optional<VideoStreamAdapter::AdaptationTarget> target =
       GetAdaptUpTarget(input_pixels, input_fps, reason);
   if (!target.has_value())
     return;
   // Apply target.
-  ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason);
+  ApplyAdaptationTarget(target.value(), input_pixels, input_fps,
-  last_adaptation_request_.emplace(AdaptationRequest{
+                        min_pixels_per_frame);
-      input_pixels, input_fps, AdaptationRequest::Mode::kAdaptUp});
   // Update VideoSourceRestrictions based on adaptation. This also informs the
   // |adaptation_listener_|.
   MaybeUpdateVideoSourceRestrictions();
@@ -693,14 +520,13 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse(
   int input_fps = encoder_stats_observer_->GetInputFrameRate();
   int min_pixels_per_frame = MinPixelsPerFrame();
   // Should we adapt, if so to what target?
-  absl::optional<AdaptationTarget> target =
+  absl::optional<VideoStreamAdapter::AdaptationTarget> target =
       GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame);
   if (!target.has_value())
     return ResourceListenerResponse::kNothing;
   // Apply target.
-  ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason);
+  ApplyAdaptationTarget(target.value(), input_pixels, input_fps,
-  last_adaptation_request_.emplace(AdaptationRequest{
+                        min_pixels_per_frame);
-      input_pixels, input_fps, AdaptationRequest::Mode::kAdaptDown});
   // Update VideoSourceRestrictions based on adaptation. This also informs the
   // |adaptation_listener_|.
   MaybeUpdateVideoSourceRestrictions();
@@ -709,9 +535,12 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse(
   RTC_LOG(INFO) << ActiveCountsToString();
   // In BALANCED, if requested FPS is higher or close to input FPS to the target
   // we tell the QualityScaler to increase its frequency.
-  if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
+  if (stream_adapter_->EffectiveDegradationPreference(GetVideoInputMode()) ==
-      target->action == AdaptationAction::kDecreaseFrameRate) {
+          DegradationPreference::BALANCED &&
-    absl::optional<int> min_diff = balanced_settings_.MinFpsDiff(input_pixels);
+      target->action ==
+          VideoStreamAdapter::AdaptationAction::kDecreaseFrameRate) {
+    absl::optional<int> min_diff =
+        stream_adapter_->balanced_settings().MinFpsDiff(input_pixels);
     if (min_diff && input_fps > 0) {
       int fps_diff = input_fps - target->value;
       if (fps_diff < min_diff.value()) {
@@ -744,14 +573,6 @@ OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const {
   return options;
 }
 
-VideoCodecType
-OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric()
-    const {
-  return encoder_settings_.has_value()
-             ? encoder_settings_->encoder_config().codec_type
-             : kVideoCodecGeneric;
-}
-
 int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault()
     const {
   // The dependency on this hardcoded resolution is inherited from old code,
@@ -924,37 +745,17 @@ OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
   return counts;
 }
 
-DegradationPreference
+VideoStreamAdapter::VideoInputMode
-OveruseFrameDetectorResourceAdaptationModule::EffectiveDegradationPreference()
+OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const {
-    const {
+  if (!has_input_video_)
-  // Balanced mode for screenshare works via automatic animation detection:
+    return VideoStreamAdapter::VideoInputMode::kNoVideo;
-  // Resolution is capped for fullscreen animated content.
-  // Adapatation is done only via framerate downgrade.
-  // Thus effective degradation preference is MAINTAIN_RESOLUTION.
   return (encoder_settings_.has_value() &&
           encoder_settings_->encoder_config().content_type ==
-              VideoEncoderConfig::ContentType::kScreen &&
+              VideoEncoderConfig::ContentType::kScreen)
-          degradation_preference_ == DegradationPreference::BALANCED)
+             ? VideoStreamAdapter::VideoInputMode::kScreenshareVideo
-             ? DegradationPreference::MAINTAIN_RESOLUTION
+             : VideoStreamAdapter::VideoInputMode::kNormalVideo;
-             : degradation_preference_;
 }
 
-bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
-    int pixels,
-    uint32_t bitrate_bps) const {
-  absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
-      encoder_settings_.has_value()
-          ? GetEncoderBitrateLimits(
-                encoder_settings_->encoder_info(),
-                VideoStreamAdapter::GetHigherResolutionThan(pixels))
-          : absl::nullopt;
-  if (!bitrate_limits.has_value() || bitrate_bps == 0) {
-    return true;  // No limit configured or bitrate provided.
-  }
-  RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels);
-  return bitrate_bps >=
-         static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
-}
 void OveruseFrameDetectorResourceAdaptationModule::
     MaybePerformQualityRampupExperiment() {
   if (!quality_scaler_resource_->is_started())

video/overuse_frame_detector_resource_adaptation_module.h

@@ -128,55 +128,25 @@ class OveruseFrameDetectorResourceAdaptationModule
       AdaptationCounters* other_active);
 
  private:
-  class VideoSourceRestrictor;
   class InitialFrameDropper;
 
   enum class State { kStopped, kStarted };
 
-  struct AdaptationRequest {
-    // The pixel count produced by the source at the time of the adaptation.
-    int input_pixel_count_;
-    // Framerate received from the source at the time of the adaptation.
-    int framerate_fps_;
-    // Indicates if request was to adapt up or down.
-    enum class Mode { kAdaptUp, kAdaptDown } mode_;
-  };
-
-  enum class AdaptationAction {
-    kIncreaseResolution,
-    kDecreaseResolution,
-    kIncreaseFrameRate,
-    kDecreaseFrameRate,
-  };
-
-  // Describes an adaptation step: increasing or decreasing resolution or frame
-  // rate to a given value.
-  struct AdaptationTarget {
-    AdaptationTarget(AdaptationAction action, int value);
-    // Which action the VideoSourceRestrictor needs to take.
-    const AdaptationAction action;
-    // Target pixel count or frame rate depending on |action|.
-    const int value;
-
-    // Allow this struct to be instantiated as an optional, even though it's in
-    // a private namespace.
-    friend class absl::optional<AdaptationTarget>;
-  };
-
   // Returns a target that we are guaranteed to be able to adapt to, or null if
   // adaptation is not desired or not possible.
-  absl::optional<AdaptationTarget> GetAdaptUpTarget(
+  absl::optional<VideoStreamAdapter::AdaptationTarget> GetAdaptUpTarget(
       int input_pixels,
       int input_fps,
       AdaptationObserverInterface::AdaptReason reason) const;
-  absl::optional<AdaptationTarget> GetAdaptDownTarget(
+  absl::optional<VideoStreamAdapter::AdaptationTarget> GetAdaptDownTarget(
       int input_pixels,
       int input_fps,
       int min_pixels_per_frame) const;
   // Applies the |target| to |source_restrictor_|.
-  void ApplyAdaptationTarget(const AdaptationTarget& target,
+  void ApplyAdaptationTarget(const VideoStreamAdapter::AdaptationTarget& target,
-                             int min_pixels_per_frame,
+                             int input_pixels,
-                             AdaptationObserverInterface::AdaptReason reason);
+                             int input_fps,
+                             int min_pixels_per_frame);
 
   // Performs the adaptation by getting the next target, applying it and
   // informing listeners of the new VideoSourceRestriction and adapt counters.
@@ -185,11 +155,11 @@ class OveruseFrameDetectorResourceAdaptationModule
       AdaptationObserverInterface::AdaptReason reason);
 
   CpuOveruseOptions GetCpuOveruseOptions() const;
-  VideoCodecType GetVideoCodecTypeOrGeneric() const;
   int LastInputFrameSizeOrDefault() const;
   int MinPixelsPerFrame() const;
   VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
       AdaptationObserverInterface::AdaptReason reason);
+  VideoStreamAdapter::VideoInputMode GetVideoInputMode() const;
 
   // Makes |video_source_restrictions_| up-to-date and informs the
   // |adaptation_listener_| if restrictions are changed, allowing the listener
@@ -204,8 +174,6 @@ class OveruseFrameDetectorResourceAdaptationModule
       absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
 
   void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason);
-  DegradationPreference EffectiveDegradationPreference() const;
-  bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const;
 
   // Checks to see if we should execute the quality rampup experiment. The
   // experiment resets all video restrictions at the start of the call in the
@@ -224,11 +192,12 @@ class OveruseFrameDetectorResourceAdaptationModule
   // The restrictions that |adaptation_listener_| is informed of.
   VideoSourceRestrictions video_source_restrictions_;
   bool has_input_video_;
+  // TODO(https://crbug.com/webrtc/11393): DegradationPreference has mostly
+  // moved to VideoStreamAdapter. Move it entirely and delete it from this
+  // class. If the responsibility of generating next steps for adaptations is
+  // owned by the adapter, this class has no buisness relying on implementation
+  // details of the adapter.
   DegradationPreference degradation_preference_;
-  const BalancedDegradationSettings balanced_settings_;
-  // Stores a snapshot of the last adaptation request triggered by an AdaptUp
-  // or AdaptDown signal.
-  absl::optional<AdaptationRequest> last_adaptation_request_;
   // Keeps track of source restrictions that this adaptation module outputs.
   const std::unique_ptr<VideoStreamAdapter> stream_adapter_;
   const std::unique_ptr<EncodeUsageResource> encode_usage_resource_;

video/video_stream_encoder.cc

@@ -171,43 +171,6 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
 }
 }  //  namespace
 
-absl::optional<VideoEncoder::ResolutionBitrateLimits> GetEncoderBitrateLimits(
-    const VideoEncoder::EncoderInfo& encoder_info,
-    int frame_size_pixels) {
-  std::vector<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
-      encoder_info.resolution_bitrate_limits;
-
-  // Sort the list of bitrate limits by resolution.
-  sort(bitrate_limits.begin(), bitrate_limits.end(),
-       [](const VideoEncoder::ResolutionBitrateLimits& lhs,
-          const VideoEncoder::ResolutionBitrateLimits& rhs) {
-         return lhs.frame_size_pixels < rhs.frame_size_pixels;
-       });
-
-  for (size_t i = 0; i < bitrate_limits.size(); ++i) {
-    RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0);
-    RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0);
-    RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
-                  bitrate_limits[i].min_bitrate_bps);
-    if (i > 0) {
-      // The bitrate limits aren't expected to decrease with resolution.
-      RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps,
-                    bitrate_limits[i - 1].min_bitrate_bps);
-      RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps,
-                    bitrate_limits[i - 1].min_start_bitrate_bps);
-      RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
-                    bitrate_limits[i - 1].max_bitrate_bps);
-    }
-
-    if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) {
-      return absl::optional<VideoEncoder::ResolutionBitrateLimits>(
-          bitrate_limits[i]);
-    }
-  }
-
-  return absl::nullopt;
-}
-
 const int VideoStreamEncoder::kDefaultLastFrameInfoWidth = 176;
 const int VideoStreamEncoder::kDefaultLastFrameInfoHeight = 144;
 
@@ -503,8 +466,8 @@ void VideoStreamEncoder::ReconfigureEncoder() {
     encoder_reset_required = true;
   }
 
-  encoder_bitrate_limits_ = GetEncoderBitrateLimits(
+  encoder_bitrate_limits_ =
-      encoder_->GetEncoderInfo(),
+      encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
           last_frame_info_->width * last_frame_info_->height);
 
   if (streams.size() == 1 && encoder_bitrate_limits_) {
@@ -1630,7 +1593,8 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
   }
 
   absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
-      GetEncoderBitrateLimits(encoder_->GetEncoderInfo(), pixel_count);
+      encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
+          pixel_count);
 
   if (encoder_bitrate_limits.has_value()) {
     // Use bitrate limits provided by encoder.

video/video_stream_encoder.h

@@ -45,10 +45,6 @@
 
 namespace webrtc {
 
-absl::optional<VideoEncoder::ResolutionBitrateLimits> GetEncoderBitrateLimits(
-    const VideoEncoder::EncoderInfo& encoder_info,
-    int frame_size_pixels);
-
 // VideoStreamEncoder represent a video encoder that accepts raw video frames as
 // input and produces an encoded bit stream.
 // Usage: