From b0f2e0ced4cd7ff23a54d0c494a9264cd62e0b1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Fri, 6 Mar 2020 13:32:03 +0100 Subject: [PATCH] [Overuse] Make VideoStreamAdapter responsible for executing adaptation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL moves GetAdaptUpTarget(), GetAdaptDownTarget() and ApplyAdaptationTarget() - and related code - to the VideoStreamAdapter. This includes pieces related to calculating how to adapt, including: - DegradationPreference - BalancedDegradationPreference - AdaptationRequest and last_adaptation_request_ - CanAdaptUpResolution() The VideoStreamAdapter's interface has changed: VideoSourceRestrictor methods are now hidden in favor of methods exposing AdaptationTarget. This CL also does some misc moves: - GetEncoderBitrateLimits is moved and renamed to VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution. - EncoderSettings moved to a separate file. // For api/video_codecs/video_encoder.[cc/h] changes, which is the // moving of a function. TBR=sprang@webrtc.org Bug: webrtc:11393 Change-Id: Ie6bd8ef644ce927d7eca6ab90a0a7bcace682f3c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169842 Reviewed-by: Henrik Boström Reviewed-by: Evan Shrubsole Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Henrik Boström Cr-Commit-Position: refs/heads/master@{#30708} --- api/video_codecs/video_encoder.cc | 37 ++ api/video_codecs/video_encoder.h | 5 + call/adaptation/BUILD.gn | 2 + call/adaptation/encoder_settings.cc | 42 ++ call/adaptation/encoder_settings.h | 46 ++ .../resource_adaptation_module_interface.cc | 21 - .../resource_adaptation_module_interface.h | 23 +- video/adaptation/BUILD.gn | 5 + video/adaptation/video_stream_adapter.cc | 408 ++++++++++++++---- video/adaptation/video_stream_adapter.h | 133 +++++- ...ame_detector_resource_adaptation_module.cc | 299 +++---------- ...rame_detector_resource_adaptation_module.h | 55 +-- video/video_stream_encoder.cc | 46 +- video/video_stream_encoder.h | 4 - 14 files changed, 647 insertions(+), 479 deletions(-) create mode 100644 call/adaptation/encoder_settings.cc create mode 100644 call/adaptation/encoder_settings.h diff --git a/api/video_codecs/video_encoder.cc b/api/video_codecs/video_encoder.cc index cf25987435..4427d6c1f1 100644 --- a/api/video_codecs/video_encoder.cc +++ b/api/video_codecs/video_encoder.cc @@ -11,6 +11,7 @@ #include "api/video_codecs/video_encoder.h" #include +#include #include "rtc_base/checks.h" #include "rtc_base/strings/string_builder.h" @@ -208,6 +209,42 @@ bool VideoEncoder::EncoderInfo::operator==(const EncoderInfo& rhs) const { return true; } +absl::optional +VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution( + int frame_size_pixels) const { + std::vector bitrate_limits = + resolution_bitrate_limits; + + // Sort the list of bitrate limits by resolution. + sort(bitrate_limits.begin(), bitrate_limits.end(), + [](const ResolutionBitrateLimits& lhs, + const ResolutionBitrateLimits& rhs) { + return lhs.frame_size_pixels < rhs.frame_size_pixels; + }); + + for (size_t i = 0; i < bitrate_limits.size(); ++i) { + RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0); + RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0); + RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, + bitrate_limits[i].min_bitrate_bps); + if (i > 0) { + // The bitrate limits aren't expected to decrease with resolution. + RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, + bitrate_limits[i - 1].min_bitrate_bps); + RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, + bitrate_limits[i - 1].min_start_bitrate_bps); + RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, + bitrate_limits[i - 1].max_bitrate_bps); + } + + if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) { + return absl::optional(bitrate_limits[i]); + } + } + + return absl::nullopt; +} + VideoEncoder::RateControlParameters::RateControlParameters() : bitrate(VideoBitrateAllocation()), framerate_fps(0.0), diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h index 34502c8ab0..064dc8ffb5 100644 --- a/api/video_codecs/video_encoder.h +++ b/api/video_codecs/video_encoder.h @@ -236,6 +236,11 @@ class RTC_EXPORT VideoEncoder { // Recommended bitrate limits for different resolutions. std::vector resolution_bitrate_limits; + // Obtains the limits from |resolution_bitrate_limits| that best matches the + // |frame_size_pixels|. + absl::optional + GetEncoderBitrateLimitsForResolution(int frame_size_pixels) const; + // If true, this encoder has internal support for generating simulcast // streams. Otherwise, an adapter class will be needed. // Even if true, the config provided to InitEncode() might not be supported, diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index 5eb5af5ef3..0288e247b4 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -10,6 +10,8 @@ import("../../webrtc.gni") rtc_library("resource_adaptation") { sources = [ + "encoder_settings.cc", + "encoder_settings.h", "resource.cc", "resource.h", "resource_adaptation_module_interface.cc", diff --git a/call/adaptation/encoder_settings.cc b/call/adaptation/encoder_settings.cc new file mode 100644 index 0000000000..84b4b17ccd --- /dev/null +++ b/call/adaptation/encoder_settings.cc @@ -0,0 +1,42 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/encoder_settings.h" + +#include + +namespace webrtc { + +EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info, + VideoEncoderConfig encoder_config, + VideoCodec video_codec) + : encoder_info_(std::move(encoder_info)), + encoder_config_(std::move(encoder_config)), + video_codec_(std::move(video_codec)) {} + +const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const { + return encoder_info_; +} + +const VideoEncoderConfig& EncoderSettings::encoder_config() const { + return encoder_config_; +} + +const VideoCodec& EncoderSettings::video_codec() const { + return video_codec_; +} + +VideoCodecType GetVideoCodecTypeOrGeneric( + const absl::optional& settings) { + return settings.has_value() ? settings->encoder_config().codec_type + : kVideoCodecGeneric; +} + +} // namespace webrtc diff --git a/call/adaptation/encoder_settings.h b/call/adaptation/encoder_settings.h new file mode 100644 index 0000000000..9cfd056409 --- /dev/null +++ b/call/adaptation/encoder_settings.h @@ -0,0 +1,46 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_ENCODER_SETTINGS_H_ +#define CALL_ADAPTATION_ENCODER_SETTINGS_H_ + +#include "absl/types/optional.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_config.h" + +namespace webrtc { + +// Information about an encoder available when reconfiguring the encoder. +class EncoderSettings { + public: + EncoderSettings(VideoEncoder::EncoderInfo encoder_info, + VideoEncoderConfig encoder_config, + VideoCodec video_codec); + + // Encoder capabilities, implementation info, etc. + const VideoEncoder::EncoderInfo& encoder_info() const; + // Configuration parameters, ultimately coming from the API and negotiation. + const VideoEncoderConfig& encoder_config() const; + // Lower level config, heavily based on the VideoEncoderConfig. + const VideoCodec& video_codec() const; + + private: + VideoEncoder::EncoderInfo encoder_info_; + VideoEncoderConfig encoder_config_; + VideoCodec video_codec_; +}; + +VideoCodecType GetVideoCodecTypeOrGeneric( + const absl::optional& settings); + +} // namespace webrtc + +#endif // CALL_ADAPTATION_ENCODER_SETTINGS_H_ diff --git a/call/adaptation/resource_adaptation_module_interface.cc b/call/adaptation/resource_adaptation_module_interface.cc index 63cfb7279f..e89d1eff2c 100644 --- a/call/adaptation/resource_adaptation_module_interface.cc +++ b/call/adaptation/resource_adaptation_module_interface.cc @@ -10,29 +10,8 @@ #include "call/adaptation/resource_adaptation_module_interface.h" -#include - namespace webrtc { -EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info, - VideoEncoderConfig encoder_config, - VideoCodec video_codec) - : encoder_info_(std::move(encoder_info)), - encoder_config_(std::move(encoder_config)), - video_codec_(std::move(video_codec)) {} - -const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const { - return encoder_info_; -} - -const VideoEncoderConfig& EncoderSettings::encoder_config() const { - return encoder_config_; -} - -const VideoCodec& EncoderSettings::video_codec() const { - return video_codec_; -} - ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {} ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {} diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h index e961897458..1248e17e02 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -14,33 +14,12 @@ #include "absl/types/optional.h" #include "api/rtp_parameters.h" #include "api/video/video_frame.h" -#include "api/video_codecs/video_encoder.h" -#include "api/video_codecs/video_encoder_config.h" +#include "call/adaptation/encoder_settings.h" #include "call/adaptation/resource.h" #include "call/adaptation/video_source_restrictions.h" namespace webrtc { -// Information about an encoder available when reconfiguring the encoder. -class EncoderSettings { - public: - EncoderSettings(VideoEncoder::EncoderInfo encoder_info, - VideoEncoderConfig encoder_config, - VideoCodec video_codec); - - // Encoder capabilities, implementation info, etc. - const VideoEncoder::EncoderInfo& encoder_info() const; - // Configuration parameters, ultimately coming from the API and negotiation. - const VideoEncoderConfig& encoder_config() const; - // Lower level config, heavily based on the VideoEncoderConfig. - const VideoCodec& video_codec() const; - - private: - VideoEncoder::EncoderInfo encoder_info_; - VideoEncoderConfig encoder_config_; - VideoCodec video_codec_; -}; - // The listener is responsible for carrying out the reconfiguration of the video // source such that the VideoSourceRestrictions are fulfilled. class ResourceAdaptationModuleListener { diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index e9e5955709..95705cb649 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -17,12 +17,17 @@ rtc_library("video_adaptation") { ] deps = [ + "../../api:rtp_parameters", + "../../api/video:video_stream_encoder", + "../../api/video_codecs:video_codecs_api", "../../call/adaptation:resource_adaptation", + "../../modules/video_coding:video_coding_utility", "../../rtc_base:checks", "../../rtc_base:logging", "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_event", "../../rtc_base:rtc_numerics", + "../../rtc_base/experiments:balanced_degradation_settings", "//third_party/abseil-cpp/absl/types:optional", ] } diff --git a/video/adaptation/video_stream_adapter.cc b/video/adaptation/video_stream_adapter.cc index 87f7e9019f..8b589c383b 100644 --- a/video/adaptation/video_stream_adapter.cc +++ b/video/adaptation/video_stream_adapter.cc @@ -14,50 +14,87 @@ #include #include "absl/types/optional.h" +#include "api/video_codecs/video_encoder.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" namespace webrtc { +namespace { + +const int kMinFramerateFps = 2; + +// Generate suggested higher and lower frame rates and resolutions, to be +// applied to the VideoSourceRestrictor. These are used in "maintain-resolution" +// and "maintain-framerate". The "balanced" degradation preference also makes +// use of BalancedDegradationPreference when generating suggestions. The +// VideoSourceRestrictor decidedes whether or not a proposed adaptation is +// valid. + +// For frame rate, the steps we take are 2/3 (down) and 3/2 (up). +int GetLowerFrameRateThan(int fps) { + RTC_DCHECK(fps != std::numeric_limits::max()); + return (fps * 2) / 3; +} +// TODO(hbos): Use absl::optional<> instead? +int GetHigherFrameRateThan(int fps) { + return fps != std::numeric_limits::max() + ? (fps * 3) / 2 + : std::numeric_limits::max(); +} + +// For resolution, the steps we take are 3/5 (down) and 5/3 (up). +// Notice the asymmetry of which restriction property is set depending on if +// we are adapting up or down: +// - VideoSourceRestrictor::DecreaseResolution() sets the max_pixels_per_frame() +// to the desired target and target_pixels_per_frame() to null. +// - VideoSourceRestrictor::IncreaseResolutionTo() sets the +// target_pixels_per_frame() to the desired target, and max_pixels_per_frame() +// is set according to VideoSourceRestrictor::GetIncreasedMaxPixelsWanted(). +int GetLowerResolutionThan(int pixel_count) { + RTC_DCHECK(pixel_count != std::numeric_limits::max()); + return (pixel_count * 3) / 5; +} +// TODO(hbos): Use absl::optional<> instead? +int GetHigherResolutionThan(int pixel_count) { + return pixel_count != std::numeric_limits::max() + ? (pixel_count * 5) / 3 + : std::numeric_limits::max(); +} + +// One of the conditions used in VideoStreamAdapter::GetAdaptUpTarget(). +// TODO(hbos): Whether or not we can adapt up due to encoder settings and +// bitrate should be expressed as a bandwidth-related Resource. +bool CanAdaptUpResolution( + const absl::optional& encoder_settings, + absl::optional encoder_target_bitrate_bps, + int input_pixels) { + uint32_t bitrate_bps = encoder_target_bitrate_bps.value_or(0); + absl::optional bitrate_limits = + encoder_settings.has_value() + ? encoder_settings->encoder_info() + .GetEncoderBitrateLimitsForResolution( + GetHigherResolutionThan(input_pixels)) + : absl::nullopt; + if (!bitrate_limits.has_value() || bitrate_bps == 0) { + return true; // No limit configured or bitrate provided. + } + RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, input_pixels); + return bitrate_bps >= + static_cast(bitrate_limits->min_start_bitrate_bps); +} + +} // namespace + +VideoStreamAdapter::AdaptationTarget::AdaptationTarget(AdaptationAction action, + int value) + : action(action), value(value) {} + // VideoSourceRestrictor is responsible for keeping track of current -// VideoSourceRestrictions. It suggests higher and lower frame rates and -// resolutions (used by "maintain-resolution" and "maintain-framerate"), but is -// ultimately not reponsible for determining when or how we should adapt up or -// down (e.g. "balanced" mode also uses BalancedDegradationPreference). +// VideoSourceRestrictions. class VideoStreamAdapter::VideoSourceRestrictor { public: - // For frame rate, the steps we take are 2/3 (down) and 3/2 (up). - static int GetLowerFrameRateThan(int fps) { - RTC_DCHECK(fps != std::numeric_limits::max()); - return (fps * 2) / 3; - } - // TODO(hbos): Use absl::optional<> instead? - static int GetHigherFrameRateThan(int fps) { - return fps != std::numeric_limits::max() - ? (fps * 3) / 2 - : std::numeric_limits::max(); - } - - // For resolution, the steps we take are 3/5 (down) and 5/3 (up). - // Notice the asymmetry of which restriction property is set depending on if - // we are adapting up or down: - // - DecreaseResolution() sets the max_pixels_per_frame() to the desired - // target and target_pixels_per_frame() to null. - // - IncreaseResolutionTo() sets the target_pixels_per_frame() to the desired - // target, and max_pixels_per_frame() is set according to - // GetIncreasedMaxPixelsWanted(). - static int GetLowerResolutionThan(int pixel_count) { - RTC_DCHECK(pixel_count != std::numeric_limits::max()); - return (pixel_count * 3) / 5; - } - // TODO(hbos): Use absl::optional<> instead? - static int GetHigherResolutionThan(int pixel_count) { - return pixel_count != std::numeric_limits::max() - ? (pixel_count * 5) / 3 - : std::numeric_limits::max(); - } - VideoSourceRestrictor() {} VideoSourceRestrictions source_restrictions() const { @@ -168,30 +205,27 @@ class VideoStreamAdapter::VideoSourceRestrictor { RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor); }; -const int VideoStreamAdapter::kMinFramerateFps = 2; - // static -int VideoStreamAdapter::GetLowerFrameRateThan(int fps) { - return VideoSourceRestrictor::GetLowerFrameRateThan(fps); -} - -// static -int VideoStreamAdapter::GetHigherFrameRateThan(int fps) { - return VideoSourceRestrictor::GetHigherFrameRateThan(fps); -} - -// static -int VideoStreamAdapter::GetLowerResolutionThan(int pixel_count) { - return VideoSourceRestrictor::GetLowerResolutionThan(pixel_count); -} - -// static -int VideoStreamAdapter::GetHigherResolutionThan(int pixel_count) { - return VideoSourceRestrictor::GetHigherResolutionThan(pixel_count); +VideoStreamAdapter::AdaptationRequest::Mode +VideoStreamAdapter::AdaptationRequest::GetModeFromAdaptationAction( + VideoStreamAdapter::AdaptationAction action) { + switch (action) { + case AdaptationAction::kIncreaseResolution: + return AdaptationRequest::Mode::kAdaptUp; + case AdaptationAction::kDecreaseResolution: + return AdaptationRequest::Mode::kAdaptDown; + case AdaptationAction::kIncreaseFrameRate: + return AdaptationRequest::Mode::kAdaptUp; + case AdaptationAction::kDecreaseFrameRate: + return AdaptationRequest::Mode::kAdaptDown; + } } VideoStreamAdapter::VideoStreamAdapter() - : source_restrictor_(std::make_unique()) {} + : source_restrictor_(std::make_unique()), + balanced_settings_(), + degradation_preference_(DegradationPreference::DISABLED), + last_adaptation_request_(absl::nullopt) {} VideoStreamAdapter::~VideoStreamAdapter() {} @@ -203,43 +237,257 @@ const AdaptationCounters& VideoStreamAdapter::adaptation_counters() const { return source_restrictor_->adaptation_counters(); } +const BalancedDegradationSettings& VideoStreamAdapter::balanced_settings() + const { + return balanced_settings_; +} + void VideoStreamAdapter::ClearRestrictions() { source_restrictor_->ClearRestrictions(); + last_adaptation_request_.reset(); } -bool VideoStreamAdapter::CanDecreaseResolutionTo(int target_pixels, - int min_pixels_per_frame) { - return source_restrictor_->CanDecreaseResolutionTo(target_pixels, - min_pixels_per_frame); +VideoStreamAdapter::SetDegradationPreferenceResult +VideoStreamAdapter::SetDegradationPreference( + DegradationPreference degradation_preference) { + bool did_clear = false; + if (degradation_preference_ != degradation_preference) { + if (degradation_preference == DegradationPreference::BALANCED || + degradation_preference_ == DegradationPreference::BALANCED) { + ClearRestrictions(); + did_clear = true; + } + } + degradation_preference_ = degradation_preference; + return did_clear ? SetDegradationPreferenceResult::kRestrictionsCleared + : SetDegradationPreferenceResult::kRestrictionsNotCleared; } -void VideoStreamAdapter::DecreaseResolutionTo(int target_pixels, - int min_pixels_per_frame) { - source_restrictor_->DecreaseResolutionTo(target_pixels, min_pixels_per_frame); +DegradationPreference VideoStreamAdapter::EffectiveDegradationPreference( + VideoInputMode input_mode) const { + // Balanced mode for screenshare works via automatic animation detection: + // Resolution is capped for fullscreen animated content. + // Adapatation is done only via framerate downgrade. + // Thus effective degradation preference is MAINTAIN_RESOLUTION. + // TODO(hbos): Don't do this. This is not what "balanced" means. If the + // application wants to maintain resolution it should set that degradation + // preference rather than depend on non-standard behaviors. + return (input_mode == VideoInputMode::kScreenshareVideo && + degradation_preference_ == DegradationPreference::BALANCED) + ? DegradationPreference::MAINTAIN_RESOLUTION + : degradation_preference_; } -bool VideoStreamAdapter::CanIncreaseResolutionTo(int target_pixels) { - return source_restrictor_->CanIncreaseResolutionTo(target_pixels); +absl::optional +VideoStreamAdapter::GetAdaptUpTarget( + const absl::optional& encoder_settings, + absl::optional encoder_target_bitrate_bps, + VideoInputMode input_mode, + int input_pixels, + int input_fps, + AdaptationObserverInterface::AdaptReason reason) const { + // Preconditions for being able to adapt up: + if (input_mode == VideoInputMode::kNoVideo) + return absl::nullopt; + // 1. We shouldn't adapt up if we're currently waiting for a previous upgrade + // to have an effect. + // TODO(hbos): What about in the case of other degradation preferences? + bool last_adaptation_was_up = + last_adaptation_request_ && + last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; + if (last_adaptation_was_up && + degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && + input_pixels <= last_adaptation_request_->input_pixel_count_) { + return absl::nullopt; + } + // 2. We shouldn't adapt up if BalancedSettings doesn't allow it, which is + // only applicable if reason is kQuality and preference is BALANCED. + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && + EffectiveDegradationPreference(input_mode) == + DegradationPreference::BALANCED && + !balanced_settings_.CanAdaptUp( + GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels, + encoder_target_bitrate_bps.value_or(0))) { + return absl::nullopt; + } + + // Attempt to find an allowed adaptation target. + switch (EffectiveDegradationPreference(input_mode)) { + case DegradationPreference::BALANCED: { + // Attempt to increase target frame rate. + int target_fps = balanced_settings_.MaxFps( + GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels); + if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) { + return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, + target_fps); + } + // Fall-through to maybe-adapting resolution, unless |balanced_settings_| + // forbids it based on bitrate. + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && + !balanced_settings_.CanAdaptUpResolution( + GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels, + encoder_target_bitrate_bps.value_or(0))) { + return absl::nullopt; + } + // Scale up resolution. + ABSL_FALLTHROUGH_INTENDED; + } + case DegradationPreference::MAINTAIN_FRAMERATE: { + // Don't adapt resolution if CanAdaptUpResolution() forbids it based on + // bitrate and limits specified by encoder capabilities. + if (reason == AdaptationObserverInterface::AdaptReason::kQuality && + !CanAdaptUpResolution(encoder_settings, encoder_target_bitrate_bps, + input_pixels)) { + return absl::nullopt; + } + // Attempt to increase pixel count. + int target_pixels = input_pixels; + if (source_restrictor_->adaptation_counters().resolution_adaptations == + 1) { + RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; + target_pixels = std::numeric_limits::max(); + } + target_pixels = GetHigherResolutionThan(target_pixels); + if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels)) + return absl::nullopt; + return AdaptationTarget(AdaptationAction::kIncreaseResolution, + target_pixels); + } + case DegradationPreference::MAINTAIN_RESOLUTION: { + // Scale up framerate. + int target_fps = input_fps; + if (source_restrictor_->adaptation_counters().fps_adaptations == 1) { + RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; + target_fps = std::numeric_limits::max(); + } + target_fps = GetHigherFrameRateThan(target_fps); + if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps)) + return absl::nullopt; + return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps); + } + case DegradationPreference::DISABLED: + return absl::nullopt; + } } -void VideoStreamAdapter::IncreaseResolutionTo(int target_pixels) { - source_restrictor_->IncreaseResolutionTo(target_pixels); +absl::optional +VideoStreamAdapter::GetAdaptDownTarget( + const absl::optional& encoder_settings, + VideoInputMode input_mode, + int input_pixels, + int input_fps, + int min_pixels_per_frame, + VideoStreamEncoderObserver* encoder_stats_observer) const { + // Preconditions for being able to adapt down: + if (input_mode == VideoInputMode::kNoVideo) + return absl::nullopt; + // 1. We are not disabled. + // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it + // causes scaling due to bandwidth constraints (QualityScalerResource) to be + // ignored, not just CPU signals. This is not a use case we want to support + // long-term; remove this enum value. + if (degradation_preference_ == DegradationPreference::DISABLED) + return absl::nullopt; + bool last_adaptation_was_down = + last_adaptation_request_ && + last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; + // 2. We shouldn't adapt down if our frame rate is below the minimum or if its + // currently unknown. + if (EffectiveDegradationPreference(input_mode) == + DegradationPreference::MAINTAIN_RESOLUTION) { + // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake + // - delete it. + if (input_fps <= 0 || + (last_adaptation_was_down && input_fps < kMinFramerateFps)) { + return absl::nullopt; + } + } + // 3. We shouldn't adapt down if we're currently waiting for a previous + // downgrade to have an effect. + // TODO(hbos): What about in the case of other degradation preferences? + if (last_adaptation_was_down && + degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && + input_pixels >= last_adaptation_request_->input_pixel_count_) { + return absl::nullopt; + } + + // Attempt to find an allowed adaptation target. + switch (EffectiveDegradationPreference(input_mode)) { + case DegradationPreference::BALANCED: { + // Try scale down framerate, if lower. + int target_fps = balanced_settings_.MinFps( + GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels); + if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { + return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, + target_fps); + } + // Scale down resolution. + ABSL_FALLTHROUGH_INTENDED; + } + case DegradationPreference::MAINTAIN_FRAMERATE: { + // Scale down resolution. + int target_pixels = GetLowerResolutionThan(input_pixels); + // TODO(https://crbug.com/webrtc/11393): Move this logic to + // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation + // target is available should not have side-effects. + if (target_pixels < min_pixels_per_frame) + encoder_stats_observer->OnMinPixelLimitReached(); + if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels, + min_pixels_per_frame)) { + return absl::nullopt; + } + return AdaptationTarget(AdaptationAction::kDecreaseResolution, + target_pixels); + } + case DegradationPreference::MAINTAIN_RESOLUTION: { + int target_fps = GetLowerFrameRateThan(input_fps); + if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) + return absl::nullopt; + return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps); + } + case DegradationPreference::DISABLED: + RTC_NOTREACHED(); + return absl::nullopt; + } } -bool VideoStreamAdapter::CanDecreaseFrameRateTo(int max_frame_rate) { - return source_restrictor_->CanDecreaseFrameRateTo(max_frame_rate); -} - -void VideoStreamAdapter::DecreaseFrameRateTo(int max_frame_rate) { - source_restrictor_->DecreaseFrameRateTo(max_frame_rate); -} - -bool VideoStreamAdapter::CanIncreaseFrameRateTo(int max_frame_rate) { - return source_restrictor_->CanIncreaseFrameRateTo(max_frame_rate); -} - -void VideoStreamAdapter::IncreaseFrameRateTo(int max_frame_rate) { - source_restrictor_->IncreaseFrameRateTo(max_frame_rate); +void VideoStreamAdapter::ApplyAdaptationTarget(const AdaptationTarget& target, + VideoInputMode input_mode, + int input_pixels, + int input_fps, + int min_pixels_per_frame) { + // Remember the input pixels and fps of this adaptation. Used to avoid + // adapting again before this adaptation has had an effect. + last_adaptation_request_.emplace(AdaptationRequest{ + input_pixels, input_fps, + AdaptationRequest::GetModeFromAdaptationAction(target.action)}); + switch (target.action) { + case AdaptationAction::kIncreaseResolution: + source_restrictor_->IncreaseResolutionTo(target.value); + return; + case AdaptationAction::kDecreaseResolution: + source_restrictor_->DecreaseResolutionTo(target.value, + min_pixels_per_frame); + return; + case AdaptationAction::kIncreaseFrameRate: + source_restrictor_->IncreaseFrameRateTo(target.value); + // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps. + // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic + // in DecrementFramerate() makes it hard to predict whether this will be + // the last step. Remove the dependency on GetConstAdaptCounter(). + if (EffectiveDegradationPreference(input_mode) == + DegradationPreference::BALANCED && + source_restrictor_->adaptation_counters().fps_adaptations == 0 && + target.value != std::numeric_limits::max()) { + RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; + source_restrictor_->IncreaseFrameRateTo( + std::numeric_limits::max()); + } + return; + case AdaptationAction::kDecreaseFrameRate: + source_restrictor_->DecreaseFrameRateTo(target.value); + return; + } } } // namespace webrtc diff --git a/video/adaptation/video_stream_adapter.h b/video/adaptation/video_stream_adapter.h index ddcd6f5ba7..40e35ecc77 100644 --- a/video/adaptation/video_stream_adapter.h +++ b/video/adaptation/video_stream_adapter.h @@ -13,7 +13,13 @@ #include +#include "absl/types/optional.h" +#include "api/rtp_parameters.h" +#include "api/video/video_stream_encoder_observer.h" +#include "call/adaptation/encoder_settings.h" #include "call/adaptation/video_source_restrictions.h" +#include "modules/video_coding/utility/quality_scaler.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" #include "video/adaptation/adaptation_counters.h" namespace webrtc { @@ -26,39 +32,128 @@ namespace webrtc { // 3. Modify the stream's restrictions in one of the valid ways. class VideoStreamAdapter { public: - static const int kMinFramerateFps; + enum class SetDegradationPreferenceResult { + kRestrictionsNotCleared, + kRestrictionsCleared, + }; - static int GetLowerFrameRateThan(int fps); - static int GetHigherFrameRateThan(int fps); - static int GetLowerResolutionThan(int pixel_count); - static int GetHigherResolutionThan(int pixel_count); + enum class VideoInputMode { + kNoVideo, + kNormalVideo, + kScreenshareVideo, + }; + + enum class AdaptationAction { + kIncreaseResolution, + kDecreaseResolution, + kIncreaseFrameRate, + kDecreaseFrameRate, + }; + + // Describes an adaptation step: increasing or decreasing resolution or frame + // rate to a given value. + // TODO(https://crbug.com/webrtc/11393): Make these private implementation + // details, and expose something that allows you to inspect the + // VideoSourceRestrictions instead. The adaptation steps could be expressed as + // a graph, for instance. + struct AdaptationTarget { + AdaptationTarget(AdaptationAction action, int value); + // Which action the VideoSourceRestrictor needs to take. + const AdaptationAction action; + // Target pixel count or frame rate depending on |action|. + const int value; + + // Allow this struct to be instantiated as an optional, even though it's in + // a private namespace. + friend class absl::optional; + }; VideoStreamAdapter(); ~VideoStreamAdapter(); - // TODO(hbos): Why isn't this const? VideoSourceRestrictions source_restrictions() const; const AdaptationCounters& adaptation_counters() const; + // TODO(hbos): Can we get rid of any external dependencies on + // BalancedDegradationPreference? How the adaptor generates possible next + // steps for adaptation should be an implementation detail. Can the relevant + // information be inferred from GetAdaptUpTarget()/GetAdaptDownTarget()? + const BalancedDegradationSettings& balanced_settings() const; void ClearRestrictions(); - // "Can adapt?" and "do adapt!" methods. - // TODO(https://crbug.com/webrtc/11393): Make the adapter responsible for - // deciding what the next step are, i.e. taking on degradation preference - // logic. Then, these can be expressed either as CanAdaptUp() and DoAdaptUp() - // or as GetNextRestrictionsUp() and ApplyRestrictions(). - bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame); - void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame); - bool CanIncreaseResolutionTo(int target_pixels); - void IncreaseResolutionTo(int target_pixels); - bool CanDecreaseFrameRateTo(int max_frame_rate); - void DecreaseFrameRateTo(int max_frame_rate); - bool CanIncreaseFrameRateTo(int max_frame_rate); - void IncreaseFrameRateTo(int max_frame_rate); + // TODO(hbos): Setting the degradation preference should not clear + // restrictions! This is not defined in the spec and is unexpected, there is a + // tiny risk that people would discover and rely on this behavior. + SetDegradationPreferenceResult SetDegradationPreference( + DegradationPreference degradation_preference); + // TODO(hbos): This is only used in one place externally by + // OveruseFrameDetectorResourceAdaptationModule - can we get rid of that + // usage? This is exposing an implementation detail. + DegradationPreference EffectiveDegradationPreference( + VideoInputMode input_mode) const; + + // Returns a target that we are guaranteed to be able to adapt to, or null if + // adaptation is not desired or not possible. + absl::optional GetAdaptUpTarget( + const absl::optional& encoder_settings, + absl::optional encoder_target_bitrate_bps, + VideoInputMode input_mode, + int input_pixels, + int input_fps, + AdaptationObserverInterface::AdaptReason reason) const; + // TODO(https://crbug.com/webrtc/11393): Remove the dependency on + // |encoder_stats_observer| - simply checking which adaptation target is + // available should not have side-effects. + absl::optional GetAdaptDownTarget( + const absl::optional& encoder_settings, + VideoInputMode input_mode, + int input_pixels, + int input_fps, + int min_pixels_per_frame, + VideoStreamEncoderObserver* encoder_stats_observer) const; + // Applies the |target| to |source_restrictor_|. + void ApplyAdaptationTarget(const AdaptationTarget& target, + VideoInputMode input_mode, + int input_pixels, + int input_fps, + int min_pixels_per_frame); private: class VideoSourceRestrictor; + // The input frame rate and resolution at the time of an adaptation in the + // direction described by |mode_| (up or down). + // TODO(https://crbug.com/webrtc/11393): Can this be renamed? Can this be + // merged with AdaptationTarget? + struct AdaptationRequest { + // The pixel count produced by the source at the time of the adaptation. + int input_pixel_count_; + // Framerate received from the source at the time of the adaptation. + int framerate_fps_; + // Indicates if request was to adapt up or down. + enum class Mode { kAdaptUp, kAdaptDown } mode_; + + // This is a static method rather than an anonymous namespace function due + // to namespace visiblity. + static Mode GetModeFromAdaptationAction(AdaptationAction action); + }; + + // Owner and modifier of the VideoSourceRestriction of this stream adaptor. const std::unique_ptr source_restrictor_; + // Decides the next adaptation target in DegradationPreference::BALANCED. + const BalancedDegradationSettings balanced_settings_; + // When deciding the next target up or down, different strategies are used + // depending on the DegradationPreference. + // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference + DegradationPreference degradation_preference_; + + // The input frame rate, resolution and adaptation direction of the last + // ApplyAdaptationTarget(). Used to avoid adapting twice if a recent + // adaptation has not had an effect on the input frame rate or resolution yet. + // TODO(hbos): Can we implement a more general "cooldown" mechanism of + // resources intead? If we already have adapted it seems like we should wait + // a while before adapting again, so that we are not acting on usage + // measurements that are made obsolete/unreliable by an "ongoing" adaptation. + absl::optional last_adaptation_request_; }; } // namespace webrtc diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index c203b8fa63..d86b7d90ca 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -164,10 +164,6 @@ class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper { int initial_framedrop_; }; -OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget:: - AdaptationTarget(AdaptationAction action, int value) - : action(action), value(value) {} - OveruseFrameDetectorResourceAdaptationModule:: OveruseFrameDetectorResourceAdaptationModule( Clock* clock, @@ -181,8 +177,6 @@ OveruseFrameDetectorResourceAdaptationModule:: experiment_cpu_load_estimator_(experiment_cpu_load_estimator), has_input_video_(false), degradation_preference_(DegradationPreference::DISABLED), - balanced_settings_(), - last_adaptation_request_(absl::nullopt), stream_adapter_(std::make_unique()), encode_usage_resource_( std::make_unique(std::move(overuse_detector))), @@ -260,17 +254,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( DegradationPreference degradation_preference) { - if (degradation_preference_ != degradation_preference) { - // Reset adaptation state, so that we're not tricked into thinking there's - // an already pending request of the same type. - last_adaptation_request_.reset(); - if (degradation_preference == DegradationPreference::BALANCED || - degradation_preference_ == DegradationPreference::BALANCED) { - stream_adapter_->ClearRestrictions(); - active_counts_.fill(AdaptationCounters()); - } - } degradation_preference_ = degradation_preference; + if (stream_adapter_->SetDegradationPreference(degradation_preference) == + VideoStreamAdapter::SetDegradationPreferenceResult:: + kRestrictionsCleared) { + active_counts_.fill(AdaptationCounters()); + } MaybeUpdateVideoSourceRestrictions(); } @@ -307,7 +296,6 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates( void OveruseFrameDetectorResourceAdaptationModule:: ResetVideoSourceRestrictions() { - last_adaptation_request_.reset(); stream_adapter_->ClearRestrictions(); active_counts_.fill(AdaptationCounters()); MaybeUpdateVideoSourceRestrictions(); @@ -397,7 +385,7 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( absl::optional experimental_thresholds; if (quality_scaling_experiment_enabled_) { experimental_thresholds = QualityScalingExperiment::GetQpThresholds( - GetVideoCodecTypeOrGeneric()); + GetVideoCodecTypeOrGeneric(encoder_settings_)); } UpdateQualityScalerSettings(experimental_thresholds ? *experimental_thresholds @@ -411,8 +399,9 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( if (degradation_preference_ == DegradationPreference::BALANCED && quality_scaler_resource_->is_started()) { absl::optional thresholds = - balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(), - LastInputFrameSizeOrDefault()); + stream_adapter_->balanced_settings().GetQpThresholds( + GetVideoCodecTypeOrGeneric(encoder_settings_), + LastInputFrameSizeOrDefault()); if (thresholds) { quality_scaler_resource_->SetQpThresholds(*thresholds); } @@ -454,212 +443,51 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured( } } -absl::optional +absl::optional OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget( int input_pixels, int input_fps, AdaptationObserverInterface::AdaptReason reason) const { - // Preconditions for being able to adapt up: - if (!has_input_video_) - return absl::nullopt; - // 1. We can't adapt up if we're already at the highest setting. + // We can't adapt up if we're already at the highest setting. // Note that this only includes counts relevant to the current degradation // preference. e.g. we previously adapted resolution, now prefer adpating fps, // only count the fps adaptations and not the previous resolution adaptations. + // // TODO(https://crbug.com/webrtc/11394): Checking the counts for reason should - // be replaced with checking the overuse state of all resources. + // be replaced with checking the overuse state of all resources. This is + // effectively trying to infer if the the Resource specified by |reason| is OK + // with adapting up by looking at active counters. If the relevant Resources + // simply told us this directly we wouldn't have to depend on stats counters + // to abort GetAdaptUpTarget(). int num_downgrades = ApplyDegradationPreference(active_counts_[reason], degradation_preference_) .Total(); RTC_DCHECK_GE(num_downgrades, 0); if (num_downgrades == 0) return absl::nullopt; - // 2. We shouldn't adapt up if we're currently waiting for a previous upgrade - // to have an effect. - // TODO(hbos): What about in the case of other degradation preferences? - bool last_adaptation_was_up = - last_adaptation_request_ && - last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; - if (last_adaptation_was_up && - degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && - input_pixels <= last_adaptation_request_->input_pixel_count_) { - return absl::nullopt; - } - // 3. We shouldn't adapt up if BalancedSettings doesn't allow it, which is - // only applicable if reason is kQuality and preference is BALANCED. - if (reason == AdaptationObserverInterface::AdaptReason::kQuality && - EffectiveDegradationPreference() == DegradationPreference::BALANCED && - !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(), input_pixels, - encoder_target_bitrate_bps_.value_or(0))) { - return absl::nullopt; - } - - // Attempt to find an allowed adaptation target. - switch (EffectiveDegradationPreference()) { - case DegradationPreference::BALANCED: { - // Attempt to increase target frame rate. - int target_fps = - balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(), input_pixels); - if (stream_adapter_->CanIncreaseFrameRateTo(target_fps)) { - return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, - target_fps); - } - // Fall-through to maybe-adapting resolution, unless |balanced_settings_| - // forbids it based on bitrate. - if (reason == AdaptationObserverInterface::AdaptReason::kQuality && - !balanced_settings_.CanAdaptUpResolution( - GetVideoCodecTypeOrGeneric(), input_pixels, - encoder_target_bitrate_bps_.value_or(0))) { - return absl::nullopt; - } - // Scale up resolution. - ABSL_FALLTHROUGH_INTENDED; - } - case DegradationPreference::MAINTAIN_FRAMERATE: { - // Don't adapt resolution if CanAdaptUpResolution() forbids it based on - // bitrate and limits specified by encoder capabilities. - if (reason == AdaptationObserverInterface::AdaptReason::kQuality && - !CanAdaptUpResolution(input_pixels, - encoder_target_bitrate_bps_.value_or(0))) { - return absl::nullopt; - } - // Attempt to increase pixel count. - int target_pixels = input_pixels; - if (stream_adapter_->adaptation_counters().resolution_adaptations == 1) { - RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; - target_pixels = std::numeric_limits::max(); - } - target_pixels = - VideoStreamAdapter::GetHigherResolutionThan(target_pixels); - if (!stream_adapter_->CanIncreaseResolutionTo(target_pixels)) - return absl::nullopt; - return AdaptationTarget(AdaptationAction::kIncreaseResolution, - target_pixels); - } - case DegradationPreference::MAINTAIN_RESOLUTION: { - // Scale up framerate. - int target_fps = input_fps; - if (stream_adapter_->adaptation_counters().fps_adaptations == 1) { - RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - target_fps = std::numeric_limits::max(); - } - target_fps = VideoStreamAdapter::GetHigherFrameRateThan(target_fps); - if (!stream_adapter_->CanIncreaseFrameRateTo(target_fps)) - return absl::nullopt; - return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps); - } - case DegradationPreference::DISABLED: - return absl::nullopt; - } + return stream_adapter_->GetAdaptUpTarget( + encoder_settings_, encoder_target_bitrate_bps_, GetVideoInputMode(), + input_pixels, input_fps, reason); } -absl::optional +absl::optional OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget( int input_pixels, int input_fps, int min_pixels_per_frame) const { - // Preconditions for being able to adapt down: - if (!has_input_video_) - return absl::nullopt; - // 1. We are not disabled. - // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it - // causes scaling due to bandwidth constraints (QualityScalerResource) to be - // ignored, not just CPU signals. This is not a use case we want to support; - // remove the enum value. - if (degradation_preference_ == DegradationPreference::DISABLED) - return absl::nullopt; - bool last_adaptation_was_down = - last_adaptation_request_ && - last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; - // 2. We shouldn't adapt down if our frame rate is below the minimum or if its - // currently unknown. - if (EffectiveDegradationPreference() == - DegradationPreference::MAINTAIN_RESOLUTION) { - // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake - // - delete it. - if (input_fps <= 0 || (last_adaptation_was_down && - input_fps < VideoStreamAdapter::kMinFramerateFps)) { - return absl::nullopt; - } - } - // 3. We shouldn't adapt down if we're currently waiting for a previous - // downgrade to have an effect. - // TODO(hbos): What about in the case of other degradation preferences? - if (last_adaptation_was_down && - degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && - input_pixels >= last_adaptation_request_->input_pixel_count_) { - return absl::nullopt; - } - - // Attempt to find an allowed adaptation target. - switch (EffectiveDegradationPreference()) { - case DegradationPreference::BALANCED: { - // Try scale down framerate, if lower. - int target_fps = - balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(), input_pixels); - if (stream_adapter_->CanDecreaseFrameRateTo(target_fps)) { - return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, - target_fps); - } - // Scale down resolution. - ABSL_FALLTHROUGH_INTENDED; - } - case DegradationPreference::MAINTAIN_FRAMERATE: { - // Scale down resolution. - int target_pixels = - VideoStreamAdapter::GetLowerResolutionThan(input_pixels); - // TODO(https://crbug.com/webrtc/11222): Move this logic to - // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation - // target is available should not have side-effects. - if (target_pixels < min_pixels_per_frame) - encoder_stats_observer_->OnMinPixelLimitReached(); - if (!stream_adapter_->CanDecreaseResolutionTo(target_pixels, - min_pixels_per_frame)) { - return absl::nullopt; - } - return AdaptationTarget(AdaptationAction::kDecreaseResolution, - target_pixels); - } - case DegradationPreference::MAINTAIN_RESOLUTION: { - int target_fps = VideoStreamAdapter::GetLowerFrameRateThan(input_fps); - if (!stream_adapter_->CanDecreaseFrameRateTo(target_fps)) - return absl::nullopt; - return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps); - } - case DegradationPreference::DISABLED: - RTC_NOTREACHED(); - return absl::nullopt; - } + return stream_adapter_->GetAdaptDownTarget( + encoder_settings_, GetVideoInputMode(), input_pixels, input_fps, + min_pixels_per_frame, encoder_stats_observer_); } void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget( - const AdaptationTarget& target, - int min_pixels_per_frame, - AdaptationObserverInterface::AdaptReason reason) { - switch (target.action) { - case AdaptationAction::kIncreaseResolution: - stream_adapter_->IncreaseResolutionTo(target.value); - return; - case AdaptationAction::kDecreaseResolution: - stream_adapter_->DecreaseResolutionTo(target.value, min_pixels_per_frame); - return; - case AdaptationAction::kIncreaseFrameRate: - stream_adapter_->IncreaseFrameRateTo(target.value); - // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps. - // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic - // in DecrementFramerate() makes it hard to predict whether this will be - // the last step. Remove the dependency on GetConstAdaptCounter(). - if (EffectiveDegradationPreference() == DegradationPreference::BALANCED && - stream_adapter_->adaptation_counters().fps_adaptations == 0 && - target.value != std::numeric_limits::max()) { - RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - stream_adapter_->IncreaseFrameRateTo(std::numeric_limits::max()); - } - return; - case AdaptationAction::kDecreaseFrameRate: - stream_adapter_->DecreaseFrameRateTo(target.value); - return; - } + const VideoStreamAdapter::AdaptationTarget& target, + int input_pixels, + int input_fps, + int min_pixels_per_frame) { + stream_adapter_->ApplyAdaptationTarget(target, GetVideoInputMode(), + input_pixels, input_fps, + min_pixels_per_frame); } void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( @@ -668,14 +496,13 @@ void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( int input_fps = encoder_stats_observer_->GetInputFrameRate(); int min_pixels_per_frame = MinPixelsPerFrame(); // Should we adapt, if so to what target? - absl::optional target = + absl::optional target = GetAdaptUpTarget(input_pixels, input_fps, reason); if (!target.has_value()) return; // Apply target. - ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason); - last_adaptation_request_.emplace(AdaptationRequest{ - input_pixels, input_fps, AdaptationRequest::Mode::kAdaptUp}); + ApplyAdaptationTarget(target.value(), input_pixels, input_fps, + min_pixels_per_frame); // Update VideoSourceRestrictions based on adaptation. This also informs the // |adaptation_listener_|. MaybeUpdateVideoSourceRestrictions(); @@ -693,14 +520,13 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( int input_fps = encoder_stats_observer_->GetInputFrameRate(); int min_pixels_per_frame = MinPixelsPerFrame(); // Should we adapt, if so to what target? - absl::optional target = + absl::optional target = GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame); if (!target.has_value()) return ResourceListenerResponse::kNothing; // Apply target. - ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason); - last_adaptation_request_.emplace(AdaptationRequest{ - input_pixels, input_fps, AdaptationRequest::Mode::kAdaptDown}); + ApplyAdaptationTarget(target.value(), input_pixels, input_fps, + min_pixels_per_frame); // Update VideoSourceRestrictions based on adaptation. This also informs the // |adaptation_listener_|. MaybeUpdateVideoSourceRestrictions(); @@ -709,9 +535,12 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( RTC_LOG(INFO) << ActiveCountsToString(); // In BALANCED, if requested FPS is higher or close to input FPS to the target // we tell the QualityScaler to increase its frequency. - if (EffectiveDegradationPreference() == DegradationPreference::BALANCED && - target->action == AdaptationAction::kDecreaseFrameRate) { - absl::optional min_diff = balanced_settings_.MinFpsDiff(input_pixels); + if (stream_adapter_->EffectiveDegradationPreference(GetVideoInputMode()) == + DegradationPreference::BALANCED && + target->action == + VideoStreamAdapter::AdaptationAction::kDecreaseFrameRate) { + absl::optional min_diff = + stream_adapter_->balanced_settings().MinFpsDiff(input_pixels); if (min_diff && input_fps > 0) { int fps_diff = input_fps - target->value; if (fps_diff < min_diff.value()) { @@ -744,14 +573,6 @@ OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const { return options; } -VideoCodecType -OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric() - const { - return encoder_settings_.has_value() - ? encoder_settings_->encoder_config().codec_type - : kVideoCodecGeneric; -} - int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault() const { // The dependency on this hardcoded resolution is inherited from old code, @@ -924,37 +745,17 @@ OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( return counts; } -DegradationPreference -OveruseFrameDetectorResourceAdaptationModule::EffectiveDegradationPreference() - const { - // Balanced mode for screenshare works via automatic animation detection: - // Resolution is capped for fullscreen animated content. - // Adapatation is done only via framerate downgrade. - // Thus effective degradation preference is MAINTAIN_RESOLUTION. +VideoStreamAdapter::VideoInputMode +OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const { + if (!has_input_video_) + return VideoStreamAdapter::VideoInputMode::kNoVideo; return (encoder_settings_.has_value() && encoder_settings_->encoder_config().content_type == - VideoEncoderConfig::ContentType::kScreen && - degradation_preference_ == DegradationPreference::BALANCED) - ? DegradationPreference::MAINTAIN_RESOLUTION - : degradation_preference_; + VideoEncoderConfig::ContentType::kScreen) + ? VideoStreamAdapter::VideoInputMode::kScreenshareVideo + : VideoStreamAdapter::VideoInputMode::kNormalVideo; } -bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( - int pixels, - uint32_t bitrate_bps) const { - absl::optional bitrate_limits = - encoder_settings_.has_value() - ? GetEncoderBitrateLimits( - encoder_settings_->encoder_info(), - VideoStreamAdapter::GetHigherResolutionThan(pixels)) - : absl::nullopt; - if (!bitrate_limits.has_value() || bitrate_bps == 0) { - return true; // No limit configured or bitrate provided. - } - RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels); - return bitrate_bps >= - static_cast(bitrate_limits->min_start_bitrate_bps); -} void OveruseFrameDetectorResourceAdaptationModule:: MaybePerformQualityRampupExperiment() { if (!quality_scaler_resource_->is_started()) diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 14607fee74..e80bd6f1f6 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -128,55 +128,25 @@ class OveruseFrameDetectorResourceAdaptationModule AdaptationCounters* other_active); private: - class VideoSourceRestrictor; class InitialFrameDropper; enum class State { kStopped, kStarted }; - struct AdaptationRequest { - // The pixel count produced by the source at the time of the adaptation. - int input_pixel_count_; - // Framerate received from the source at the time of the adaptation. - int framerate_fps_; - // Indicates if request was to adapt up or down. - enum class Mode { kAdaptUp, kAdaptDown } mode_; - }; - - enum class AdaptationAction { - kIncreaseResolution, - kDecreaseResolution, - kIncreaseFrameRate, - kDecreaseFrameRate, - }; - - // Describes an adaptation step: increasing or decreasing resolution or frame - // rate to a given value. - struct AdaptationTarget { - AdaptationTarget(AdaptationAction action, int value); - // Which action the VideoSourceRestrictor needs to take. - const AdaptationAction action; - // Target pixel count or frame rate depending on |action|. - const int value; - - // Allow this struct to be instantiated as an optional, even though it's in - // a private namespace. - friend class absl::optional; - }; - // Returns a target that we are guaranteed to be able to adapt to, or null if // adaptation is not desired or not possible. - absl::optional GetAdaptUpTarget( + absl::optional GetAdaptUpTarget( int input_pixels, int input_fps, AdaptationObserverInterface::AdaptReason reason) const; - absl::optional GetAdaptDownTarget( + absl::optional GetAdaptDownTarget( int input_pixels, int input_fps, int min_pixels_per_frame) const; // Applies the |target| to |source_restrictor_|. - void ApplyAdaptationTarget(const AdaptationTarget& target, - int min_pixels_per_frame, - AdaptationObserverInterface::AdaptReason reason); + void ApplyAdaptationTarget(const VideoStreamAdapter::AdaptationTarget& target, + int input_pixels, + int input_fps, + int min_pixels_per_frame); // Performs the adaptation by getting the next target, applying it and // informing listeners of the new VideoSourceRestriction and adapt counters. @@ -185,11 +155,11 @@ class OveruseFrameDetectorResourceAdaptationModule AdaptationObserverInterface::AdaptReason reason); CpuOveruseOptions GetCpuOveruseOptions() const; - VideoCodecType GetVideoCodecTypeOrGeneric() const; int LastInputFrameSizeOrDefault() const; int MinPixelsPerFrame() const; VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( AdaptationObserverInterface::AdaptReason reason); + VideoStreamAdapter::VideoInputMode GetVideoInputMode() const; // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener @@ -204,8 +174,6 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional qp_thresholds); void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason); - DegradationPreference EffectiveDegradationPreference() const; - bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const; // Checks to see if we should execute the quality rampup experiment. The // experiment resets all video restrictions at the start of the call in the @@ -224,11 +192,12 @@ class OveruseFrameDetectorResourceAdaptationModule // The restrictions that |adaptation_listener_| is informed of. VideoSourceRestrictions video_source_restrictions_; bool has_input_video_; + // TODO(https://crbug.com/webrtc/11393): DegradationPreference has mostly + // moved to VideoStreamAdapter. Move it entirely and delete it from this + // class. If the responsibility of generating next steps for adaptations is + // owned by the adapter, this class has no buisness relying on implementation + // details of the adapter. DegradationPreference degradation_preference_; - const BalancedDegradationSettings balanced_settings_; - // Stores a snapshot of the last adaptation request triggered by an AdaptUp - // or AdaptDown signal. - absl::optional last_adaptation_request_; // Keeps track of source restrictions that this adaptation module outputs. const std::unique_ptr stream_adapter_; const std::unique_ptr encode_usage_resource_; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index b99e91185d..58fba37bae 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -171,43 +171,6 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo( } } // namespace -absl::optional GetEncoderBitrateLimits( - const VideoEncoder::EncoderInfo& encoder_info, - int frame_size_pixels) { - std::vector bitrate_limits = - encoder_info.resolution_bitrate_limits; - - // Sort the list of bitrate limits by resolution. - sort(bitrate_limits.begin(), bitrate_limits.end(), - [](const VideoEncoder::ResolutionBitrateLimits& lhs, - const VideoEncoder::ResolutionBitrateLimits& rhs) { - return lhs.frame_size_pixels < rhs.frame_size_pixels; - }); - - for (size_t i = 0; i < bitrate_limits.size(); ++i) { - RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0); - RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0); - RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, - bitrate_limits[i].min_bitrate_bps); - if (i > 0) { - // The bitrate limits aren't expected to decrease with resolution. - RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, - bitrate_limits[i - 1].min_bitrate_bps); - RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, - bitrate_limits[i - 1].min_start_bitrate_bps); - RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, - bitrate_limits[i - 1].max_bitrate_bps); - } - - if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) { - return absl::optional( - bitrate_limits[i]); - } - } - - return absl::nullopt; -} - const int VideoStreamEncoder::kDefaultLastFrameInfoWidth = 176; const int VideoStreamEncoder::kDefaultLastFrameInfoHeight = 144; @@ -503,9 +466,9 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_reset_required = true; } - encoder_bitrate_limits_ = GetEncoderBitrateLimits( - encoder_->GetEncoderInfo(), - last_frame_info_->width * last_frame_info_->height); + encoder_bitrate_limits_ = + encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( + last_frame_info_->width * last_frame_info_->height); if (streams.size() == 1 && encoder_bitrate_limits_) { // Bitrate limits can be set by app (in SDP or RtpEncodingParameters) or/and @@ -1630,7 +1593,8 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { } absl::optional encoder_bitrate_limits = - GetEncoderBitrateLimits(encoder_->GetEncoderInfo(), pixel_count); + encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( + pixel_count); if (encoder_bitrate_limits.has_value()) { // Use bitrate limits provided by encoder. diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 64f5e440d6..904a741f42 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -45,10 +45,6 @@ namespace webrtc { -absl::optional GetEncoderBitrateLimits( - const VideoEncoder::EncoderInfo& encoder_info, - int frame_size_pixels); - // VideoStreamEncoder represent a video encoder that accepts raw video frames as // input and produces an encoded bit stream. // Usage: