diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index f8bb6d8884..f4e9792bba 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -50,6 +50,7 @@ rtc_library("video_adaptation") { "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/base:core_headers", "//third_party/abseil-cpp/absl/types:optional", + "//third_party/abseil-cpp/absl/types:variant", ] } diff --git a/video/adaptation/resource_adaptation_processor.cc b/video/adaptation/resource_adaptation_processor.cc index 7958be4446..eccf0f2e2b 100644 --- a/video/adaptation/resource_adaptation_processor.cc +++ b/video/adaptation/resource_adaptation_processor.cc @@ -461,15 +461,16 @@ void ResourceAdaptationProcessor::OnResourceUnderuse( const int input_pixels = LastInputFrameSizeOrDefault(); const int input_fps = encoder_stats_observer_->GetInputFrameRate(); // Should we adapt, if so to what target? - absl::optional target = + VideoStreamAdapter::AdaptationTargetOrReason target_or_reason = stream_adapter_->GetAdaptUpTarget(encoder_settings_, encoder_target_bitrate_bps_, input_mode, input_pixels, input_fps, reason); - if (!target.has_value()) + if (!target_or_reason.has_target()) return; // Apply target. - stream_adapter_->ApplyAdaptationTarget(target.value(), encoder_settings_, - input_mode, input_pixels, input_fps); + stream_adapter_->ApplyAdaptationTarget(target_or_reason.target(), + encoder_settings_, input_mode, + input_pixels, input_fps); // Update VideoSourceRestrictions based on adaptation. This also informs the // |adaptation_listener_|. MaybeUpdateVideoSourceRestrictions(); @@ -487,15 +488,17 @@ ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse( const int input_pixels = LastInputFrameSizeOrDefault(); const int input_fps = encoder_stats_observer_->GetInputFrameRate(); // Should we adapt, if so to what target? - absl::optional target = + VideoStreamAdapter::AdaptationTargetOrReason target_or_reason = stream_adapter_->GetAdaptDownTarget(encoder_settings_, input_mode, - input_pixels, input_fps, - encoder_stats_observer_); - if (!target.has_value()) + input_pixels, input_fps); + if (target_or_reason.min_pixel_limit_reached()) + encoder_stats_observer_->OnMinPixelLimitReached(); + if (!target_or_reason.has_target()) return ResourceListenerResponse::kNothing; // Apply target. ResourceListenerResponse response = stream_adapter_->ApplyAdaptationTarget( - target.value(), encoder_settings_, input_mode, input_pixels, input_fps); + target_or_reason.target(), encoder_settings_, input_mode, input_pixels, + input_fps); // Update VideoSourceRestrictions based on adaptation. This also informs the // |adaptation_listener_|. MaybeUpdateVideoSourceRestrictions(); diff --git a/video/adaptation/video_stream_adapter.cc b/video/adaptation/video_stream_adapter.cc index 078410a94b..d0c0ebeff6 100644 --- a/video/adaptation/video_stream_adapter.cc +++ b/video/adaptation/video_stream_adapter.cc @@ -12,8 +12,10 @@ #include #include +#include #include "absl/types/optional.h" +#include "absl/types/variant.h" #include "api/video_codecs/video_encoder.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/logging.h" @@ -98,6 +100,49 @@ VideoStreamAdapter::AdaptationTarget::AdaptationTarget(AdaptationAction action, int value) : action(action), value(value) {} +VideoStreamAdapter::AdaptationTargetOrReason::AdaptationTargetOrReason( + AdaptationTarget target, + bool min_pixel_limit_reached) + : target_or_reason_(target), + min_pixel_limit_reached_(min_pixel_limit_reached) {} + +VideoStreamAdapter::AdaptationTargetOrReason::AdaptationTargetOrReason( + CannotAdaptReason reason, + bool min_pixel_limit_reached) + : target_or_reason_(reason), + min_pixel_limit_reached_(min_pixel_limit_reached) {} + +// implicit +VideoStreamAdapter::AdaptationTargetOrReason::AdaptationTargetOrReason( + AdaptationTarget target) + : target_or_reason_(target), min_pixel_limit_reached_(false) {} + +// implicit +VideoStreamAdapter::AdaptationTargetOrReason::AdaptationTargetOrReason( + CannotAdaptReason reason) + : target_or_reason_(reason), min_pixel_limit_reached_(false) {} + +bool VideoStreamAdapter::AdaptationTargetOrReason::has_target() const { + return absl::holds_alternative(target_or_reason_); +} + +const VideoStreamAdapter::AdaptationTarget& +VideoStreamAdapter::AdaptationTargetOrReason::target() const { + RTC_DCHECK(has_target()); + return absl::get(target_or_reason_); +} + +VideoStreamAdapter::CannotAdaptReason +VideoStreamAdapter::AdaptationTargetOrReason::reason() const { + RTC_DCHECK(!has_target()); + return absl::get(target_or_reason_); +} + +bool VideoStreamAdapter::AdaptationTargetOrReason::min_pixel_limit_reached() + const { + return min_pixel_limit_reached_; +} + // VideoSourceRestrictor is responsible for keeping track of current // VideoSourceRestrictions. class VideoStreamAdapter::VideoSourceRestrictor { @@ -270,7 +315,7 @@ VideoStreamAdapter::SetDegradationPreference( : SetDegradationPreferenceResult::kRestrictionsNotCleared; } -absl::optional +VideoStreamAdapter::AdaptationTargetOrReason VideoStreamAdapter::GetAdaptUpTarget( const absl::optional& encoder_settings, absl::optional encoder_target_bitrate_bps, @@ -278,32 +323,31 @@ VideoStreamAdapter::GetAdaptUpTarget( int input_pixels, int input_fps, AdaptationObserverInterface::AdaptReason reason) const { - // Preconditions for being able to adapt up: - if (input_mode == VideoInputMode::kNoVideo) - return absl::nullopt; - // 1. We shouldn't adapt up if we're currently waiting for a previous upgrade - // to have an effect. - // TODO(hbos): What about in the case of other degradation preferences? + // Don't adapt if we don't have sufficient input. + if (input_mode == VideoInputMode::kNoVideo) { + return CannotAdaptReason::kInsufficientInput; + } + // Don't adapt if we're awaiting a previous adaptation to have an effect. bool last_adaptation_was_up = last_adaptation_request_ && last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; if (last_adaptation_was_up && degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && input_pixels <= last_adaptation_request_->input_pixel_count_) { - return absl::nullopt; + return CannotAdaptReason::kAwaitingPreviousAdaptation; } - // 2. We shouldn't adapt up if BalancedSettings doesn't allow it, which is - // only applicable if reason is kQuality and preference is BALANCED. + // Don't adapt if BalancedDegradationSettings applies and determines this will + // exceed bitrate constraints. if (reason == AdaptationObserverInterface::AdaptReason::kQuality && EffectiveDegradationPreference(input_mode) == DegradationPreference::BALANCED && !balanced_settings_.CanAdaptUp( GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels, encoder_target_bitrate_bps.value_or(0))) { - return absl::nullopt; + return CannotAdaptReason::kIsBitrateConstrained; } - // Attempt to find an allowed adaptation target. + // Maybe propose targets based on degradation preference. switch (EffectiveDegradationPreference(input_mode)) { case DegradationPreference::BALANCED: { // Attempt to increase target frame rate. @@ -319,7 +363,7 @@ VideoStreamAdapter::GetAdaptUpTarget( !balanced_settings_.CanAdaptUpResolution( GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels, encoder_target_bitrate_bps.value_or(0))) { - return absl::nullopt; + return CannotAdaptReason::kIsBitrateConstrained; } // Scale up resolution. ABSL_FALLTHROUGH_INTENDED; @@ -330,7 +374,7 @@ VideoStreamAdapter::GetAdaptUpTarget( if (reason == AdaptationObserverInterface::AdaptReason::kQuality && !CanAdaptUpResolution(encoder_settings, encoder_target_bitrate_bps, input_pixels)) { - return absl::nullopt; + return CannotAdaptReason::kIsBitrateConstrained; } // Attempt to increase pixel count. int target_pixels = input_pixels; @@ -340,8 +384,9 @@ VideoStreamAdapter::GetAdaptUpTarget( target_pixels = std::numeric_limits::max(); } target_pixels = GetHigherResolutionThan(target_pixels); - if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels)) - return absl::nullopt; + if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels)) { + return CannotAdaptReason::kLimitReached; + } return AdaptationTarget(AdaptationAction::kIncreaseResolution, target_pixels); } @@ -353,57 +398,50 @@ VideoStreamAdapter::GetAdaptUpTarget( target_fps = std::numeric_limits::max(); } target_fps = GetHigherFrameRateThan(target_fps); - if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps)) - return absl::nullopt; + if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps)) { + return CannotAdaptReason::kLimitReached; + } return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps); } case DegradationPreference::DISABLED: - return absl::nullopt; + return CannotAdaptReason::kAdaptationDisabled; } } -absl::optional +VideoStreamAdapter::AdaptationTargetOrReason VideoStreamAdapter::GetAdaptDownTarget( const absl::optional& encoder_settings, VideoInputMode input_mode, int input_pixels, - int input_fps, - VideoStreamEncoderObserver* encoder_stats_observer) const { + int input_fps) const { const int min_pixels_per_frame = MinPixelsPerFrame(encoder_settings); - // Preconditions for being able to adapt down: - if (input_mode == VideoInputMode::kNoVideo) - return absl::nullopt; - // 1. We are not disabled. - // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it - // causes scaling due to bandwidth constraints (QualityScalerResource) to be - // ignored, not just CPU signals. This is not a use case we want to support - // long-term; remove this enum value. - if (degradation_preference_ == DegradationPreference::DISABLED) - return absl::nullopt; + // Don't adapt if we don't have sufficient input or adaptation is disabled. + if (input_mode == VideoInputMode::kNoVideo) { + return CannotAdaptReason::kInsufficientInput; + } + if (degradation_preference_ == DegradationPreference::DISABLED) { + return CannotAdaptReason::kAdaptationDisabled; + } bool last_adaptation_was_down = last_adaptation_request_ && last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; - // 2. We shouldn't adapt down if our frame rate is below the minimum or if its - // currently unknown. if (EffectiveDegradationPreference(input_mode) == DegradationPreference::MAINTAIN_RESOLUTION) { // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake // - delete it. if (input_fps <= 0 || (last_adaptation_was_down && input_fps < kMinFramerateFps)) { - return absl::nullopt; + return CannotAdaptReason::kInsufficientInput; } } - // 3. We shouldn't adapt down if we're currently waiting for a previous - // downgrade to have an effect. - // TODO(hbos): What about in the case of other degradation preferences? + // Don't adapt if we're awaiting a previous adaptation to have an effect. if (last_adaptation_was_down && degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && input_pixels >= last_adaptation_request_->input_pixel_count_) { - return absl::nullopt; + return CannotAdaptReason::kAwaitingPreviousAdaptation; } - // Attempt to find an allowed adaptation target. + // Maybe propose targets based on degradation preference. switch (EffectiveDegradationPreference(input_mode)) { case DegradationPreference::BALANCED: { // Try scale down framerate, if lower. @@ -419,27 +457,27 @@ VideoStreamAdapter::GetAdaptDownTarget( case DegradationPreference::MAINTAIN_FRAMERATE: { // Scale down resolution. int target_pixels = GetLowerResolutionThan(input_pixels); - // TODO(https://crbug.com/webrtc/11393): Move this logic to - // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation - // target is available should not have side-effects. - if (target_pixels < min_pixels_per_frame) - encoder_stats_observer->OnMinPixelLimitReached(); + bool min_pixel_limit_reached = target_pixels < min_pixels_per_frame; if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels, min_pixels_per_frame)) { - return absl::nullopt; + return AdaptationTargetOrReason(CannotAdaptReason::kLimitReached, + min_pixel_limit_reached); } - return AdaptationTarget(AdaptationAction::kDecreaseResolution, - target_pixels); + return AdaptationTargetOrReason( + AdaptationTarget(AdaptationAction::kDecreaseResolution, + target_pixels), + min_pixel_limit_reached); } case DegradationPreference::MAINTAIN_RESOLUTION: { int target_fps = GetLowerFrameRateThan(input_fps); - if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) - return absl::nullopt; + if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { + return CannotAdaptReason::kLimitReached; + } return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps); } case DegradationPreference::DISABLED: RTC_NOTREACHED(); - return absl::nullopt; + return CannotAdaptReason::kAdaptationDisabled; } } diff --git a/video/adaptation/video_stream_adapter.h b/video/adaptation/video_stream_adapter.h index d4d9ff25b7..16b1b44387 100644 --- a/video/adaptation/video_stream_adapter.h +++ b/video/adaptation/video_stream_adapter.h @@ -14,8 +14,8 @@ #include #include "absl/types/optional.h" +#include "absl/types/variant.h" #include "api/rtp_parameters.h" -#include "api/video/video_stream_encoder_observer.h" #include "call/adaptation/encoder_settings.h" #include "call/adaptation/resource.h" #include "call/adaptation/video_source_restrictions.h" @@ -69,6 +69,78 @@ class VideoStreamAdapter { friend class absl::optional; }; + // Reasons for not being able to get an AdaptationTarget that can be applied. + enum class CannotAdaptReason { + // DegradationPreference is DISABLED. + // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it + // causes all adaptation to be ignored, even QP-scaling. + kAdaptationDisabled, + // Adaptation is refused because we don't have video, the input frame rate + // is not known yet or is less than the minimum allowed (below the limit). + kInsufficientInput, + // The minimum or maximum adaptation has already been reached. There are no + // more steps to take. + kLimitReached, + // The resolution or frame rate requested by a recent adaptation has not yet + // been reflected in the input resolution or frame rate; adaptation is + // refused to avoid "double-adapting". + // TODO(hbos): Can this be rephrased as a resource usage measurement + // cooldown mechanism? In a multi-stream setup, we need to wait before + // adapting again across streams. The best way to achieve this is probably + // to not act on racy resource usage measurements, regardless of individual + // adapters. When this logic is moved or replaced then remove this enum + // value. + kAwaitingPreviousAdaptation, + // The adaptation that would have been proposed by the adapter violates + // bitrate constraints and is therefore rejected. + // TODO(hbos): This is a version of being resource limited, except in order + // to know if we are constrained we need to have a proposed adaptation in + // mind, thus the resource alone cannot determine this in isolation. + // Proposal: ask resources for permission to apply a proposed adaptation. + // This allows rejecting a given resolution or frame rate based on bitrate + // limits without coupling it with the adapter's proposal logic. When this + // is done, remove this enum value. + kIsBitrateConstrained, + }; + + // Describes the next adaptation target that can be applied, or a reason + // explaining why there is no next adaptation step to take. + // TODO(hbos): Make "AdaptationTarget" a private implementation detail and + // expose the resulting VideoSourceRestrictions as the publically accessible + // "target" instead. + class AdaptationTargetOrReason { + public: + AdaptationTargetOrReason(AdaptationTarget target, + bool min_pixel_limit_reached); + AdaptationTargetOrReason(CannotAdaptReason reason, + bool min_pixel_limit_reached); + // Not explicit - we want to use AdaptationTarget and CannotAdaptReason as + // return values. + AdaptationTargetOrReason(AdaptationTarget target); // NOLINT + AdaptationTargetOrReason(CannotAdaptReason reason); // NOLINT + + bool has_target() const; + const AdaptationTarget& target() const; + CannotAdaptReason reason() const; + // This is true if the next step down would have exceeded the minimum + // resolution limit. Used for stats reporting. This is similar to + // kLimitReached but only applies to resolution adaptations. It is also + // currently implemented as "the next step would have exceeded", which is + // subtly diffrent than "we are currently reaching the limit" - we could + // stay above the limit forever, not taking any steps because the steps + // would have been too big. (This is unlike how we adapt frame rate, where + // we adapt to kMinFramerateFps before reporting kLimitReached.) + // TODO(hbos): Adapt to the limit and indicate if the limit was reached + // independently of degradation preference. If stats reporting wants to + // filter this out by degradation preference it can take on that + // responsibility; the adapter should not inherit this detail. + bool min_pixel_limit_reached() const; + + private: + const absl::variant target_or_reason_; + const bool min_pixel_limit_reached_; + }; + VideoStreamAdapter(); ~VideoStreamAdapter(); @@ -77,7 +149,7 @@ class VideoStreamAdapter { // TODO(hbos): Can we get rid of any external dependencies on // BalancedDegradationPreference? How the adaptor generates possible next // steps for adaptation should be an implementation detail. Can the relevant - // information be inferred from GetAdaptUpTarget()/GetAdaptDownTarget()? + // information be inferred from AdaptationTargetOrReason? const BalancedDegradationSettings& balanced_settings() const; void ClearRestrictions(); @@ -87,24 +159,20 @@ class VideoStreamAdapter { SetDegradationPreferenceResult SetDegradationPreference( DegradationPreference degradation_preference); - // Returns a target that we are guaranteed to be able to adapt to, or null if - // adaptation is not desired or not possible. - absl::optional GetAdaptUpTarget( + // Returns a target that we are guaranteed to be able to adapt to, or the + // reason why there is no such target. + AdaptationTargetOrReason GetAdaptUpTarget( const absl::optional& encoder_settings, absl::optional encoder_target_bitrate_bps, VideoInputMode input_mode, int input_pixels, int input_fps, AdaptationObserverInterface::AdaptReason reason) const; - // TODO(https://crbug.com/webrtc/11393): Remove the dependency on - // |encoder_stats_observer| - simply checking which adaptation target is - // available should not have side-effects. - absl::optional GetAdaptDownTarget( + AdaptationTargetOrReason GetAdaptDownTarget( const absl::optional& encoder_settings, VideoInputMode input_mode, int input_pixels, - int input_fps, - VideoStreamEncoderObserver* encoder_stats_observer) const; + int input_fps) const; // Applies the |target| to |source_restrictor_|. // TODO(hbos): Delete ResourceListenerResponse! ResourceListenerResponse ApplyAdaptationTarget(