From d7d2f27b1bb58b29a11191e6df5a16d41fe2ed5a Mon Sep 17 00:00:00 2001 From: Evan Shrubsole Date: Fri, 3 Jul 2020 15:16:43 +0200 Subject: [PATCH] [Adaptation] Adaptation holds restrictions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This refactoring enables us to do multi-step adaptations instead of the normal single step adaptations. This work is required pre-requisite work to remove the DropDueToSize methods from the ResourceAdaptationProcessorInterface. This work also gives a path to fixing webrtc:11719, since double adaptations can be done in a controlled manner with this API. Bug: webrtc:11700,webrtc:11719 Change-Id: Ic7d27a49c82d0cf7ebaf79d7bdf0bc6f382770ed Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/178200 Commit-Queue: Evan Shrubsole Reviewed-by: Henrik Boström Cr-Commit-Position: refs/heads/master@{#31631} --- call/adaptation/BUILD.gn | 1 + .../resource_adaptation_processor.cc | 47 +- .../resource_adaptation_processor.h | 9 +- call/adaptation/video_stream_adapter.cc | 640 ++++++++---------- call/adaptation/video_stream_adapter.h | 154 ++--- .../video_stream_adapter_unittest.cc | 42 +- 6 files changed, 395 insertions(+), 498 deletions(-) diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index d920fd27db..084b7bd856 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -55,6 +55,7 @@ rtc_library("resource_adaptation") { absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/types:optional", + "//third_party/abseil-cpp/absl/types:variant", ] } diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/resource_adaptation_processor.cc index 3f256b15d7..277c6737c7 100644 --- a/call/adaptation/resource_adaptation_processor.cc +++ b/call/adaptation/resource_adaptation_processor.cc @@ -281,9 +281,7 @@ ResourceAdaptationProcessor::OnResourceUnderuse( } VideoSourceRestrictions restrictions_before = stream_adapter_->source_restrictions(); - VideoStreamAdapter::RestrictionsWithCounters peek_restrictions = - stream_adapter_->PeekNextRestrictions(adaptation); - VideoSourceRestrictions restrictions_after = peek_restrictions.restrictions; + VideoSourceRestrictions restrictions_after = adaptation.restrictions(); // Check that resource is most limited... std::vector> most_limited_resources; VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions; @@ -305,7 +303,7 @@ ResourceAdaptationProcessor::OnResourceUnderuse( // If the most restricted resource is less limited than current restrictions // then proceed with adapting up. if (!most_limited_resources.empty() && - most_limited_restrictions.adaptation_counters.Total() >= + most_limited_restrictions.counters.Total() >= stream_adapter_->adaptation_counters().Total()) { // If |reason_resource| is not one of the most limiting resources then abort // adaptation. @@ -322,7 +320,8 @@ ResourceAdaptationProcessor::OnResourceUnderuse( if (most_limited_resources.size() > 1) { // If there are multiple most limited resources, all must signal underuse // before the adaptation is applied. - UpdateResourceLimitations(reason_resource, peek_restrictions); + UpdateResourceLimitations(reason_resource, adaptation.restrictions(), + adaptation.counters()); processing_in_progress_ = false; rtc::StringBuilder message; message << "Resource \"" << reason_resource->Name() @@ -374,11 +373,10 @@ ResourceAdaptationProcessor::OnResourceOveruse( // Apply adaptation. VideoSourceRestrictions restrictions_before = stream_adapter_->source_restrictions(); - VideoStreamAdapter::RestrictionsWithCounters peek_next_restrictions = - stream_adapter_->PeekNextRestrictions(adaptation); - VideoSourceRestrictions restrictions_after = - peek_next_restrictions.restrictions; - stream_adapter_->ApplyAdaptation(adaptation, reason_resource); + VideoSourceRestrictions restrictions_after = adaptation.restrictions(); + UpdateResourceLimitations(reason_resource, adaptation.restrictions(), + adaptation.counters()); + stream_adapter_->ApplyAdaptation(adaptation, nullptr); for (auto* adaptation_listener : adaptation_listeners_) { adaptation_listener->OnAdaptationApplied( adaptation.input_state(), restrictions_before, restrictions_after, @@ -423,13 +421,13 @@ ResourceAdaptationProcessor::FindMostLimitedResources() const { adaptation_limits_by_resources_) { const auto& restrictions_with_counters = resource_and_adaptation_limit_.second; - if (restrictions_with_counters.adaptation_counters.Total() > - most_limited_restrictions.adaptation_counters.Total()) { + if (restrictions_with_counters.counters.Total() > + most_limited_restrictions.counters.Total()) { most_limited_restrictions = restrictions_with_counters; most_limited_resources.clear(); most_limited_resources.push_back(resource_and_adaptation_limit_.first); - } else if (most_limited_restrictions.adaptation_counters == - restrictions_with_counters.adaptation_counters) { + } else if (most_limited_restrictions.counters == + restrictions_with_counters.counters) { most_limited_resources.push_back(resource_and_adaptation_limit_.first); } } @@ -439,16 +437,18 @@ ResourceAdaptationProcessor::FindMostLimitedResources() const { void ResourceAdaptationProcessor::UpdateResourceLimitations( rtc::scoped_refptr reason_resource, - const VideoStreamAdapter::RestrictionsWithCounters& restrictions) { + const VideoSourceRestrictions& restrictions, + const VideoAdaptationCounters& counters) { auto& adaptation_limits = adaptation_limits_by_resources_[reason_resource]; - if (adaptation_limits == restrictions) { + if (adaptation_limits.restrictions == restrictions && + adaptation_limits.counters == counters) { return; } - adaptation_limits = restrictions; + adaptation_limits = {restrictions, counters}; std::map, VideoAdaptationCounters> limitations; for (const auto& p : adaptation_limits_by_resources_) { - limitations.insert(std::make_pair(p.first, p.second.adaptation_counters)); + limitations.insert(std::make_pair(p.first, p.second.counters)); } for (auto limitations_listener : resource_limitations_listeners_) { limitations_listener->OnResourceLimitationChanged(reason_resource, @@ -468,8 +468,7 @@ void ResourceAdaptationProcessor:: VideoStreamAdapter::RestrictionsWithCounters most_limited = FindMostLimitedResources().second; - if (removed_limitations.adaptation_counters.Total() <= - most_limited.adaptation_counters.Total()) { + if (removed_limitations.counters.Total() <= most_limited.counters.Total()) { // The removed limitations were less limited than the most limited resource. // Don't change the current restrictions. return; @@ -477,7 +476,7 @@ void ResourceAdaptationProcessor:: // Apply the new most limited resource as the next restrictions. Adaptation adapt_to = stream_adapter_->GetAdaptationTo( - most_limited.adaptation_counters, most_limited.restrictions); + most_limited.counters, most_limited.restrictions); RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid); stream_adapter_->ApplyAdaptation(adapt_to, nullptr); for (auto* adaptation_listener : adaptation_listeners_) { @@ -489,7 +488,7 @@ void ResourceAdaptationProcessor:: RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to " "next most limited restrictions: " << most_limited.restrictions.ToString() << " with counters " - << most_limited.adaptation_counters.ToString(); + << most_limited.counters.ToString(); } void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated( @@ -499,8 +498,8 @@ void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated( const VideoSourceRestrictions& unfiltered_restrictions) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); if (reason) { - UpdateResourceLimitations(reason, - {unfiltered_restrictions, adaptation_counters}); + UpdateResourceLimitations(reason, unfiltered_restrictions, + adaptation_counters); } else if (adaptation_counters.Total() == 0) { // Adaptations are cleared. adaptation_limits_by_resources_.clear(); diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h index 098050ddf8..05338044f3 100644 --- a/call/adaptation/resource_adaptation_processor.h +++ b/call/adaptation/resource_adaptation_processor.h @@ -22,6 +22,7 @@ #include "api/rtp_parameters.h" #include "api/scoped_refptr.h" #include "api/task_queue/task_queue_base.h" +#include "api/video/video_adaptation_counters.h" #include "api/video/video_frame.h" #include "api/video/video_stream_encoder_observer.h" #include "call/adaptation/adaptation_constraint.h" @@ -151,10 +152,10 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // changes to ensure |effective_degradation_preference_| is up-to-date. void MaybeUpdateEffectiveDegradationPreference(); - void UpdateResourceLimitations( - rtc::scoped_refptr reason_resource, - const VideoStreamAdapter::RestrictionsWithCounters& - peek_next_restrictions) RTC_RUN_ON(resource_adaptation_queue_); + void UpdateResourceLimitations(rtc::scoped_refptr reason_resource, + const VideoSourceRestrictions& restrictions, + const VideoAdaptationCounters& counters) + RTC_RUN_ON(resource_adaptation_queue_); // Searches |adaptation_limits_by_resources_| for each resource with the // highest total adaptation counts. Adaptation up may only occur if the diff --git a/call/adaptation/video_stream_adapter.cc b/call/adaptation/video_stream_adapter.cc index 94ef25d98f..39de9fb8ea 100644 --- a/call/adaptation/video_stream_adapter.cc +++ b/call/adaptation/video_stream_adapter.cc @@ -15,11 +15,17 @@ #include #include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "api/video/video_adaptation_counters.h" #include "api/video/video_adaptation_reason.h" #include "api/video_codecs/video_encoder.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" +#include "rtc_base/checks.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/synchronization/sequence_checker.h" namespace webrtc { @@ -27,13 +33,6 @@ const int kMinFrameRateFps = 2; namespace { -// Generate suggested higher and lower frame rates and resolutions, to be -// applied to the VideoSourceRestrictor. These are used in "maintain-resolution" -// and "maintain-framerate". The "balanced" degradation preference also makes -// use of BalancedDegradationPreference when generating suggestions. The -// VideoSourceRestrictor decidedes whether or not a proposed adaptation is -// valid. - // For frame rate, the steps we take are 2/3 (down) and 3/2 (up). int GetLowerFrameRateThan(int fps) { RTC_DCHECK(fps != std::numeric_limits::max()); @@ -59,6 +58,56 @@ int GetLowerResolutionThan(int pixel_count) { return (pixel_count * 3) / 5; } +int GetIncreasedMaxPixelsWanted(int target_pixels) { + if (target_pixels == std::numeric_limits::max()) + return std::numeric_limits::max(); + // When we decrease resolution, we go down to at most 3/5 of current pixels. + // Thus to increase resolution, we need 3/5 to get back to where we started. + // When going up, the desired max_pixels_per_frame() has to be significantly + // higher than the target because the source's native resolutions might not + // match the target. We pick 12/5 of the target. + // + // (This value was historically 4 times the old target, which is (3/5)*4 of + // the new target - or 12/5 - assuming the target is adjusted according to + // the above steps.) + RTC_DCHECK(target_pixels != std::numeric_limits::max()); + return (target_pixels * 12) / 5; +} + +bool CanDecreaseResolutionTo(int target_pixels, + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions) { + int max_pixels_per_frame = + rtc::dchecked_cast(restrictions.max_pixels_per_frame().value_or( + std::numeric_limits::max())); + return target_pixels < max_pixels_per_frame && + target_pixels >= input_state.min_pixels_per_frame(); +} + +bool CanIncreaseResolutionTo(int target_pixels, + const VideoSourceRestrictions& restrictions) { + int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); + int max_pixels_per_frame = + rtc::dchecked_cast(restrictions.max_pixels_per_frame().value_or( + std::numeric_limits::max())); + return max_pixels_wanted > max_pixels_per_frame; +} + +bool CanDecreaseFrameRateTo(int max_frame_rate, + const VideoSourceRestrictions& restrictions) { + const int fps_wanted = std::max(kMinFrameRateFps, max_frame_rate); + return fps_wanted < + rtc::dchecked_cast(restrictions.max_frame_rate().value_or( + std::numeric_limits::max())); +} + +bool CanIncreaseFrameRateTo(int max_frame_rate, + const VideoSourceRestrictions& restrictions) { + return max_frame_rate > + rtc::dchecked_cast(restrictions.max_frame_rate().value_or( + std::numeric_limits::max())); +} + } // namespace VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default; @@ -105,45 +154,17 @@ const char* Adaptation::StatusToString(Adaptation::Status status) { } } -Adaptation::Step::Step(StepType type, int target) : type(type), target(target) { - RTC_DCHECK_NE(type, Adaptation::StepType::kForce); -} - -Adaptation::Step::Step(VideoSourceRestrictions restrictions, - VideoAdaptationCounters counters) - : type(Adaptation::StepType::kForce), - restrictions(restrictions), - counters(counters) {} - Adaptation::Adaptation(int validation_id, - Step step, - VideoStreamInputState input_state) - : validation_id_(validation_id), - status_(Status::kValid), - step_(std::move(step)), - min_pixel_limit_reached_(false), - input_state_(input_state) {} - -Adaptation::Adaptation(int validation_id, - Step step, + VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters, VideoStreamInputState input_state, bool min_pixel_limit_reached) : validation_id_(validation_id), status_(Status::kValid), - step_(std::move(step)), min_pixel_limit_reached_(min_pixel_limit_reached), - input_state_(input_state) {} - -Adaptation::Adaptation(int validation_id, - Status invalid_status, - VideoStreamInputState input_state) - : validation_id_(validation_id), - status_(invalid_status), - step_(absl::nullopt), - min_pixel_limit_reached_(false), - input_state_(input_state) { - RTC_DCHECK_NE(status_, Status::kValid); -} + input_state_(std::move(input_state)), + restrictions_(std::move(restrictions)), + counters_(std::move(counters)) {} Adaptation::Adaptation(int validation_id, Status invalid_status, @@ -151,9 +172,8 @@ Adaptation::Adaptation(int validation_id, bool min_pixel_limit_reached) : validation_id_(validation_id), status_(invalid_status), - step_(absl::nullopt), min_pixel_limit_reached_(min_pixel_limit_reached), - input_state_(input_state) { + input_state_(std::move(input_state)) { RTC_DCHECK_NE(status_, Status::kValid); } @@ -165,193 +185,25 @@ bool Adaptation::min_pixel_limit_reached() const { return min_pixel_limit_reached_; } -const Adaptation::Step& Adaptation::step() const { - RTC_DCHECK_EQ(status_, Status::kValid); - return step_.value(); -} const VideoStreamInputState& Adaptation::input_state() const { return input_state_; } -// VideoSourceRestrictor is responsible for keeping track of current -// VideoSourceRestrictions. -class VideoStreamAdapter::VideoSourceRestrictor { - public: - VideoSourceRestrictor() {} +const VideoSourceRestrictions& Adaptation::restrictions() const { + return restrictions_; +} - VideoSourceRestrictions source_restrictions() const { - return source_restrictions_; - } - const VideoAdaptationCounters& adaptation_counters() const { - return adaptations_; - } - void ClearRestrictions() { - source_restrictions_ = VideoSourceRestrictions(); - adaptations_ = VideoAdaptationCounters(); - } - - void ForceRestrictions(const VideoSourceRestrictions& restrictions, - const VideoAdaptationCounters& counters) { - source_restrictions_ = restrictions; - adaptations_ = counters; - } - - void set_min_pixels_per_frame(int min_pixels_per_frame) { - min_pixels_per_frame_ = min_pixels_per_frame; - } - - int min_pixels_per_frame() const { return min_pixels_per_frame_; } - - bool CanDecreaseResolutionTo(int target_pixels) const { - int max_pixels_per_frame = rtc::dchecked_cast( - source_restrictions_.max_pixels_per_frame().value_or( - std::numeric_limits::max())); - return target_pixels < max_pixels_per_frame && - target_pixels >= min_pixels_per_frame_; - } - - bool CanIncreaseResolutionTo(int target_pixels) const { - int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); - int max_pixels_per_frame = rtc::dchecked_cast( - source_restrictions_.max_pixels_per_frame().value_or( - std::numeric_limits::max())); - return max_pixels_wanted > max_pixels_per_frame; - } - - bool CanDecreaseFrameRateTo(int max_frame_rate) const { - const int fps_wanted = std::max(kMinFrameRateFps, max_frame_rate); - return fps_wanted < rtc::dchecked_cast( - source_restrictions_.max_frame_rate().value_or( - std::numeric_limits::max())); - } - - bool CanIncreaseFrameRateTo(int max_frame_rate) const { - return max_frame_rate > rtc::dchecked_cast( - source_restrictions_.max_frame_rate().value_or( - std::numeric_limits::max())); - } - - void ApplyAdaptationStep(const Adaptation::Step& step, - DegradationPreference degradation_preference) { - switch (step.type) { - case Adaptation::StepType::kIncreaseResolution: - RTC_DCHECK(step.target); - IncreaseResolutionTo(step.target.value()); - break; - case Adaptation::StepType::kDecreaseResolution: - RTC_DCHECK(step.target); - DecreaseResolutionTo(step.target.value()); - break; - case Adaptation::StepType::kIncreaseFrameRate: - RTC_DCHECK(step.target); - IncreaseFrameRateTo(step.target.value()); - // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps. - // GetAdaptationUp() should tell us the correct value, but BALANCED - // logic in DecrementFramerate() makes it hard to predict whether this - // will be the last step. Remove the dependency on - // adaptation_counters(). - if (degradation_preference == DegradationPreference::BALANCED && - adaptation_counters().fps_adaptations == 0 && - step.target != std::numeric_limits::max()) { - RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - IncreaseFrameRateTo(std::numeric_limits::max()); - } - break; - case Adaptation::StepType::kDecreaseFrameRate: - RTC_DCHECK(step.target); - DecreaseFrameRateTo(step.target.value()); - break; - case Adaptation::StepType::kForce: - RTC_DCHECK(step.restrictions); - RTC_DCHECK(step.counters); - ForceRestrictions(step.restrictions.value(), step.counters.value()); - break; - } - } - - private: - static int GetIncreasedMaxPixelsWanted(int target_pixels) { - if (target_pixels == std::numeric_limits::max()) - return std::numeric_limits::max(); - // When we decrease resolution, we go down to at most 3/5 of current pixels. - // Thus to increase resolution, we need 3/5 to get back to where we started. - // When going up, the desired max_pixels_per_frame() has to be significantly - // higher than the target because the source's native resolutions might not - // match the target. We pick 12/5 of the target. - // - // (This value was historically 4 times the old target, which is (3/5)*4 of - // the new target - or 12/5 - assuming the target is adjusted according to - // the above steps.) - RTC_DCHECK(target_pixels != std::numeric_limits::max()); - return (target_pixels * 12) / 5; - } - - void DecreaseResolutionTo(int target_pixels) { - RTC_DCHECK(CanDecreaseResolutionTo(target_pixels)); - RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " - << target_pixels; - source_restrictions_.set_max_pixels_per_frame( - target_pixels != std::numeric_limits::max() - ? absl::optional(target_pixels) - : absl::nullopt); - source_restrictions_.set_target_pixels_per_frame(absl::nullopt); - ++adaptations_.resolution_adaptations; - } - - void IncreaseResolutionTo(int target_pixels) { - RTC_DCHECK(CanIncreaseResolutionTo(target_pixels)); - int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); - RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " - << max_pixels_wanted; - source_restrictions_.set_max_pixels_per_frame( - max_pixels_wanted != std::numeric_limits::max() - ? absl::optional(max_pixels_wanted) - : absl::nullopt); - source_restrictions_.set_target_pixels_per_frame( - max_pixels_wanted != std::numeric_limits::max() - ? absl::optional(target_pixels) - : absl::nullopt); - --adaptations_.resolution_adaptations; - RTC_DCHECK_GE(adaptations_.resolution_adaptations, 0); - } - - void DecreaseFrameRateTo(int max_frame_rate) { - RTC_DCHECK(CanDecreaseFrameRateTo(max_frame_rate)); - max_frame_rate = std::max(kMinFrameRateFps, max_frame_rate); - RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate; - source_restrictions_.set_max_frame_rate( - max_frame_rate != std::numeric_limits::max() - ? absl::optional(max_frame_rate) - : absl::nullopt); - ++adaptations_.fps_adaptations; - } - - void IncreaseFrameRateTo(int max_frame_rate) { - RTC_DCHECK(CanIncreaseFrameRateTo(max_frame_rate)); - RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate; - source_restrictions_.set_max_frame_rate( - max_frame_rate != std::numeric_limits::max() - ? absl::optional(max_frame_rate) - : absl::nullopt); - --adaptations_.fps_adaptations; - RTC_DCHECK_GE(adaptations_.fps_adaptations, 0); - } - - // Needed by CanDecreaseResolutionTo(). - int min_pixels_per_frame_ = 0; - // Current State. - VideoSourceRestrictions source_restrictions_; - VideoAdaptationCounters adaptations_; -}; +const VideoAdaptationCounters& Adaptation::counters() const { + return counters_; +} VideoStreamAdapter::VideoStreamAdapter( VideoStreamInputStateProvider* input_state_provider) - : source_restrictor_(std::make_unique()), - input_state_provider_(input_state_provider), + : input_state_provider_(input_state_provider), balanced_settings_(), adaptation_validation_id_(0), degradation_preference_(DegradationPreference::DISABLED), - last_adaptation_request_(absl::nullopt), + awaiting_frame_size_change_(absl::nullopt), last_video_source_restrictions_() { sequence_checker_.Detach(); } @@ -360,12 +212,12 @@ VideoStreamAdapter::~VideoStreamAdapter() {} VideoSourceRestrictions VideoStreamAdapter::source_restrictions() const { RTC_DCHECK_RUN_ON(&sequence_checker_); - return source_restrictor_->source_restrictions(); + return current_restrictions_.restrictions; } const VideoAdaptationCounters& VideoStreamAdapter::adaptation_counters() const { RTC_DCHECK_RUN_ON(&sequence_checker_); - return source_restrictor_->adaptation_counters(); + return current_restrictions_.counters; } void VideoStreamAdapter::ClearRestrictions() { @@ -373,8 +225,9 @@ void VideoStreamAdapter::ClearRestrictions() { // Invalidate any previously returned Adaptation. RTC_LOG(INFO) << "Resetting restrictions"; ++adaptation_validation_id_; - source_restrictor_->ClearRestrictions(); - last_adaptation_request_.reset(); + current_restrictions_ = {VideoSourceRestrictions(), + VideoAdaptationCounters()}; + awaiting_frame_size_change_ = absl::nullopt; BroadcastVideoRestrictionsUpdate(nullptr); } @@ -415,88 +268,88 @@ void VideoStreamAdapter::SetDegradationPreference( } } +struct VideoStreamAdapter::RestrictionsOrStateVisitor { + Adaptation operator()(const RestrictionsWithCounters& r) const { + return Adaptation(adaptation_validation_id, r.restrictions, r.counters, + input_state, min_pixel_limit_reached()); + } + Adaptation operator()(const Adaptation::Status& status) const { + RTC_DCHECK_NE(status, Adaptation::Status::kValid); + return Adaptation(adaptation_validation_id, status, input_state, + min_pixel_limit_reached()); + } + bool min_pixel_limit_reached() const { + return input_state.frame_size_pixels().has_value() && + GetLowerResolutionThan(input_state.frame_size_pixels().value()) < + input_state.min_pixels_per_frame(); + } + + const int adaptation_validation_id; + const VideoStreamInputState& input_state; +}; + +Adaptation VideoStreamAdapter::RestrictionsOrStateToAdaptation( + VideoStreamAdapter::RestrictionsOrState step_or_state, + const VideoStreamInputState& input_state) const { + RTC_DCHECK(!step_or_state.valueless_by_exception()); + return absl::visit( + RestrictionsOrStateVisitor{adaptation_validation_id_, input_state}, + step_or_state); +} + +Adaptation VideoStreamAdapter::GetAdaptationUp( + const VideoStreamInputState& input_state) const { + return RestrictionsOrStateToAdaptation(GetAdaptationUpStep(input_state), + input_state); +} + Adaptation VideoStreamAdapter::GetAdaptationUp() { RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); VideoStreamInputState input_state = input_state_provider_->InputState(); ++adaptation_validation_id_; + Adaptation adaptation = GetAdaptationUp(input_state); + return adaptation; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::GetAdaptationUpStep( + const VideoStreamInputState& input_state) const { if (!HasSufficientInputForAdaptation(input_state)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kInsufficientInput, input_state); + return Adaptation::Status::kInsufficientInput; } - source_restrictor_->set_min_pixels_per_frame( - input_state.min_pixels_per_frame()); // Don't adapt if we're awaiting a previous adaptation to have an effect. - bool last_request_increased_resolution = - last_adaptation_request_ && last_adaptation_request_->step_type_ == - Adaptation::StepType::kIncreaseResolution; - if (last_request_increased_resolution && + if (awaiting_frame_size_change_ && + awaiting_frame_size_change_->pixels_increased && degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && input_state.frame_size_pixels().value() <= - last_adaptation_request_->input_pixel_count_) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kAwaitingPreviousAdaptation, - input_state); + awaiting_frame_size_change_->frame_size_pixels) { + return Adaptation::Status::kAwaitingPreviousAdaptation; } // Maybe propose targets based on degradation preference. switch (degradation_preference_) { case DegradationPreference::BALANCED: { // Attempt to increase target frame rate. - int target_fps = - balanced_settings_.MaxFps(input_state.video_codec_type(), - input_state.frame_size_pixels().value()); - if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) { - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kIncreaseFrameRate, - target_fps), - input_state); + RestrictionsOrState increase_frame_rate = + IncreaseFramerate(input_state, current_restrictions_); + if (absl::holds_alternative( + increase_frame_rate)) { + return increase_frame_rate; } - // Scale up resolution. + // else, increase resolution. ABSL_FALLTHROUGH_INTENDED; } case DegradationPreference::MAINTAIN_FRAMERATE: { // Attempt to increase pixel count. - int target_pixels = input_state.frame_size_pixels().value(); - if (source_restrictor_->adaptation_counters().resolution_adaptations == - 1) { - RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; - target_pixels = std::numeric_limits::max(); - } - target_pixels = GetHigherResolutionThan(target_pixels); - if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached, input_state); - } - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kIncreaseResolution, - target_pixels), - input_state); + return IncreaseResolution(input_state, current_restrictions_); } case DegradationPreference::MAINTAIN_RESOLUTION: { // Scale up framerate. - int target_fps = input_state.frames_per_second(); - if (source_restrictor_->adaptation_counters().fps_adaptations == 1) { - RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - target_fps = std::numeric_limits::max(); - } - target_fps = GetHigherFrameRateThan(target_fps); - if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached, input_state); - } - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kIncreaseFrameRate, - target_fps), - input_state); + return IncreaseFramerate(input_state, current_restrictions_); } case DegradationPreference::DISABLED: RTC_NOTREACHED(); - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached, input_state); + return Adaptation::Status::kLimitReached; } } @@ -505,93 +358,167 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() { RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); VideoStreamInputState input_state = input_state_provider_->InputState(); ++adaptation_validation_id_; + return RestrictionsOrStateToAdaptation(GetAdaptationDownStep(input_state), + input_state); +} + +VideoStreamAdapter::RestrictionsOrState +VideoStreamAdapter::GetAdaptationDownStep( + const VideoStreamInputState& input_state) const { if (!HasSufficientInputForAdaptation(input_state)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kInsufficientInput, input_state); + return Adaptation::Status::kInsufficientInput; } - source_restrictor_->set_min_pixels_per_frame( - input_state.min_pixels_per_frame()); // Don't adapt if we're awaiting a previous adaptation to have an effect or // if we switched degradation preference. - bool last_request_decreased_resolution = - last_adaptation_request_ && last_adaptation_request_->step_type_ == - Adaptation::StepType::kDecreaseResolution; - if (last_request_decreased_resolution && + if (awaiting_frame_size_change_ && + !awaiting_frame_size_change_->pixels_increased && degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && input_state.frame_size_pixels().value() >= - last_adaptation_request_->input_pixel_count_) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kAwaitingPreviousAdaptation, - input_state); + awaiting_frame_size_change_->frame_size_pixels) { + return Adaptation::Status::kAwaitingPreviousAdaptation; } - // Maybe propose targets based on degradation preference. switch (degradation_preference_) { case DegradationPreference::BALANCED: { // Try scale down framerate, if lower. - int target_fps = - balanced_settings_.MinFps(input_state.video_codec_type(), - input_state.frame_size_pixels().value()); - if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kDecreaseFrameRate, - target_fps), - input_state); + RestrictionsOrState decrease_frame_rate = + DecreaseFramerate(input_state, current_restrictions_); + if (absl::holds_alternative( + decrease_frame_rate)) { + return decrease_frame_rate; } - // Scale down resolution. + // else, decrease resolution. ABSL_FALLTHROUGH_INTENDED; } case DegradationPreference::MAINTAIN_FRAMERATE: { - // Scale down resolution. - int target_pixels = - GetLowerResolutionThan(input_state.frame_size_pixels().value()); - bool min_pixel_limit_reached = - target_pixels < source_restrictor_->min_pixels_per_frame(); - if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached, input_state, - min_pixel_limit_reached); - } - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kDecreaseResolution, - target_pixels), - input_state, min_pixel_limit_reached); + return DecreaseResolution(input_state, current_restrictions_); } case DegradationPreference::MAINTAIN_RESOLUTION: { - int target_fps = GetLowerFrameRateThan(input_state.frames_per_second()); - if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached, input_state); - } - return Adaptation( - adaptation_validation_id_, - Adaptation::Step(Adaptation::StepType::kDecreaseFrameRate, - target_fps), - input_state); + return DecreaseFramerate(input_state, current_restrictions_); } case DegradationPreference::DISABLED: RTC_NOTREACHED(); - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kLimitReached, input_state); + return Adaptation::Status::kLimitReached; } } -VideoStreamAdapter::RestrictionsWithCounters -VideoStreamAdapter::PeekNextRestrictions(const Adaptation& adaptation) const { - RTC_DCHECK_RUN_ON(&sequence_checker_); - RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); - RTC_LOG(LS_INFO) << "PeekNextRestrictions called"; - if (adaptation.status() != Adaptation::Status::kValid) { - return {source_restrictor_->source_restrictions(), - source_restrictor_->adaptation_counters()}; +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) { + int target_pixels = + GetLowerResolutionThan(input_state.frame_size_pixels().value()); + if (!CanDecreaseResolutionTo(target_pixels, input_state, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; } - VideoSourceRestrictor restrictor_copy = *source_restrictor_; - restrictor_copy.ApplyAdaptationStep(adaptation.step(), - degradation_preference_); - return {restrictor_copy.source_restrictions(), - restrictor_copy.adaptation_counters()}; + RestrictionsWithCounters new_restrictions = current_restrictions; + RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " << target_pixels; + new_restrictions.restrictions.set_max_pixels_per_frame( + target_pixels != std::numeric_limits::max() + ? absl::optional(target_pixels) + : absl::nullopt); + new_restrictions.restrictions.set_target_pixels_per_frame(absl::nullopt); + ++new_restrictions.counters.resolution_adaptations; + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const { + int max_frame_rate; + if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) { + max_frame_rate = GetLowerFrameRateThan(input_state.frames_per_second()); + } else if (degradation_preference_ == DegradationPreference::BALANCED) { + max_frame_rate = + balanced_settings_.MinFps(input_state.video_codec_type(), + input_state.frame_size_pixels().value()); + } else { + RTC_NOTREACHED(); + max_frame_rate = GetLowerFrameRateThan(input_state.frames_per_second()); + } + if (!CanDecreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + RestrictionsWithCounters new_restrictions = current_restrictions; + max_frame_rate = std::max(kMinFrameRateFps, max_frame_rate); + RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate; + new_restrictions.restrictions.set_max_frame_rate( + max_frame_rate != std::numeric_limits::max() + ? absl::optional(max_frame_rate) + : absl::nullopt); + ++new_restrictions.counters.fps_adaptations; + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::IncreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) { + int target_pixels = input_state.frame_size_pixels().value(); + if (current_restrictions.counters.resolution_adaptations == 1) { + RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; + target_pixels = std::numeric_limits::max(); + } + target_pixels = GetHigherResolutionThan(target_pixels); + if (!CanIncreaseResolutionTo(target_pixels, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); + RestrictionsWithCounters new_restrictions = current_restrictions; + RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " + << max_pixels_wanted; + new_restrictions.restrictions.set_max_pixels_per_frame( + max_pixels_wanted != std::numeric_limits::max() + ? absl::optional(max_pixels_wanted) + : absl::nullopt); + new_restrictions.restrictions.set_target_pixels_per_frame( + max_pixels_wanted != std::numeric_limits::max() + ? absl::optional(target_pixels) + : absl::nullopt); + --new_restrictions.counters.resolution_adaptations; + RTC_DCHECK_GE(new_restrictions.counters.resolution_adaptations, 0); + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::IncreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const { + int max_frame_rate; + if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) { + max_frame_rate = GetHigherFrameRateThan(input_state.frames_per_second()); + } else if (degradation_preference_ == DegradationPreference::BALANCED) { + max_frame_rate = + balanced_settings_.MaxFps(input_state.video_codec_type(), + input_state.frame_size_pixels().value()); + // In BALANCED, the max_frame_rate must be checked before proceeding. This + // is because the MaxFps might be the current Fps and so the balanced + // settings may want to scale up the resolution.= + if (!CanIncreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + } else { + RTC_NOTREACHED(); + max_frame_rate = GetHigherFrameRateThan(input_state.frames_per_second()); + } + if (current_restrictions.counters.fps_adaptations == 1) { + RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; + max_frame_rate = std::numeric_limits::max(); + } + if (!CanIncreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate; + RestrictionsWithCounters new_restrictions = current_restrictions; + new_restrictions.restrictions.set_max_frame_rate( + max_frame_rate != std::numeric_limits::max() + ? absl::optional(max_frame_rate) + : absl::nullopt); + --new_restrictions.counters.fps_adaptations; + RTC_DCHECK_GE(new_restrictions.counters.fps_adaptations, 0); + return new_restrictions; } void VideoStreamAdapter::ApplyAdaptation( @@ -599,17 +526,22 @@ void VideoStreamAdapter::ApplyAdaptation( rtc::scoped_refptr resource) { RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); - RTC_LOG(LS_INFO) << "ApplyAdaptation called"; if (adaptation.status() != Adaptation::Status::kValid) return; // Remember the input pixels and fps of this adaptation. Used to avoid // adapting again before this adaptation has had an effect. - last_adaptation_request_.emplace(AdaptationRequest{ - adaptation.input_state_.frame_size_pixels().value(), - adaptation.input_state_.frames_per_second(), adaptation.step().type}); - // Adapt! - source_restrictor_->ApplyAdaptationStep(adaptation.step(), - degradation_preference_); + if (DidIncreaseResolution(current_restrictions_.restrictions, + adaptation.restrictions())) { + awaiting_frame_size_change_.emplace( + true, adaptation.input_state().frame_size_pixels().value()); + } else if (DidDecreaseResolution(current_restrictions_.restrictions, + adaptation.restrictions())) { + awaiting_frame_size_change_.emplace( + false, adaptation.input_state().frame_size_pixels().value()); + } else { + awaiting_frame_size_change_ = absl::nullopt; + } + current_restrictions_ = {adaptation.restrictions(), adaptation.counters()}; BroadcastVideoRestrictionsUpdate(resource); } @@ -619,8 +551,8 @@ Adaptation VideoStreamAdapter::GetAdaptationTo( // Adapts up/down from the current levels so counters are equal. RTC_DCHECK_RUN_ON(&sequence_checker_); VideoStreamInputState input_state = input_state_provider_->InputState(); - return Adaptation(adaptation_validation_id_, - Adaptation::Step(restrictions, counters), input_state); + return Adaptation(adaptation_validation_id_, restrictions, counters, + input_state, false); } void VideoStreamAdapter::BroadcastVideoRestrictionsUpdate( @@ -633,10 +565,10 @@ void VideoStreamAdapter::BroadcastVideoRestrictionsUpdate( } for (auto* restrictions_listener : restrictions_listeners_) { restrictions_listener->OnVideoSourceRestrictionsUpdated( - filtered, source_restrictor_->adaptation_counters(), resource, + filtered, current_restrictions_.counters, resource, source_restrictions()); } - last_video_source_restrictions_ = source_restrictor_->source_restrictions(); + last_video_source_restrictions_ = current_restrictions_.restrictions; last_filtered_restrictions_ = filtered; } @@ -648,4 +580,10 @@ bool VideoStreamAdapter::HasSufficientInputForAdaptation( input_state.frames_per_second() >= kMinFrameRateFps); } +VideoStreamAdapter::AwaitingFrameSizeChange::AwaitingFrameSizeChange( + bool pixels_increased, + int frame_size_pixels) + : pixels_increased(pixels_increased), + frame_size_pixels(frame_size_pixels) {} + } // namespace webrtc diff --git a/call/adaptation/video_stream_adapter.h b/call/adaptation/video_stream_adapter.h index df334785ec..fc88a581cb 100644 --- a/call/adaptation/video_stream_adapter.h +++ b/call/adaptation/video_stream_adapter.h @@ -12,9 +12,11 @@ #define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ #include +#include #include #include "absl/types/optional.h" +#include "absl/types/variant.h" #include "api/adaptation/resource.h" #include "api/rtp_parameters.h" #include "api/video/video_adaptation_counters.h" @@ -24,6 +26,7 @@ #include "call/adaptation/video_stream_input_state_provider.h" #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/thread_annotations.h" namespace webrtc { @@ -53,9 +56,9 @@ VideoSourceRestrictions FilterRestrictionsByDegradationPreference( int GetHigherResolutionThan(int pixel_count); -// Represents one step that the VideoStreamAdapter can take when adapting the -// VideoSourceRestrictions up or down. Or, if adaptation is not valid, provides -// a Status code indicating the reason for not adapting. +// Either represents the next VideoSourceRestrictions the VideoStreamAdapter +// will take, or provides a Status code indicating the reason for not adapting +// if the adaptation is not valid. class Adaptation final { public: enum class Status { @@ -75,69 +78,39 @@ class Adaptation final { static const char* StatusToString(Status status); - // The status of this Adaptation. To find out how this Adaptation affects - // VideoSourceRestrictions, see VideoStreamAdapter::PeekNextRestrictions(). Status status() const; + const VideoStreamInputState& input_state() const; + const VideoSourceRestrictions& restrictions() const; + const VideoAdaptationCounters& counters() const; // Used for stats reporting. bool min_pixel_limit_reached() const; - const VideoStreamInputState& input_state() const; - private: - // The adapter needs to know about step type and step target in order to - // construct and perform an Adaptation, which is a detail we do not want to - // expose to the public interface. friend class VideoStreamAdapter; - enum class StepType { - kIncreaseResolution, - kDecreaseResolution, - kIncreaseFrameRate, - kDecreaseFrameRate, - kForce - }; - - struct Step { - Step(StepType type, int target); - // StepType is kForce - Step(VideoSourceRestrictions restrictions, - VideoAdaptationCounters counters); - const StepType type; - // Pixel or frame rate depending on |type|. - // Only set when |type| is not kForce. - const absl::optional target; - // Only set when |type| is kForce. - const absl::optional restrictions; - // Only set when |type| is kForce. - const absl::optional counters; - }; - - // Constructs with a valid adaptation Step. Status is kValid. - Adaptation(int validation_id, Step step, VideoStreamInputState input_state); + // Constructs with a valid adaptation. Status is kValid. Adaptation(int validation_id, - Step step, + VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters, VideoStreamInputState input_state, bool min_pixel_limit_reached); // Constructor when adaptation is not valid. Status MUST NOT be kValid. - Adaptation(int validation_id, - Status invalid_status, - VideoStreamInputState input_state); Adaptation(int validation_id, Status invalid_status, VideoStreamInputState input_state, bool min_pixel_limit_reached); - const Step& step() const; // Only callable if |status_| is kValid. - // An Adaptation can become invalidated if the state of VideoStreamAdapter is // modified before the Adaptation is applied. To guard against this, this ID // has to match VideoStreamAdapter::adaptation_validation_id_ when applied. + // TODO(https://crbug.com/webrtc/11700): Remove the validation_id_. const int validation_id_; const Status status_; - const absl::optional step_; // Only present if |status_| is kValid. const bool min_pixel_limit_reached_; // Input state when adaptation was made. const VideoStreamInputState input_state_; + const VideoSourceRestrictions restrictions_; + const VideoAdaptationCounters counters_; }; // Owns the VideoSourceRestriction for a single stream and is responsible for @@ -173,58 +146,60 @@ class VideoStreamAdapter { Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, const VideoSourceRestrictions& restrictions); - struct RestrictionsWithCounters { - VideoSourceRestrictions restrictions; - VideoAdaptationCounters adaptation_counters; - - bool operator==(const RestrictionsWithCounters& other) { - return restrictions == other.restrictions && - adaptation_counters == other.adaptation_counters; - } - - bool operator!=(const RestrictionsWithCounters& other) { - return !(*this == other); - } - }; - - // Returns the restrictions that result from applying the adaptation, without - // actually applying it. If the adaptation is not valid, current restrictions - // are returned. - RestrictionsWithCounters PeekNextRestrictions( - const Adaptation& adaptation) const; - // Updates source_restrictions() based according to the Adaptation. These - // adaptations will be attributed to the Resource |resource| if the |resource| - // is non-null. If |resource| is null the adaptation will be changed in - // general, and thus could be adapted up in the future from other resources. + // Updates source_restrictions() the Adaptation. void ApplyAdaptation(const Adaptation& adaptation, rtc::scoped_refptr resource); - private: - class VideoSourceRestrictor; + struct RestrictionsWithCounters { + VideoSourceRestrictions restrictions; + VideoAdaptationCounters counters; + }; + private: void BroadcastVideoRestrictionsUpdate( const rtc::scoped_refptr& resource); bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); - // The input frame rate and resolution at the time of an adaptation in the - // direction described by |mode_| (up or down). - // TODO(https://crbug.com/webrtc/11393): Can this be renamed? Can this be - // merged with AdaptationTarget? - struct AdaptationRequest { - // The pixel count produced by the source at the time of the adaptation. - int input_pixel_count_; - // Framerate received from the source at the time of the adaptation. - int framerate_fps_; - // Degradation preference for the request. - Adaptation::StepType step_type_; - }; + using RestrictionsOrState = + absl::variant; + RestrictionsOrState GetAdaptationUpStep( + const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + RestrictionsOrState GetAdaptationDownStep( + const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + + Adaptation GetAdaptationUp(const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + + static RestrictionsOrState DecreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions); + static RestrictionsOrState IncreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions); + // Framerate methods are member functions because they need internal state + // if the degradation preference is BALANCED. + RestrictionsOrState DecreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const + RTC_RUN_ON(&sequence_checker_); + RestrictionsOrState IncreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const + RTC_RUN_ON(&sequence_checker_); + + struct RestrictionsOrStateVisitor; + Adaptation RestrictionsOrStateToAdaptation( + RestrictionsOrState step_or_state, + const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); SequenceChecker sequence_checker_ RTC_GUARDED_BY(&sequence_checker_); - // Owner and modifier of the VideoSourceRestriction of this stream adaptor. - const std::unique_ptr source_restrictor_ - RTC_GUARDED_BY(&sequence_checker_); // Gets the input state which is the basis of all adaptations. // Thread safe. VideoStreamInputStateProvider* input_state_provider_; @@ -238,14 +213,18 @@ class VideoStreamAdapter { // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference DegradationPreference degradation_preference_ RTC_GUARDED_BY(&sequence_checker_); - // The input frame rate, resolution and adaptation direction of the last - // ApplyAdaptationTarget(). Used to avoid adapting twice if a recent - // adaptation has not had an effect on the input frame rate or resolution yet. + // Used to avoid adapting twice. Stores the resolution at the time of the last + // adaptation. // TODO(hbos): Can we implement a more general "cooldown" mechanism of // resources intead? If we already have adapted it seems like we should wait // a while before adapting again, so that we are not acting on usage // measurements that are made obsolete/unreliable by an "ongoing" adaptation. - absl::optional last_adaptation_request_ + struct AwaitingFrameSizeChange { + AwaitingFrameSizeChange(bool pixels_increased, int frame_size); + const bool pixels_increased; + const int frame_size_pixels; + }; + absl::optional awaiting_frame_size_change_ RTC_GUARDED_BY(&sequence_checker_); // The previous restrictions value. Starts as unrestricted. VideoSourceRestrictions last_video_source_restrictions_ @@ -255,6 +234,9 @@ class VideoStreamAdapter { std::vector restrictions_listeners_ RTC_GUARDED_BY(&sequence_checker_); + + RestrictionsWithCounters current_restrictions_ + RTC_GUARDED_BY(&sequence_checker_); }; } // namespace webrtc diff --git a/call/adaptation/video_stream_adapter_unittest.cc b/call/adaptation/video_stream_adapter_unittest.cc index 87e1f67501..1df5a54fd0 100644 --- a/call/adaptation/video_stream_adapter_unittest.cc +++ b/call/adaptation/video_stream_adapter_unittest.cc @@ -719,11 +719,9 @@ TEST_F(VideoStreamAdapterTest, RestrictionBroadcasted) { // Broadcast on ApplyAdaptation. { Adaptation adaptation = adapter_.GetAdaptationDown(); - VideoStreamAdapter::RestrictionsWithCounters peek = - adapter_.PeekNextRestrictions(adaptation); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(1, listener.calls()); - EXPECT_EQ(peek.restrictions, listener.last_restrictions()); + EXPECT_EQ(adaptation.restrictions(), listener.last_restrictions()); } // Broadcast on ClearRestrictions(). @@ -732,7 +730,7 @@ TEST_F(VideoStreamAdapterTest, RestrictionBroadcasted) { EXPECT_EQ(VideoSourceRestrictions(), listener.last_restrictions()); } -TEST_F(VideoStreamAdapterTest, PeekNextRestrictions) { +TEST_F(VideoStreamAdapterTest, AdaptationHasNextRestrcitions) { // Any non-disabled DegradationPreference will do. adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, @@ -741,49 +739,27 @@ TEST_F(VideoStreamAdapterTest, PeekNextRestrictions) { { Adaptation adaptation = adapter_.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kLimitReached, adaptation.status()); - VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters = - adapter_.PeekNextRestrictions(adaptation); - EXPECT_EQ(restrictions_with_counters.restrictions, - adapter_.source_restrictions()); - EXPECT_EQ(0, restrictions_with_counters.adaptation_counters.Total()); + EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions()); + EXPECT_EQ(0, adaptation.counters().Total()); } // When we adapt down. { Adaptation adaptation = adapter_.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); - VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters = - adapter_.PeekNextRestrictions(adaptation); fake_stream.ApplyAdaptation(adaptation); - EXPECT_EQ(restrictions_with_counters.restrictions, - adapter_.source_restrictions()); - EXPECT_EQ(restrictions_with_counters.adaptation_counters, - adapter_.adaptation_counters()); + EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions()); + EXPECT_EQ(adaptation.counters(), adapter_.adaptation_counters()); } // When we adapt up. { Adaptation adaptation = adapter_.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); - VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters = - adapter_.PeekNextRestrictions(adaptation); fake_stream.ApplyAdaptation(adaptation); - EXPECT_EQ(restrictions_with_counters.restrictions, - adapter_.source_restrictions()); - EXPECT_EQ(restrictions_with_counters.adaptation_counters, - adapter_.adaptation_counters()); + EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions()); + EXPECT_EQ(adaptation.counters(), adapter_.adaptation_counters()); } } -TEST_F(VideoStreamAdapterTest, PeekRestrictionsDoesNotBroadcast) { - FakeVideoStreamAdapterListner listener; - adapter_.AddRestrictionsListener(&listener); - adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, - kDefaultMinPixelsPerFrame); - Adaptation adaptation = adapter_.GetAdaptationDown(); - adapter_.PeekNextRestrictions(adaptation); - EXPECT_EQ(0, listener.calls()); -} - TEST_F(VideoStreamAdapterTest, SetDegradationPreferenceToOrFromBalancedClearsRestrictions) { adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); @@ -829,7 +805,7 @@ TEST(VideoStreamAdapterDeathTest, AdaptDownInvalidatesAdaptations) { input_state_provider.SetInputState(1280 * 720, 30, kDefaultMinPixelsPerFrame); Adaptation adaptation = adapter.GetAdaptationDown(); adapter.GetAdaptationDown(); - EXPECT_DEATH(adapter.PeekNextRestrictions(adaptation), ""); + EXPECT_DEATH(adapter.ApplyAdaptation(adaptation, nullptr), ""); } #endif // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)