diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index 24dee6a0f0..94656cf0a9 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -14,6 +14,8 @@ rtc_library("resource_adaptation") { "encoder_settings.h", "resource.cc", "resource.h", + "resource_adaptation_processor.cc", + "resource_adaptation_processor.h", "resource_adaptation_processor_interface.cc", "resource_adaptation_processor_interface.h", "video_source_restrictions.cc", diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/resource_adaptation_processor.cc new file mode 100644 index 0000000000..d75f62b2f4 --- /dev/null +++ b/call/adaptation/resource_adaptation_processor.cc @@ -0,0 +1,209 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/resource_adaptation_processor.h" + +#include + +#include "absl/algorithm/container.h" + +namespace webrtc { + +ResourceAdaptationProcessor::ResourceAdaptationProcessor( + VideoStreamInputStateProvider* input_state_provider, + VideoStreamEncoderObserver* encoder_stats_observer) + : input_state_provider_(input_state_provider), + encoder_stats_observer_(encoder_stats_observer), + resources_(), + degradation_preference_(DegradationPreference::DISABLED), + effective_degradation_preference_(DegradationPreference::DISABLED), + is_screenshare_(false), + stream_adapter_(std::make_unique()), + last_reported_source_restrictions_() {} + +ResourceAdaptationProcessor::~ResourceAdaptationProcessor() {} + +DegradationPreference ResourceAdaptationProcessor::degradation_preference() + const { + return degradation_preference_; +} + +DegradationPreference +ResourceAdaptationProcessor::effective_degradation_preference() const { + return effective_degradation_preference_; +} + +void ResourceAdaptationProcessor::StartResourceAdaptation() { + for (auto* resource : resources_) { + resource->RegisterListener(this); + } +} + +void ResourceAdaptationProcessor::StopResourceAdaptation() { + for (auto* resource : resources_) { + resource->UnregisterListener(this); + } +} + +void ResourceAdaptationProcessor::AddAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) { + adaptation_listeners_.push_back(adaptation_listener); +} + +void ResourceAdaptationProcessor::AddResource(Resource* resource) { + resources_.push_back(resource); +} + +void ResourceAdaptationProcessor::SetDegradationPreference( + DegradationPreference degradation_preference) { + degradation_preference_ = degradation_preference; + MaybeUpdateEffectiveDegradationPreference(); +} + +void ResourceAdaptationProcessor::SetIsScreenshare(bool is_screenshare) { + is_screenshare_ = is_screenshare; + MaybeUpdateEffectiveDegradationPreference(); +} + +void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { + effective_degradation_preference_ = + (is_screenshare_ && + degradation_preference_ == DegradationPreference::BALANCED) + ? DegradationPreference::MAINTAIN_RESOLUTION + : degradation_preference_; + stream_adapter_->SetDegradationPreference(effective_degradation_preference_); + MaybeUpdateVideoSourceRestrictions(nullptr); +} + +void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { + stream_adapter_->ClearRestrictions(); + MaybeUpdateVideoSourceRestrictions(nullptr); +} + +void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions( + const Resource* reason) { + VideoSourceRestrictions new_soure_restrictions = + FilterRestrictionsByDegradationPreference( + stream_adapter_->source_restrictions(), + effective_degradation_preference_); + if (last_reported_source_restrictions_ != new_soure_restrictions) { + last_reported_source_restrictions_ = std::move(new_soure_restrictions); + for (auto* adaptation_listener : adaptation_listeners_) { + adaptation_listener->OnVideoSourceRestrictionsUpdated( + last_reported_source_restrictions_, + stream_adapter_->adaptation_counters(), reason); + } + } +} + +ResourceListenerResponse +ResourceAdaptationProcessor::OnResourceUsageStateMeasured( + const Resource& resource) { + switch (resource.usage_state()) { + case ResourceUsageState::kOveruse: + return OnResourceOveruse(resource); + case ResourceUsageState::kStable: + // TODO(https://crbug.com/webrtc/11172): Delete kStable in favor of null. + RTC_NOTREACHED(); + return ResourceListenerResponse::kNothing; + case ResourceUsageState::kUnderuse: + OnResourceUnderuse(resource); + return ResourceListenerResponse::kNothing; + } +} + +bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation( + const VideoStreamInputState& input_state) const { + return input_state.HasInputFrameSizeAndFramesPerSecond() && + (effective_degradation_preference_ != + DegradationPreference::MAINTAIN_RESOLUTION || + input_state.frames_per_second() >= kMinFrameRateFps); +} + +void ResourceAdaptationProcessor::OnResourceUnderuse( + const Resource& reason_resource) { + VideoStreamInputState input_state = input_state_provider_->InputState(); + if (effective_degradation_preference_ == DegradationPreference::DISABLED || + !HasSufficientInputForAdaptation(input_state)) { + return; + } + // Update video input states and encoder settings for accurate adaptation. + stream_adapter_->SetInput(input_state); + // How can this stream be adapted up? + Adaptation adaptation = stream_adapter_->GetAdaptationUp(); + if (adaptation.status() != Adaptation::Status::kValid) + return; + // Are all resources OK with this adaptation being applied? + VideoSourceRestrictions restrictions_before = + stream_adapter_->source_restrictions(); + VideoSourceRestrictions restrictions_after = + stream_adapter_->PeekNextRestrictions(adaptation); + if (!absl::c_all_of(resources_, [&input_state, &restrictions_before, + &restrictions_after, + &reason_resource](const Resource* resource) { + return resource->IsAdaptationUpAllowed(input_state, restrictions_before, + restrictions_after, + reason_resource); + })) { + return; + } + // Apply adaptation. + stream_adapter_->ApplyAdaptation(adaptation); + // Update VideoSourceRestrictions based on adaptation. This also informs the + // |adaptation_listeners_|. + MaybeUpdateVideoSourceRestrictions(&reason_resource); +} + +ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse( + const Resource& reason_resource) { + VideoStreamInputState input_state = input_state_provider_->InputState(); + if (!input_state.has_input()) { + return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; + } + if (effective_degradation_preference_ == DegradationPreference::DISABLED || + !HasSufficientInputForAdaptation(input_state)) { + return ResourceListenerResponse::kNothing; + } + // Update video input states and encoder settings for accurate adaptation. + stream_adapter_->SetInput(input_state); + // How can this stream be adapted up? + Adaptation adaptation = stream_adapter_->GetAdaptationDown(); + if (adaptation.min_pixel_limit_reached()) + encoder_stats_observer_->OnMinPixelLimitReached(); + if (adaptation.status() != Adaptation::Status::kValid) + return ResourceListenerResponse::kNothing; + // Apply adaptation. + ResourceListenerResponse response = + stream_adapter_->ApplyAdaptation(adaptation); + // Update VideoSourceRestrictions based on adaptation. This also informs the + // |adaptation_listeners_|. + MaybeUpdateVideoSourceRestrictions(&reason_resource); + return response; +} + +void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize( + const Resource& reason_resource) { + VideoAdaptationCounters counters_before = + stream_adapter_->adaptation_counters(); + OnResourceOveruse(reason_resource); + if (degradation_preference_ == DegradationPreference::BALANCED && + stream_adapter_->adaptation_counters().fps_adaptations > + counters_before.fps_adaptations) { + // Oops, we adapted frame rate. Adapt again, maybe it will adapt resolution! + // Though this is not guaranteed... + OnResourceOveruse(reason_resource); + } + if (stream_adapter_->adaptation_counters().resolution_adaptations > + counters_before.resolution_adaptations) { + encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); + } +} + +} // namespace webrtc diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h new file mode 100644 index 0000000000..e877c82502 --- /dev/null +++ b/call/adaptation/resource_adaptation_processor.h @@ -0,0 +1,100 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ +#define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/rtp_parameters.h" +#include "api/video/video_frame.h" +#include "api/video/video_stream_encoder_observer.h" +#include "call/adaptation/resource.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_adapter.h" +#include "call/adaptation/video_stream_input_state.h" +#include "call/adaptation/video_stream_input_state_provider.h" + +namespace webrtc { + +class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, + public ResourceListener { + public: + ResourceAdaptationProcessor( + VideoStreamInputStateProvider* input_state_provider, + VideoStreamEncoderObserver* encoder_stats_observer); + ~ResourceAdaptationProcessor() override; + + // ResourceAdaptationProcessorInterface implementation. + DegradationPreference degradation_preference() const override; + DegradationPreference effective_degradation_preference() const override; + + void StartResourceAdaptation() override; + void StopResourceAdaptation() override; + void AddAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) override; + void AddResource(Resource* resource) override; + + void SetDegradationPreference( + DegradationPreference degradation_preference) override; + void SetIsScreenshare(bool is_screenshare) override; + void ResetVideoSourceRestrictions() override; + + // ResourceListener implementation. + // Triggers OnResourceUnderuse() or OnResourceOveruse(). + ResourceListenerResponse OnResourceUsageStateMeasured( + const Resource& resource) override; + + // May trigger 1-2 adaptations. It is meant to reduce resolution - useful if a + // frame was dropped due to its size - but if you look at the implementation + // you'll find that this is not guaranteed. It could adapt frame rate, which + // does not address the issue. + // TODO(hbos): Can we replace this by something which actually satisfies the + // resolution constraints, or get rid of it altogether? + void TriggerAdaptationDueToFrameDroppedDueToSize( + const Resource& reason_resource); + + private: + bool HasSufficientInputForAdaptation( + const VideoStreamInputState& input_state) const; + + // Performs the adaptation by getting the next target, applying it and + // informing listeners of the new VideoSourceRestriction and adaptation + // counters. + void OnResourceUnderuse(const Resource& reason_resource); + ResourceListenerResponse OnResourceOveruse(const Resource& reason_resource); + + // Needs to be invoked any time |degradation_preference_| or |is_screenshare_| + // changes to ensure |effective_degradation_preference_| is up-to-date. + void MaybeUpdateEffectiveDegradationPreference(); + // If the filtered source restrictions are different than + // |last_reported_source_restrictions_|, inform the listeners. + void MaybeUpdateVideoSourceRestrictions(const Resource* reason); + + // Input and output. + VideoStreamInputStateProvider* const input_state_provider_; + VideoStreamEncoderObserver* const encoder_stats_observer_; + std::vector adaptation_listeners_; + std::vector resources_; + // Adaptation strategy settings. + DegradationPreference degradation_preference_; + DegradationPreference effective_degradation_preference_; + bool is_screenshare_; + // Responsible for generating and applying possible adaptations. + const std::unique_ptr stream_adapter_; + VideoSourceRestrictions last_reported_source_restrictions_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h index 5d84ffa088..89ad903b7d 100644 --- a/call/adaptation/resource_adaptation_processor_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h @@ -43,13 +43,29 @@ class ResourceAdaptationProcessorInterface { public: virtual ~ResourceAdaptationProcessorInterface(); - virtual void StartResourceAdaptation( - ResourceAdaptationProcessorListener* adaptation_listener) = 0; + virtual DegradationPreference degradation_preference() const = 0; + // Reinterprets "balanced + screenshare" as "maintain-resolution". + // TODO(hbos): Don't do this. This is not what "balanced" means. If the + // application wants to maintain resolution it should set that degradation + // preference rather than depend on non-standard behaviors. + virtual DegradationPreference effective_degradation_preference() const = 0; + + // Starts or stops listening to resources, effectively enabling or disabling + // processing. + // TODO(https://crbug.com/webrtc/11172): Automatically register and unregister + // with AddResource() and RemoveResource() instead. When the processor is + // multi-stream aware, stream-specific resouces will get added and removed + // over time. + virtual void StartResourceAdaptation() = 0; virtual void StopResourceAdaptation() = 0; - // The resource must out-live the module. + virtual void AddAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) = 0; virtual void AddResource(Resource* resource) = 0; + virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; + virtual void SetIsScreenshare(bool is_screenshare) = 0; + virtual void ResetVideoSourceRestrictions() = 0; }; } // namespace webrtc diff --git a/video/adaptation/encode_usage_resource.h b/video/adaptation/encode_usage_resource.h index 6c1827db95..a41211ee03 100644 --- a/video/adaptation/encode_usage_resource.h +++ b/video/adaptation/encode_usage_resource.h @@ -35,6 +35,8 @@ class EncodeUsageResource : public Resource, explicit EncodeUsageResource( std::unique_ptr overuse_detector); + bool is_started() const { return is_started_; } + void StartCheckForOveruse(CpuOveruseOptions options); void StopCheckForOveruse(); diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index f219d3789e..d2a695e67c 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -157,7 +157,7 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts:: // have this catch-all reason- and stats-based approach. int num_downgrades = FilterVideoAdaptationCountersByDegradationPreference( manager_->active_counts_[reason], - manager_->effective_degradation_preference()) + manager_->effective_degradation_preference_) .Total(); RTC_DCHECK_GE(num_downgrades, 0); return num_downgrades > 0; @@ -219,9 +219,9 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: // TODO(hbos): Why are we allowing violating balanced settings if adapting due // CPU? Shouldn't this condition be checked regardless of reason? if (reason == VideoAdaptationReason::kQuality && - manager_->effective_degradation_preference() == + manager_->effective_degradation_preference_ == DegradationPreference::BALANCED && - !manager_->stream_adapter_->balanced_settings().CanAdaptUp( + !manager_->balanced_settings_.CanAdaptUp( input_state.video_codec_type(), input_state.frame_size_pixels().value(), manager_->encoder_target_bitrate_bps_.value_or(0))) { @@ -229,7 +229,7 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: } if (reason == VideoAdaptationReason::kQuality && DidIncreaseResolution(restrictions_before, restrictions_after) && - !manager_->stream_adapter_->balanced_settings().CanAdaptUpResolution( + !manager_->balanced_settings_.CanAdaptUpResolution( input_state.video_codec_type(), input_state.frame_size_pixels().value(), manager_->encoder_target_bitrate_bps_.value_or(0))) { @@ -240,24 +240,24 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, + ResourceAdaptationProcessor* adaptation_processor, + VideoStreamEncoderObserver* encoder_stats_observer, Clock* clock, bool experiment_cpu_load_estimator, - std::unique_ptr overuse_detector, - VideoStreamEncoderObserver* encoder_stats_observer, - ResourceAdaptationProcessorListener* adaptation_listener) + std::unique_ptr overuse_detector) : prevent_adapt_up_due_to_active_counts_(this), prevent_increase_resolution_due_to_bitrate_resource_(this), prevent_adapt_up_in_balanced_resource_(this), encode_usage_resource_(std::move(overuse_detector)), quality_scaler_resource_(), input_state_provider_(input_state_provider), - adaptation_listener_(adaptation_listener), - clock_(clock), - state_(State::kStopped), - experiment_cpu_load_estimator_(experiment_cpu_load_estimator), + adaptation_processor_(adaptation_processor), + encoder_stats_observer_(encoder_stats_observer), degradation_preference_(DegradationPreference::DISABLED), effective_degradation_preference_(DegradationPreference::DISABLED), - stream_adapter_(std::make_unique()), + video_source_restrictions_(), + clock_(clock), + experiment_cpu_load_estimator_(experiment_cpu_load_estimator), initial_frame_dropper_( std::make_unique(&quality_scaler_resource_)), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), @@ -265,53 +265,43 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( quality_rampup_done_(false), quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), encoder_settings_(absl::nullopt), - encoder_stats_observer_(encoder_stats_observer), active_counts_() { - RTC_DCHECK(adaptation_listener_); RTC_DCHECK(encoder_stats_observer_); - AddResource(&prevent_adapt_up_due_to_active_counts_, - VideoAdaptationReason::kQuality); - AddResource(&prevent_increase_resolution_due_to_bitrate_resource_, - VideoAdaptationReason::kQuality); - AddResource(&prevent_adapt_up_in_balanced_resource_, - VideoAdaptationReason::kQuality); - AddResource(&encode_usage_resource_, VideoAdaptationReason::kCpu); - AddResource(&quality_scaler_resource_, VideoAdaptationReason::kQuality); + MapResourceToReason(&prevent_adapt_up_due_to_active_counts_, + VideoAdaptationReason::kQuality); + MapResourceToReason(&prevent_increase_resolution_due_to_bitrate_resource_, + VideoAdaptationReason::kQuality); + MapResourceToReason(&prevent_adapt_up_in_balanced_resource_, + VideoAdaptationReason::kQuality); + MapResourceToReason(&encode_usage_resource_, VideoAdaptationReason::kCpu); + MapResourceToReason(&quality_scaler_resource_, + VideoAdaptationReason::kQuality); } VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() { - RTC_DCHECK_EQ(state_, State::kStopped); + RTC_DCHECK(!encode_usage_resource_.is_started()); } -void VideoStreamEncoderResourceManager::StartResourceAdaptation( - ResourceAdaptationProcessorListener* adaptation_listener) { - RTC_DCHECK_EQ(state_, State::kStopped); +void VideoStreamEncoderResourceManager::SetDegradationPreferences( + DegradationPreference degradation_preference, + DegradationPreference effective_degradation_preference) { + degradation_preference_ = degradation_preference; + effective_degradation_preference_ = effective_degradation_preference; + UpdateStatsAdaptationSettings(); +} + +void VideoStreamEncoderResourceManager::StartEncodeUsageResource() { + RTC_DCHECK(!encode_usage_resource_.is_started()); RTC_DCHECK(encoder_settings_.has_value()); - // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener - // should be passed in and why. If resources are separated from modules then - // those resources may be started or stopped separately from the module. - RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); encode_usage_resource_.StartCheckForOveruse(GetCpuOveruseOptions()); - for (auto& resource_and_reason : resources_) { - resource_and_reason.resource->RegisterListener(this); - } - state_ = State::kStarted; } -void VideoStreamEncoderResourceManager::StopResourceAdaptation() { +void VideoStreamEncoderResourceManager::StopManagedResources() { encode_usage_resource_.StopCheckForOveruse(); quality_scaler_resource_.StopCheckForOveruse(); - for (auto& resource_and_reason : resources_) { - resource_and_reason.resource->UnregisterListener(this); - } - state_ = State::kStopped; } -void VideoStreamEncoderResourceManager::AddResource(Resource* resource) { - return AddResource(resource, VideoAdaptationReason::kCpu); -} - -void VideoStreamEncoderResourceManager::AddResource( +void VideoStreamEncoderResourceManager::MapResourceToReason( Resource* resource, VideoAdaptationReason reason) { RTC_DCHECK(resource); @@ -323,17 +313,18 @@ void VideoStreamEncoderResourceManager::AddResource( resources_.emplace_back(resource, reason); } -void VideoStreamEncoderResourceManager::SetDegradationPreference( - DegradationPreference degradation_preference) { - degradation_preference_ = degradation_preference; - UpdateStatsAdaptationSettings(); - MaybeUpdateEffectiveDegradationPreference(); +std::vector VideoStreamEncoderResourceManager::MappedResources() + const { + std::vector resources; + for (auto const& resource_and_reason : resources_) { + resources.push_back(resource_and_reason.resource); + } + return resources; } void VideoStreamEncoderResourceManager::SetEncoderSettings( EncoderSettings encoder_settings) { encoder_settings_ = std::move(encoder_settings); - MaybeUpdateEffectiveDegradationPreference(); quality_rampup_experiment_.SetMaxBitrate( LastInputFrameSizeOrDefault(), @@ -362,25 +353,9 @@ void VideoStreamEncoderResourceManager::SetEncoderRates( encoder_rates_ = encoder_rates; } -void VideoStreamEncoderResourceManager::ResetVideoSourceRestrictions() { - stream_adapter_->ClearRestrictions(); - MaybeUpdateVideoSourceRestrictions(nullptr); -} - void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() { - VideoAdaptationCounters counters_before = - stream_adapter_->adaptation_counters(); - OnResourceOveruse(quality_scaler_resource_); - if (degradation_preference_ == DegradationPreference::BALANCED && - stream_adapter_->adaptation_counters().fps_adaptations > - counters_before.fps_adaptations) { - // Adapt framerate in same step as resolution. - OnResourceOveruse(quality_scaler_resource_); - } - if (stream_adapter_->adaptation_counters().resolution_adaptations > - counters_before.resolution_adaptations) { - encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); - } + adaptation_processor_->TriggerAdaptationDueToFrameDroppedDueToSize( + quality_scaler_resource_); initial_frame_dropper_->OnFrameDroppedDueToSize(); } @@ -461,7 +436,7 @@ void VideoStreamEncoderResourceManager::ConfigureQualityScaler( if (degradation_preference_ == DegradationPreference::BALANCED && quality_scaler_resource_.is_started()) { absl::optional thresholds = - stream_adapter_->balanced_settings().GetQpThresholds( + balanced_settings_.GetQpThresholds( GetVideoCodecTypeOrGeneric(encoder_settings_), LastInputFrameSizeOrDefault()); if (thresholds) { @@ -471,30 +446,6 @@ void VideoStreamEncoderResourceManager::ConfigureQualityScaler( UpdateStatsAdaptationSettings(); } -ResourceListenerResponse -VideoStreamEncoderResourceManager::OnResourceUsageStateMeasured( - const Resource& resource) { - switch (resource.usage_state()) { - case ResourceUsageState::kOveruse: - return OnResourceOveruse(resource); - case ResourceUsageState::kStable: - // Do nothing. - // TODO(https://crbug.com/webrtc/11172): Delete kStable in favor of null. - return ResourceListenerResponse::kNothing; - case ResourceUsageState::kUnderuse: - OnResourceUnderuse(resource); - return ResourceListenerResponse::kNothing; - } -} - -bool VideoStreamEncoderResourceManager::HasSufficientInputForAdaptation( - const VideoStreamInputState& input_state) const { - return input_state.HasInputFrameSizeAndFramesPerSecond() && - (effective_degradation_preference_ != - DegradationPreference::MAINTAIN_RESOLUTION || - input_state.frames_per_second() >= kMinFrameRateFps); -} - VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource( const Resource& resource) const { const auto& registered_resource = @@ -506,67 +457,6 @@ VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource( return registered_resource->reason; } -void VideoStreamEncoderResourceManager::OnResourceUnderuse( - const Resource& reason_resource) { - VideoStreamInputState input_state = input_state_provider_->InputState(); - if (effective_degradation_preference_ == DegradationPreference::DISABLED || - !HasSufficientInputForAdaptation(input_state)) { - return; - } - // Update video input states and encoder settings for accurate adaptation. - stream_adapter_->SetInput(input_state); - // How can this stream be adapted up? - Adaptation adaptation = stream_adapter_->GetAdaptationUp(); - if (adaptation.status() != Adaptation::Status::kValid) - return; - // Are all resources OK with this adaptation being applied? - VideoSourceRestrictions restrictions_before = - stream_adapter_->source_restrictions(); - VideoSourceRestrictions restrictions_after = - stream_adapter_->PeekNextRestrictions(adaptation); - if (!absl::c_all_of(resources_, [&input_state, &restrictions_before, - &restrictions_after, &reason_resource]( - ResourceAndReason resource_and_reason) { - return resource_and_reason.resource->IsAdaptationUpAllowed( - input_state, restrictions_before, restrictions_after, - reason_resource); - })) { - return; - } - // Apply adaptation. - stream_adapter_->ApplyAdaptation(adaptation); - // Update VideoSourceRestrictions based on adaptation. This also informs the - // |adaptation_listener_|. - MaybeUpdateVideoSourceRestrictions(&reason_resource); -} - -ResourceListenerResponse VideoStreamEncoderResourceManager::OnResourceOveruse( - const Resource& reason_resource) { - VideoStreamInputState input_state = input_state_provider_->InputState(); - if (!input_state.has_input()) { - return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; - } - if (effective_degradation_preference_ == DegradationPreference::DISABLED || - !HasSufficientInputForAdaptation(input_state)) { - return ResourceListenerResponse::kNothing; - } - // Update video input states and encoder settings for accurate adaptation. - stream_adapter_->SetInput(input_state); - // How can this stream be adapted down? - Adaptation adaptation = stream_adapter_->GetAdaptationDown(); - if (adaptation.min_pixel_limit_reached()) - encoder_stats_observer_->OnMinPixelLimitReached(); - if (adaptation.status() != Adaptation::Status::kValid) - return ResourceListenerResponse::kNothing; - // Apply adaptation. - ResourceListenerResponse response = - stream_adapter_->ApplyAdaptation(adaptation); - // Update VideoSourceRestrictions based on adaptation. This also informs the - // |adaptation_listener_|. - MaybeUpdateVideoSourceRestrictions(&reason_resource); - return response; -} - // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle // pipelining encoders better (multiple input frames before something comes // out). This should effectively turn off CPU adaptations for systems that @@ -594,43 +484,11 @@ int VideoStreamEncoderResourceManager::LastInputFrameSizeOrDefault() const { kDefaultInputPixelsWidth * kDefaultInputPixelsHeight); } -void VideoStreamEncoderResourceManager:: - MaybeUpdateEffectiveDegradationPreference() { - bool is_screenshare = encoder_settings_.has_value() && - encoder_settings_->encoder_config().content_type == - VideoEncoderConfig::ContentType::kScreen; - effective_degradation_preference_ = - (is_screenshare && - degradation_preference_ == DegradationPreference::BALANCED) - ? DegradationPreference::MAINTAIN_RESOLUTION - : degradation_preference_; - stream_adapter_->SetDegradationPreference(effective_degradation_preference_); - MaybeUpdateVideoSourceRestrictions(nullptr); -} - -void VideoStreamEncoderResourceManager::MaybeUpdateVideoSourceRestrictions( - const Resource* reason_resource) { - VideoSourceRestrictions new_restrictions = - FilterRestrictionsByDegradationPreference( - stream_adapter_->source_restrictions(), degradation_preference_); - if (video_source_restrictions_ != new_restrictions) { - video_source_restrictions_ = std::move(new_restrictions); - // TODO(https://crbug.com/webrtc/11172): Support multiple listeners and - // loop through them here instead of calling two hardcoded listeners (|this| - // and |adaptation_listener_|). - OnVideoSourceRestrictionsUpdated(video_source_restrictions_, - stream_adapter_->adaptation_counters(), - reason_resource); - adaptation_listener_->OnVideoSourceRestrictionsUpdated( - video_source_restrictions_, stream_adapter_->adaptation_counters(), - reason_resource); - } -} - void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, const Resource* reason) { + video_source_restrictions_ = restrictions; VideoAdaptationCounters previous_adaptation_counters = active_counts_[VideoAdaptationReason::kQuality] + active_counts_[VideoAdaptationReason::kCpu]; @@ -798,7 +656,7 @@ void VideoStreamEncoderResourceManager::MaybePerformQualityRampupExperiment() { if (try_quality_rampup && qp_counts.resolution_adaptations > 0 && cpu_counts.Total() == 0) { RTC_LOG(LS_INFO) << "Reset quality limitations."; - ResetVideoSourceRestrictions(); + adaptation_processor_->ResetVideoSourceRestrictions(); quality_rampup_done_ = true; } } diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index d99b6ec56a..d07c31b312 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -29,6 +29,7 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_config.h" #include "call/adaptation/resource.h" +#include "call/adaptation/resource_adaptation_processor.h" #include "call/adaptation/resource_adaptation_processor_interface.h" #include "call/adaptation/video_stream_adapter.h" #include "call/adaptation/video_stream_input_state_provider.h" @@ -48,48 +49,35 @@ namespace webrtc { extern const int kDefaultInputPixelsWidth; extern const int kDefaultInputPixelsHeight; -// This class is used by the VideoStreamEncoder and is responsible for adapting -// resolution up or down based on encode usage percent. It keeps track of video -// source settings, adaptation counters and may get influenced by -// VideoStreamEncoder's quality scaler through AdaptUp() and AdaptDown() calls. +// Owns adaptation-related Resources pertaining to a single VideoStreamEncoder +// and passes on the relevant input from the encoder to the resources. The +// resources provide resource usage states to the ResourceAdaptationProcessor +// which is responsible for reconfiguring streams in order not to overuse +// resources. // -// This class is single-threaded. The caller is responsible for ensuring safe -// usage. -// TODO(hbos): Add unittests specific to this class, it is currently only tested -// indirectly in video_stream_encoder_unittest.cc and other tests exercising -// VideoStreamEncoder. +// The manager is also involved with various mitigations not part of the +// ResourceAdaptationProcessor code such as the inital frame dropping. class VideoStreamEncoderResourceManager - : public ResourceAdaptationProcessorInterface, - public ResourceListener, - public ResourceAdaptationProcessorListener { + : public ResourceAdaptationProcessorListener { public: - // The processor can be constructed on any sequence, but must be initialized - // and used on a single sequence, e.g. the encoder queue. VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, + ResourceAdaptationProcessor* adaptation_processor, + VideoStreamEncoderObserver* encoder_stats_observer, Clock* clock, bool experiment_cpu_load_estimator, - std::unique_ptr overuse_detector, - VideoStreamEncoderObserver* encoder_stats_observer, - ResourceAdaptationProcessorListener* adaptation_listener); + std::unique_ptr overuse_detector); ~VideoStreamEncoderResourceManager() override; - DegradationPreference degradation_preference() const { - return degradation_preference_; - } - DegradationPreference effective_degradation_preference() const { - return effective_degradation_preference_; - } + void SetDegradationPreferences( + DegradationPreference degradation_preference, + DegradationPreference effective_degradation_preference); - // ResourceAdaptationProcessorInterface implementation. - void StartResourceAdaptation( - ResourceAdaptationProcessorListener* adaptation_listener) override; - void StopResourceAdaptation() override; - // Uses a default VideoAdaptationReason of kCpu. - void AddResource(Resource* resource) override; - void AddResource(Resource* resource, VideoAdaptationReason reason); - void SetDegradationPreference( - DegradationPreference degradation_preference) override; + // Starts the encode usage resource. The quality scaler resource is + // automatically started on being configured. + void StartEncodeUsageResource(); + // Stops the encode usage and quality scaler resources if not already stopped. + void StopManagedResources(); // Settings that affect the VideoStreamEncoder-specific resources. void SetEncoderSettings(EncoderSettings encoder_settings); @@ -110,14 +98,20 @@ class VideoStreamEncoderResourceManager int64_t time_sent_in_us, absl::optional encode_duration_us); void OnFrameDropped(EncodedImageCallback::DropReason reason); + + // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order + // to be able to update |active_counts_|, which is used... + // - Legacy getStats() purposes. + // - Preventing adapting up in some circumstances (which may be questionable). + // TODO(hbos): Can we get rid of this? + void MapResourceToReason(Resource* resource, VideoAdaptationReason reason); + std::vector MappedResources() const; // If true, the VideoStreamEncoder should eexecute its logic to maybe drop // frames baseed on size and bitrate. bool DropInitialFrames() const; - // ResourceUsageListener implementation. - ResourceListenerResponse OnResourceUsageStateMeasured( - const Resource& resource) override; - + // ResourceAdaptationProcessorListener implementation. + // Updates |video_source_restrictions_| and |active_counts_|. void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, @@ -138,10 +132,6 @@ class VideoStreamEncoderResourceManager private: class InitialFrameDropper; - enum class State { kStopped, kStarted }; - - bool HasSufficientInputForAdaptation( - const VideoStreamInputState& input_state) const; VideoAdaptationReason GetReasonFromResource(const Resource& resource) const; // Performs the adaptation by getting the next target, applying it and @@ -152,15 +142,6 @@ class VideoStreamEncoderResourceManager CpuOveruseOptions GetCpuOveruseOptions() const; int LastInputFrameSizeOrDefault() const; - // Reinterprets "balanced + screenshare" as "maintain-resolution". - // When screensharing, as far as ResourceAdaptationProcessor logic is - // concerned, we ALWAYS use "maintain-resolution". However, on a different - // layer we may cap the video resolution to 720p to make high fps - // screensharing feasible. This means that on the API layer the preference is - // "balanced" (allowing reduction in both resolution and frame rate) but on - // this layer (not responsible for caping to 720p) the preference is the same - // as "maintain-resolution". - void MaybeUpdateEffectiveDegradationPreference(); // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener // to reconfigure the source accordingly. @@ -183,7 +164,6 @@ class VideoStreamEncoderResourceManager // TODO(https://crbug.com/webrtc/11222) Move experiment details into an inner // class. void MaybePerformQualityRampupExperiment(); - void ResetVideoSourceRestrictions(); void ResetActiveCounts(); std::string ActiveCountsToString() const; @@ -256,25 +236,23 @@ class VideoStreamEncoderResourceManager QualityScalerResource quality_scaler_resource_; VideoStreamInputStateProvider* const input_state_provider_; - ResourceAdaptationProcessorListener* const adaptation_listener_; - Clock* clock_; - State state_; - const bool experiment_cpu_load_estimator_; - // The restrictions that |adaptation_listener_| is informed of. - VideoSourceRestrictions video_source_restrictions_; + ResourceAdaptationProcessor* const adaptation_processor_; + VideoStreamEncoderObserver* const encoder_stats_observer_; + DegradationPreference degradation_preference_; DegradationPreference effective_degradation_preference_; - // Keeps track of source restrictions that this adaptation processor outputs. - const std::unique_ptr stream_adapter_; + VideoSourceRestrictions video_source_restrictions_; + + const BalancedDegradationSettings balanced_settings_; + Clock* clock_; + const bool experiment_cpu_load_estimator_; const std::unique_ptr initial_frame_dropper_; const bool quality_scaling_experiment_enabled_; - // This is the last non-zero target bitrate for the encoder. absl::optional encoder_target_bitrate_bps_; absl::optional encoder_rates_; bool quality_rampup_done_; QualityRampupExperiment quality_rampup_experiment_; absl::optional encoder_settings_; - VideoStreamEncoderObserver* const encoder_stats_observer_; // Ties a resource to a reason for statistical reporting. This AdaptReason is // also used by this module to make decisions about how to adapt up/down. diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index d82b24058d..af97ce8da4 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -255,25 +255,31 @@ VideoStreamEncoder::VideoStreamEncoder( automatic_animation_detection_experiment_( ParseAutomatincAnimationDetectionFieldTrial()), encoder_switch_requested_(false), - video_source_sink_controller_(std::make_unique( - /*sink=*/this, - /*source=*/nullptr)), - input_state_provider_(std::make_unique( - encoder_stats_observer)), - stream_resource_manager_( - std::make_unique( - input_state_provider_.get(), - clock_, - settings_.experiment_cpu_load_estimator, - std::move(overuse_detector), - encoder_stats_observer, - /*adaptation_listener=*/this)), + input_state_provider_(encoder_stats_observer), + resource_adaptation_processor_(&input_state_provider_, + encoder_stats_observer), + stream_resource_manager_(&input_state_provider_, + &resource_adaptation_processor_, + encoder_stats_observer, + clock_, + settings_.experiment_cpu_load_estimator, + std::move(overuse_detector)), + video_source_sink_controller_(/*sink=*/this, + /*source=*/nullptr), encoder_queue_(task_queue_factory->CreateTaskQueue( "EncoderQueue", TaskQueueFactory::Priority::NORMAL)) { RTC_DCHECK(encoder_stats_observer); RTC_DCHECK_GE(number_of_cores, 1); + resource_adaptation_processor_.AddAdaptationListener( + &stream_resource_manager_); + resource_adaptation_processor_.AddAdaptationListener(this); + + // Add the stream resource manager's resources to the processor. + for (Resource* resource : stream_resource_manager_.MappedResources()) + resource_adaptation_processor_.AddResource(resource); + for (auto& state : encoder_buffer_state_) state.fill(std::numeric_limits::max()); } @@ -286,10 +292,11 @@ VideoStreamEncoder::~VideoStreamEncoder() { void VideoStreamEncoder::Stop() { RTC_DCHECK_RUN_ON(&thread_checker_); - video_source_sink_controller_->SetSource(nullptr); + video_source_sink_controller_.SetSource(nullptr); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); - stream_resource_manager_->StopResourceAdaptation(); + stream_resource_manager_.StopManagedResources(); + resource_adaptation_processor_.StopResourceAdaptation(); rate_allocator_ = nullptr; bitrate_observer_ = nullptr; ReleaseEncoder(); @@ -325,20 +332,25 @@ void VideoStreamEncoder::SetSource( rtc::VideoSourceInterface* source, const DegradationPreference& degradation_preference) { RTC_DCHECK_RUN_ON(&thread_checker_); - video_source_sink_controller_->SetSource(source); + video_source_sink_controller_.SetSource(source); encoder_queue_.PostTask([this, source, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); - input_state_provider_->OnHasInputChanged(source); - stream_resource_manager_->SetDegradationPreference(degradation_preference); - if (encoder_) - stream_resource_manager_->ConfigureQualityScaler( + input_state_provider_.OnHasInputChanged(source); + resource_adaptation_processor_.SetDegradationPreference( + degradation_preference); + stream_resource_manager_.SetDegradationPreferences( + resource_adaptation_processor_.degradation_preference(), + resource_adaptation_processor_.effective_degradation_preference()); + if (encoder_) { + stream_resource_manager_.ConfigureQualityScaler( encoder_->GetEncoderInfo()); + } }); } void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { - video_source_sink_controller_->SetRotationApplied(rotation_applied); - video_source_sink_controller_->PushSourceSinkSettings(); + video_source_sink_controller_.SetRotationApplied(rotation_applied); + video_source_sink_controller_.PushSourceSinkSettings(); encoder_queue_.PostTask([this, sink] { RTC_DCHECK_RUN_ON(&encoder_queue_); sink_ = sink; @@ -351,7 +363,7 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { encoder_target_bitrate_bps_ = start_bitrate_bps != 0 ? absl::optional(start_bitrate_bps) : absl::nullopt; - stream_resource_manager_->SetStartBitrate( + stream_resource_manager_.SetStartBitrate( DataRate::BitsPerSec(start_bitrate_bps)); }); } @@ -568,12 +580,11 @@ void VideoStreamEncoder::ReconfigureEncoder() { max_framerate = std::max(stream.max_framerate, max_framerate); } int alignment = encoder_->GetEncoderInfo().requested_resolution_alignment; - if (max_framerate != - video_source_sink_controller_->frame_rate_upper_limit() || - alignment != video_source_sink_controller_->resolution_alignment()) { - video_source_sink_controller_->SetFrameRateUpperLimit(max_framerate); - video_source_sink_controller_->SetResolutionAlignment(alignment); - video_source_sink_controller_->PushSourceSinkSettings(); + if (max_framerate != video_source_sink_controller_.frame_rate_upper_limit() || + alignment != video_source_sink_controller_.resolution_alignment()) { + video_source_sink_controller_.SetFrameRateUpperLimit(max_framerate); + video_source_sink_controller_.SetResolutionAlignment(alignment); + video_source_sink_controller_.PushSourceSinkSettings(); } if (codec.maxBitrate == 0) { @@ -656,8 +667,10 @@ void VideoStreamEncoder::ReconfigureEncoder() { } if (pending_encoder_creation_) { - stream_resource_manager_->StopResourceAdaptation(); - stream_resource_manager_->StartResourceAdaptation(this); + stream_resource_manager_.StopManagedResources(); + resource_adaptation_processor_.StopResourceAdaptation(); + stream_resource_manager_.StartEncodeUsageResource(); + resource_adaptation_processor_.StartResourceAdaptation(); pending_encoder_creation_ = false; } @@ -730,14 +743,19 @@ void VideoStreamEncoder::ReconfigureEncoder() { std::move(streams), is_svc, encoder_config_.content_type, encoder_config_.min_transmit_bitrate_bps); - stream_resource_manager_->ConfigureQualityScaler(info); + stream_resource_manager_.ConfigureQualityScaler(info); } void VideoStreamEncoder::OnEncoderSettingsChanged() { EncoderSettings encoder_settings(encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_); - input_state_provider_->OnEncoderSettingsChanged(encoder_settings); - stream_resource_manager_->SetEncoderSettings(encoder_settings); + resource_adaptation_processor_.SetIsScreenshare( + encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen); + stream_resource_manager_.SetDegradationPreferences( + resource_adaptation_processor_.degradation_preference(), + resource_adaptation_processor_.effective_degradation_preference()); + input_state_provider_.OnEncoderSettingsChanged(encoder_settings); + stream_resource_manager_.SetEncoderSettings(encoder_settings); } void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { @@ -974,14 +992,14 @@ void VideoStreamEncoder::SetEncoderRates( frame_encode_metadata_writer_.OnSetRates( rate_settings.rate_control.bitrate, static_cast(rate_settings.rate_control.framerate_fps + 0.5)); - stream_resource_manager_->SetEncoderRates(rate_settings.rate_control); + stream_resource_manager_.SetEncoderRates(rate_settings.rate_control); } } void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, int64_t time_when_posted_us) { RTC_DCHECK_RUN_ON(&encoder_queue_); - input_state_provider_->OnFrameSizeObserved(video_frame.size()); + input_state_provider_.OnFrameSizeObserved(video_frame.size()); if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || video_frame.height() != last_frame_info_->height || @@ -1043,7 +1061,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, if (DropDueToSize(video_frame.size())) { RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; - stream_resource_manager_->OnFrameDroppedDueToSize(); + stream_resource_manager_.OnFrameDroppedDueToSize(); // Storing references to a native buffer risks blocking frame capture. if (video_frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) { @@ -1057,7 +1075,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, } return; } - stream_resource_manager_->OnMaybeEncodeFrame(); + stream_resource_manager_.OnMaybeEncodeFrame(); if (EncoderPaused()) { // Storing references to a native buffer risks blocking frame capture. @@ -1240,7 +1258,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); - stream_resource_manager_->OnEncodeStarted(out_frame, time_when_posted_us); + stream_resource_manager_.OnEncodeStarted(out_frame, time_when_posted_us); RTC_DCHECK_LE(send_codec_.width, out_frame.width()); RTC_DCHECK_LE(send_codec_.height, out_frame.height()); @@ -1515,7 +1533,7 @@ void VideoStreamEncoder::OnDroppedFrame(DropReason reason) { sink_->OnDroppedFrame(reason); encoder_queue_.PostTask([this, reason] { RTC_DCHECK_RUN_ON(&encoder_queue_); - stream_resource_manager_->OnFrameDropped(reason); + stream_resource_manager_.OnFrameDropped(reason); }); } @@ -1614,7 +1632,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, if (target_bitrate.bps() != 0) encoder_target_bitrate_bps_ = target_bitrate.bps(); - stream_resource_manager_->SetTargetBitrate(target_bitrate); + stream_resource_manager_.SetTargetBitrate(target_bitrate); if (video_suspension_changed) { RTC_LOG(LS_INFO) << "Video suspend state changed to: " @@ -1631,7 +1649,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, } bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { - if (!stream_resource_manager_->DropInitialFrames() || + if (!stream_resource_manager_.DropInitialFrames() || !encoder_target_bitrate_bps_.has_value()) { return false; } @@ -1659,8 +1677,8 @@ void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( const VideoAdaptationCounters& adaptation_counters, const Resource* reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); - video_source_sink_controller_->SetRestrictions(std::move(restrictions)); - video_source_sink_controller_->PushSourceSinkSettings(); + video_source_sink_controller_.SetRestrictions(std::move(restrictions)); + video_source_sink_controller_.PushSourceSinkSettings(); } void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image, @@ -1705,8 +1723,8 @@ void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image, } } - stream_resource_manager_->OnEncodeCompleted(encoded_image, time_sent_us, - encode_duration_us); + stream_resource_manager_.OnEncodeCompleted(encoded_image, time_sent_us, + encode_duration_us); if (bitrate_adjuster_) { bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index); } @@ -1861,7 +1879,7 @@ void VideoStreamEncoder::CheckForAnimatedContent( if (!automatic_animation_detection_experiment_.enabled || encoder_config_.content_type != VideoEncoderConfig::ContentType::kScreen || - stream_resource_manager_->degradation_preference() != + resource_adaptation_processor_.degradation_preference() != DegradationPreference::BALANCED) { return; } @@ -1919,16 +1937,17 @@ void VideoStreamEncoder::CheckForAnimatedContent( RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent " "animation detection."; } - video_source_sink_controller_->SetPixelsPerFrameUpperLimit( + video_source_sink_controller_.SetPixelsPerFrameUpperLimit( should_cap_resolution ? absl::optional(kMaxAnimationPixels) : absl::nullopt); - video_source_sink_controller_->PushSourceSinkSettings(); + video_source_sink_controller_.PushSourceSinkSettings(); } } void VideoStreamEncoder::InjectAdaptationResource( Resource* resource, VideoAdaptationReason reason) { - stream_resource_manager_->AddResource(resource, reason); + stream_resource_manager_.MapResourceToReason(resource, reason); + resource_adaptation_processor_.AddResource(resource); } } // namespace webrtc diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 4ad7edcc97..f593cf2a81 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -397,23 +397,29 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // track of whether a request has been made or not. bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_); - // The controller updates the sink wants based on restrictions that come from - // the resource adaptation processor or adaptation due to bandwidth - // adaptation. + // Provies video stream input states: current resolution and frame rate. + VideoStreamInputStateProvider input_state_provider_ + RTC_GUARDED_BY(&encoder_queue_); + // Responsible for adapting input resolution or frame rate to ensure resources + // (e.g. CPU or bandwidth) are not overused. + ResourceAdaptationProcessor resource_adaptation_processor_ + RTC_GUARDED_BY(&encoder_queue_); + // Handles input, output and stats reporting related to VideoStreamEncoder + // specific resources, such as "encode usage percent" measurements and "QP + // scaling". Also involved with various mitigations such as inital frame + // dropping. + VideoStreamEncoderResourceManager stream_resource_manager_ + RTC_GUARDED_BY(&encoder_queue_); + // Carries out the VideoSourceRestrictions provided by the + // ResourceAdaptationProcessor, i.e. reconfigures the source of video frames + // to provide us with different resolution or frame rate. // - // This is used on the encoder queue, with a few exceptions: + // Used on the |encoder_queue_| with a few exceptions: // - VideoStreamEncoder::SetSource() invokes SetSource(). // - VideoStreamEncoder::SetSink() invokes SetRotationApplied() and // PushSourceSinkSettings(). // - VideoStreamEncoder::Stop() invokes SetSource(). - // TODO(hbos): If these can be moved to the encoder queue, - // VideoSourceSinkController can be made single-threaded, and its lock can be - // replaced with a sequence checker. - std::unique_ptr video_source_sink_controller_; - std::unique_ptr input_state_provider_ - RTC_GUARDED_BY(&encoder_queue_); - std::unique_ptr stream_resource_manager_ - RTC_GUARDED_BY(&encoder_queue_); + VideoSourceSinkController video_source_sink_controller_; // All public methods are proxied to |encoder_queue_|. It must must be // destroyed first to make sure no tasks are run that use other members.