diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index deac3156d6..c54665c03e 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -29,6 +29,7 @@ rtc_library("resource_adaptation") { ] deps = [ "../../api:rtp_parameters", + "../../api:scoped_refptr", "../../api/video:video_adaptation", "../../api/video:video_frame", "../../api/video:video_stream_encoder", @@ -36,7 +37,9 @@ rtc_library("resource_adaptation") { "../../modules/video_coding:video_coding_utility", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_task_queue", "../../rtc_base/experiments:balanced_degradation_settings", + "../../rtc_base/synchronization:sequence_checker", "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/types:optional", ] @@ -56,6 +59,7 @@ if (rtc_include_tests) { deps = [ ":resource_adaptation", ":resource_adaptation_test_utilities", + "../../api:scoped_refptr", "../../api/video:video_adaptation", "../../api/video_codecs:video_codecs_api", "../../rtc_base:checks", diff --git a/call/adaptation/resource.cc b/call/adaptation/resource.cc index 1f90934258..52343ee644 100644 --- a/call/adaptation/resource.cc +++ b/call/adaptation/resource.cc @@ -19,7 +19,10 @@ ResourceListener::~ResourceListener() {} Resource::Resource() : usage_state_(absl::nullopt), listener_(nullptr) {} -Resource::~Resource() {} +Resource::~Resource() { + RTC_DCHECK(!listener_) + << "There is a listener depending on a Resource being destroyed."; +} void Resource::SetResourceListener(ResourceListener* listener) { // If you want to change listener you need to unregister the old listener by @@ -40,7 +43,7 @@ bool Resource::IsAdaptationUpAllowed( const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const { + rtc::scoped_refptr reason_resource) const { return true; } @@ -48,13 +51,13 @@ void Resource::OnAdaptationApplied( const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) {} + rtc::scoped_refptr reason_resource) {} void Resource::OnResourceUsageStateMeasured(ResourceUsageState usage_state) { usage_state_ = usage_state; if (!listener_) return; - listener_->OnResourceUsageStateMeasured(*this); + listener_->OnResourceUsageStateMeasured(this); } } // namespace webrtc diff --git a/call/adaptation/resource.h b/call/adaptation/resource.h index 1f58dc127c..6c7af488cf 100644 --- a/call/adaptation/resource.h +++ b/call/adaptation/resource.h @@ -15,8 +15,10 @@ #include #include "absl/types/optional.h" +#include "api/scoped_refptr.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state.h" +#include "rtc_base/ref_count.h" namespace webrtc { @@ -34,15 +36,16 @@ class ResourceListener { virtual ~ResourceListener(); // Informs the listener of a new measurement of resource usage. This means - // that |resource.usage_state()| is now up-to-date. - virtual void OnResourceUsageStateMeasured(const Resource& resource) = 0; + // that |resource->usage_state()| is now up-to-date. + virtual void OnResourceUsageStateMeasured( + rtc::scoped_refptr resource) = 0; }; -class Resource { +class Resource : public rtc::RefCountInterface { public: // By default, usage_state() is null until a measurement is made. Resource(); - virtual ~Resource(); + ~Resource() override; void SetResourceListener(ResourceListener* listener); @@ -56,12 +59,12 @@ class Resource { const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const; + rtc::scoped_refptr reason_resource) const; virtual void OnAdaptationApplied( const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource); + rtc::scoped_refptr reason_resource); virtual std::string name() const = 0; diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/resource_adaptation_processor.cc index 79fb9daab2..0224ac3bb2 100644 --- a/call/adaptation/resource_adaptation_processor.cc +++ b/call/adaptation/resource_adaptation_processor.cc @@ -10,6 +10,7 @@ #include "call/adaptation/resource_adaptation_processor.h" +#include #include #include "absl/algorithm/container.h" @@ -19,7 +20,9 @@ namespace webrtc { ResourceAdaptationProcessor::ResourceAdaptationProcessor( VideoStreamInputStateProvider* input_state_provider, VideoStreamEncoderObserver* encoder_stats_observer) - : input_state_provider_(input_state_provider), + : sequence_checker_(), + is_resource_adaptation_enabled_(false), + input_state_provider_(input_state_provider), encoder_stats_observer_(encoder_stats_observer), resources_(), degradation_preference_(DegradationPreference::DISABLED), @@ -27,53 +30,115 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( is_screenshare_(false), stream_adapter_(std::make_unique()), last_reported_source_restrictions_(), - processing_in_progress_(false) {} + processing_in_progress_(false) { + sequence_checker_.Detach(); +} -ResourceAdaptationProcessor::~ResourceAdaptationProcessor() = default; +ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(!is_resource_adaptation_enabled_); + RTC_DCHECK(adaptation_listeners_.empty()) + << "There are listener(s) depending on a ResourceAdaptationProcessor " + << "being destroyed."; + RTC_DCHECK(resources_.empty()) + << "There are resource(s) attached to a ResourceAdaptationProcessor " + << "being destroyed."; +} + +void ResourceAdaptationProcessor::InitializeOnResourceAdaptationQueue() { + // Allows |sequence_checker_| to attach to the resource adaptation queue. + // The caller is responsible for ensuring that this is the current queue. + RTC_DCHECK_RUN_ON(&sequence_checker_); +} DegradationPreference ResourceAdaptationProcessor::degradation_preference() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); return degradation_preference_; } DegradationPreference ResourceAdaptationProcessor::effective_degradation_preference() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); return effective_degradation_preference_; } void ResourceAdaptationProcessor::StartResourceAdaptation() { - for (auto* resource : resources_) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + if (is_resource_adaptation_enabled_) + return; + for (const auto& resource : resources_) { resource->SetResourceListener(this); } + is_resource_adaptation_enabled_ = true; } void ResourceAdaptationProcessor::StopResourceAdaptation() { - for (auto* resource : resources_) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + if (!is_resource_adaptation_enabled_) + return; + for (const auto& resource : resources_) { resource->SetResourceListener(nullptr); } + is_resource_adaptation_enabled_ = false; } void ResourceAdaptationProcessor::AddAdaptationListener( ResourceAdaptationProcessorListener* adaptation_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(std::find(adaptation_listeners_.begin(), + adaptation_listeners_.end(), + adaptation_listener) == adaptation_listeners_.end()); adaptation_listeners_.push_back(adaptation_listener); } -void ResourceAdaptationProcessor::AddResource(Resource* resource) { +void ResourceAdaptationProcessor::RemoveAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + auto it = std::find(adaptation_listeners_.begin(), + adaptation_listeners_.end(), adaptation_listener); + RTC_DCHECK(it != adaptation_listeners_.end()); + adaptation_listeners_.erase(it); +} + +void ResourceAdaptationProcessor::AddResource( + rtc::scoped_refptr resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + // TODO(hbos): Allow adding resources while |is_resource_adaptation_enabled_| + // by registering as a listener of the resource on adding it. + RTC_DCHECK(!is_resource_adaptation_enabled_); + RTC_DCHECK(std::find(resources_.begin(), resources_.end(), resource) == + resources_.end()); resources_.push_back(resource); } +void ResourceAdaptationProcessor::RemoveResource( + rtc::scoped_refptr resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + // TODO(hbos): Allow removing resources while + // |is_resource_adaptation_enabled_| by unregistering as a listener of the + // resource on removing it. + RTC_DCHECK(!is_resource_adaptation_enabled_); + auto it = std::find(resources_.begin(), resources_.end(), resource); + RTC_DCHECK(it != resources_.end()); + resources_.erase(it); +} + void ResourceAdaptationProcessor::SetDegradationPreference( DegradationPreference degradation_preference) { + RTC_DCHECK_RUN_ON(&sequence_checker_); degradation_preference_ = degradation_preference; MaybeUpdateEffectiveDegradationPreference(); } void ResourceAdaptationProcessor::SetIsScreenshare(bool is_screenshare) { + RTC_DCHECK_RUN_ON(&sequence_checker_); is_screenshare_ = is_screenshare; MaybeUpdateEffectiveDegradationPreference(); } void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { + RTC_DCHECK_RUN_ON(&sequence_checker_); effective_degradation_preference_ = (is_screenshare_ && degradation_preference_ == DegradationPreference::BALANCED) @@ -84,13 +149,15 @@ void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { } void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { + RTC_DCHECK_RUN_ON(&sequence_checker_); stream_adapter_->ClearRestrictions(); adaptations_counts_by_resource_.clear(); MaybeUpdateVideoSourceRestrictions(nullptr); } void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions( - const Resource* reason) { + rtc::scoped_refptr reason) { + RTC_DCHECK_RUN_ON(&sequence_checker_); VideoSourceRestrictions new_source_restrictions = FilterRestrictionsByDegradationPreference( stream_adapter_->source_restrictions(), @@ -109,9 +176,10 @@ void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions( } void ResourceAdaptationProcessor::OnResourceUsageStateMeasured( - const Resource& resource) { - RTC_DCHECK(resource.usage_state().has_value()); - switch (resource.usage_state().value()) { + rtc::scoped_refptr resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(resource->usage_state().has_value()); + switch (resource->usage_state().value()) { case ResourceUsageState::kOveruse: OnResourceOveruse(resource); break; @@ -123,6 +191,7 @@ void ResourceAdaptationProcessor::OnResourceUsageStateMeasured( bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation( const VideoStreamInputState& input_state) const { + RTC_DCHECK_RUN_ON(&sequence_checker_); return input_state.HasInputFrameSizeAndFramesPerSecond() && (effective_degradation_preference_ != DegradationPreference::MAINTAIN_RESOLUTION || @@ -130,14 +199,15 @@ bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation( } void ResourceAdaptationProcessor::OnResourceUnderuse( - const Resource& reason_resource) { + rtc::scoped_refptr reason_resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(!processing_in_progress_); processing_in_progress_ = true; // Clear all usage states. In order to re-run adaptation logic, resources need // to provide new resource usage measurements. // TODO(hbos): Support not unconditionally clearing usage states by having the // ResourceAdaptationProcessor check in on its resources at certain intervals. - for (Resource* resource : resources_) { + for (const auto& resource : resources_) { resource->ClearUsageState(); } VideoStreamInputState input_state = input_state_provider_->InputState(); @@ -146,7 +216,7 @@ void ResourceAdaptationProcessor::OnResourceUnderuse( processing_in_progress_ = false; return; } - if (!IsResourceAllowedToAdaptUp(&reason_resource)) { + if (!IsResourceAllowedToAdaptUp(reason_resource)) { processing_in_progress_ = false; return; } @@ -164,8 +234,8 @@ void ResourceAdaptationProcessor::OnResourceUnderuse( VideoSourceRestrictions restrictions_after = stream_adapter_->PeekNextRestrictions(adaptation); if (!absl::c_all_of(resources_, [&input_state, &restrictions_before, - &restrictions_after, - &reason_resource](const Resource* resource) { + &restrictions_after, &reason_resource]( + rtc::scoped_refptr resource) { return resource->IsAdaptationUpAllowed(input_state, restrictions_before, restrictions_after, reason_resource); @@ -175,25 +245,26 @@ void ResourceAdaptationProcessor::OnResourceUnderuse( } // Apply adaptation. stream_adapter_->ApplyAdaptation(adaptation); - for (Resource* resource : resources_) { + for (const auto& resource : resources_) { resource->OnAdaptationApplied(input_state, restrictions_before, restrictions_after, reason_resource); } // Update VideoSourceRestrictions based on adaptation. This also informs the // |adaptation_listeners_|. - MaybeUpdateVideoSourceRestrictions(&reason_resource); + MaybeUpdateVideoSourceRestrictions(reason_resource); processing_in_progress_ = false; } void ResourceAdaptationProcessor::OnResourceOveruse( - const Resource& reason_resource) { + rtc::scoped_refptr reason_resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(!processing_in_progress_); processing_in_progress_ = true; // Clear all usage states. In order to re-run adaptation logic, resources need // to provide new resource usage measurements. // TODO(hbos): Support not unconditionally clearing usage states by having the // ResourceAdaptationProcessor check in on its resources at certain intervals. - for (Resource* resource : resources_) { + for (const auto& resource : resources_) { resource->ClearUsageState(); } VideoStreamInputState input_state = input_state_provider_->InputState(); @@ -223,18 +294,19 @@ void ResourceAdaptationProcessor::OnResourceOveruse( VideoSourceRestrictions restrictions_after = stream_adapter_->PeekNextRestrictions(adaptation); stream_adapter_->ApplyAdaptation(adaptation); - for (Resource* resource : resources_) { + for (const auto& resource : resources_) { resource->OnAdaptationApplied(input_state, restrictions_before, restrictions_after, reason_resource); } // Update VideoSourceRestrictions based on adaptation. This also informs the // |adaptation_listeners_|. - MaybeUpdateVideoSourceRestrictions(&reason_resource); + MaybeUpdateVideoSourceRestrictions(reason_resource); processing_in_progress_ = false; } void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize( - const Resource& reason_resource) { + rtc::scoped_refptr reason_resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); VideoAdaptationCounters counters_before = stream_adapter_->adaptation_counters(); OnResourceOveruse(reason_resource); @@ -252,7 +324,8 @@ void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize( } void ResourceAdaptationProcessor::UpdateResourceDegradationCounts( - const Resource* resource) { + rtc::scoped_refptr resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(resource); int delta = stream_adapter_->adaptation_counters().Total(); for (const auto& adaptations : adaptations_counts_by_resource_) { @@ -265,7 +338,8 @@ void ResourceAdaptationProcessor::UpdateResourceDegradationCounts( } bool ResourceAdaptationProcessor::IsResourceAllowedToAdaptUp( - const Resource* resource) const { + rtc::scoped_refptr resource) const { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(resource); const auto& adaptations = adaptations_counts_by_resource_.find(resource); return adaptations != adaptations_counts_by_resource_.end() && diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h index de38751399..cf1e187026 100644 --- a/call/adaptation/resource_adaptation_processor.h +++ b/call/adaptation/resource_adaptation_processor.h @@ -17,6 +17,7 @@ #include "absl/types/optional.h" #include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" #include "api/video/video_frame.h" #include "api/video/video_stream_encoder_observer.h" #include "call/adaptation/resource.h" @@ -25,9 +26,26 @@ #include "call/adaptation/video_stream_adapter.h" #include "call/adaptation/video_stream_input_state.h" #include "call/adaptation/video_stream_input_state_provider.h" +#include "rtc_base/synchronization/sequence_checker.h" namespace webrtc { +// The Resource Adaptation Processor is responsible for reacting to resource +// usage measurements (e.g. overusing or underusing CPU). When a resource is +// overused the Processor is responsible for performing mitigations in order to +// consume less resources. +// +// Today we have one Processor per VideoStreamEncoder and the Processor is only +// capable of restricting resolution or frame rate of the encoded stream. In the +// future we should have a single Processor responsible for all encoded streams, +// and it should be capable of reconfiguring other things than just +// VideoSourceRestrictions (e.g. reduce render frame rate). +// See Resource-Adaptation hotlist: +// https://bugs.chromium.org/u/590058293/hotlists/Resource-Adaptation +// +// The ResourceAdaptationProcessor is single-threaded. It may be constructed on +// any thread but MUST subsequently be used and destroyed on a single sequence, +// i.e. the "resource adaptation task queue". class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, public ResourceListener { public: @@ -36,6 +54,8 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, VideoStreamEncoderObserver* encoder_stats_observer); ~ResourceAdaptationProcessor() override; + void InitializeOnResourceAdaptationQueue() override; + // ResourceAdaptationProcessorInterface implementation. DegradationPreference degradation_preference() const override; DegradationPreference effective_degradation_preference() const override; @@ -44,7 +64,10 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, void StopResourceAdaptation() override; void AddAdaptationListener( ResourceAdaptationProcessorListener* adaptation_listener) override; - void AddResource(Resource* resource) override; + void RemoveAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) override; + void AddResource(rtc::scoped_refptr resource) override; + void RemoveResource(rtc::scoped_refptr resource) override; void SetDegradationPreference( DegradationPreference degradation_preference) override; @@ -53,13 +76,14 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // ResourceListener implementation. // Triggers OnResourceUnderuse() or OnResourceOveruse(). - void OnResourceUsageStateMeasured(const Resource& resource) override; + void OnResourceUsageStateMeasured( + rtc::scoped_refptr resource) override; // May trigger 1-2 adaptations. It is meant to reduce resolution but this is // not guaranteed. It may adapt frame rate, which does not address the issue. // TODO(hbos): Can we get rid of this? void TriggerAdaptationDueToFrameDroppedDueToSize( - const Resource& reason_resource) override; + rtc::scoped_refptr reason_resource) override; private: bool HasSufficientInputForAdaptation( @@ -68,18 +92,18 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // Performs the adaptation by getting the next target, applying it and // informing listeners of the new VideoSourceRestriction and adaptation // counters. - void OnResourceUnderuse(const Resource& reason_resource); - void OnResourceOveruse(const Resource& reason_resource); + void OnResourceUnderuse(rtc::scoped_refptr reason_resource); + void OnResourceOveruse(rtc::scoped_refptr reason_resource); // Needs to be invoked any time |degradation_preference_| or |is_screenshare_| // changes to ensure |effective_degradation_preference_| is up-to-date. void MaybeUpdateEffectiveDegradationPreference(); // If the filtered source restrictions are different than // |last_reported_source_restrictions_|, inform the listeners. - void MaybeUpdateVideoSourceRestrictions(const Resource* reason); + void MaybeUpdateVideoSourceRestrictions(rtc::scoped_refptr reason); // Updates the number of times the resource has degraded based on the latest // degradation applied. - void UpdateResourceDegradationCounts(const Resource* resource); + void UpdateResourceDegradationCounts(rtc::scoped_refptr resource); // Returns true if a Resource has been overused in the pass and is responsible // for creating a VideoSourceRestriction. The current algorithm counts the // number of times the resource caused an adaptation and allows adapting up @@ -87,21 +111,33 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // traditionally been handled. // TODO(crbug.com/webrtc/11553) Change this algorithm to look at the resources // restrictions rather than just the counters. - bool IsResourceAllowedToAdaptUp(const Resource* resource) const; + bool IsResourceAllowedToAdaptUp(rtc::scoped_refptr resource) const; + webrtc::SequenceChecker sequence_checker_; + bool is_resource_adaptation_enabled_ RTC_GUARDED_BY(sequence_checker_); // Input and output. - VideoStreamInputStateProvider* const input_state_provider_; - VideoStreamEncoderObserver* const encoder_stats_observer_; - std::vector adaptation_listeners_; - std::vector resources_; - std::map adaptations_counts_by_resource_; + VideoStreamInputStateProvider* const input_state_provider_ + RTC_GUARDED_BY(sequence_checker_); + VideoStreamEncoderObserver* const encoder_stats_observer_ + RTC_GUARDED_BY(sequence_checker_); + std::vector adaptation_listeners_ + RTC_GUARDED_BY(sequence_checker_); + std::vector> resources_ + RTC_GUARDED_BY(sequence_checker_); + // Purely used for statistics, does not ensure mapped resources stay alive. + std::map adaptations_counts_by_resource_ + RTC_GUARDED_BY(sequence_checker_); // Adaptation strategy settings. - DegradationPreference degradation_preference_; - DegradationPreference effective_degradation_preference_; - bool is_screenshare_; + DegradationPreference degradation_preference_ + RTC_GUARDED_BY(sequence_checker_); + DegradationPreference effective_degradation_preference_ + RTC_GUARDED_BY(sequence_checker_); + bool is_screenshare_ RTC_GUARDED_BY(sequence_checker_); // Responsible for generating and applying possible adaptations. - const std::unique_ptr stream_adapter_; - VideoSourceRestrictions last_reported_source_restrictions_; + const std::unique_ptr stream_adapter_ + RTC_GUARDED_BY(sequence_checker_); + VideoSourceRestrictions last_reported_source_restrictions_ + RTC_GUARDED_BY(sequence_checker_); // Prevents recursion. // // This is used to prevent triggering resource adaptation in the process of @@ -113,7 +149,7 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // Resource::OnAdaptationApplied() -> // Resource::OnResourceUsageStateMeasured() -> // ResourceAdaptationProcessor::OnResourceOveruse() // Boom, not allowed. - bool processing_in_progress_; + bool processing_in_progress_ RTC_GUARDED_BY(sequence_checker_); }; } // namespace webrtc diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h index 6984273a29..d6295c4d75 100644 --- a/call/adaptation/resource_adaptation_processor_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h @@ -13,11 +13,13 @@ #include "absl/types/optional.h" #include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" #include "api/video/video_adaptation_counters.h" #include "api/video/video_frame.h" #include "call/adaptation/encoder_settings.h" #include "call/adaptation/resource.h" #include "call/adaptation/video_source_restrictions.h" +#include "rtc_base/task_queue.h" namespace webrtc { @@ -33,16 +35,19 @@ class ResourceAdaptationProcessorListener { virtual void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - const Resource* reason) = 0; + rtc::scoped_refptr reason) = 0; }; -// Responsible for reconfiguring encoded streams based on resource consumption, -// such as scaling down resolution or frame rate when CPU is overused. This -// interface is meant to be injectable into VideoStreamEncoder. +// The Resource Adaptation Processor is responsible for reacting to resource +// usage measurements (e.g. overusing or underusing CPU). When a resource is +// overused the Processor is responsible for performing mitigations in order to +// consume less resources. class ResourceAdaptationProcessorInterface { public: virtual ~ResourceAdaptationProcessorInterface(); + virtual void InitializeOnResourceAdaptationQueue() = 0; + virtual DegradationPreference degradation_preference() const = 0; // Reinterprets "balanced + screenshare" as "maintain-resolution". // TODO(hbos): Don't do this. This is not what "balanced" means. If the @@ -60,7 +65,10 @@ class ResourceAdaptationProcessorInterface { virtual void StopResourceAdaptation() = 0; virtual void AddAdaptationListener( ResourceAdaptationProcessorListener* adaptation_listener) = 0; - virtual void AddResource(Resource* resource) = 0; + virtual void RemoveAdaptationListener( + ResourceAdaptationProcessorListener* adaptation_listener) = 0; + virtual void AddResource(rtc::scoped_refptr resource) = 0; + virtual void RemoveResource(rtc::scoped_refptr resource) = 0; virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; @@ -74,7 +82,7 @@ class ResourceAdaptationProcessorInterface { // reasons. Can we replace this by something which actually satisfies the // resolution constraints or get rid of it altogether? virtual void TriggerAdaptationDueToFrameDroppedDueToSize( - const Resource& reason_resource) = 0; + rtc::scoped_refptr reason_resource) = 0; }; } // namespace webrtc diff --git a/call/adaptation/resource_adaptation_processor_unittest.cc b/call/adaptation/resource_adaptation_processor_unittest.cc index 7e7fe590dc..68dc4ba050 100644 --- a/call/adaptation/resource_adaptation_processor_unittest.cc +++ b/call/adaptation/resource_adaptation_processor_unittest.cc @@ -10,6 +10,7 @@ #include "call/adaptation/resource_adaptation_processor.h" +#include "api/scoped_refptr.h" #include "api/video/video_adaptation_counters.h" #include "call/adaptation/resource.h" #include "call/adaptation/resource_adaptation_processor_interface.h" @@ -43,13 +44,13 @@ class ResourceAdaptationProcessorListenerForTesting const VideoAdaptationCounters& adaptation_counters() const { return adaptation_counters_; } - const Resource* reason() const { return reason_; } + rtc::scoped_refptr reason() const { return reason_; } // ResourceAdaptationProcessorListener implementation. void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - const Resource* reason) override { + rtc::scoped_refptr reason) override { ++restrictions_updated_count_; restrictions_ = restrictions; adaptation_counters_ = adaptation_counters; @@ -60,7 +61,7 @@ class ResourceAdaptationProcessorListenerForTesting size_t restrictions_updated_count_; VideoSourceRestrictions restrictions_; VideoAdaptationCounters adaptation_counters_; - const Resource* reason_; + rtc::scoped_refptr reason_; }; class ResourceAdaptationProcessorTest : public ::testing::Test { @@ -68,16 +69,20 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { ResourceAdaptationProcessorTest() : frame_rate_provider_(), input_state_provider_(&frame_rate_provider_), - resource_("FakeResource"), - other_resource_("OtherFakeResource"), + resource_(new FakeResource("FakeResource")), + other_resource_(new FakeResource("OtherFakeResource")), processor_(&input_state_provider_, /*encoder_stats_observer=*/&frame_rate_provider_) { + processor_.InitializeOnResourceAdaptationQueue(); processor_.AddAdaptationListener(&processor_listener_); - processor_.AddResource(&resource_); - processor_.AddResource(&other_resource_); + processor_.AddResource(resource_); + processor_.AddResource(other_resource_); } ~ResourceAdaptationProcessorTest() override { processor_.StopResourceAdaptation(); + processor_.RemoveResource(resource_); + processor_.RemoveResource(other_resource_); + processor_.RemoveAdaptationListener(&processor_listener_); } void SetInputStates(bool has_input, int fps, int frame_size) { @@ -97,8 +102,8 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { protected: FakeFrameRateProvider frame_rate_provider_; VideoStreamInputStateProvider input_state_provider_; - FakeResource resource_; - FakeResource other_resource_; + rtc::scoped_refptr resource_; + rtc::scoped_refptr other_resource_; ResourceAdaptationProcessor processor_; ResourceAdaptationProcessorListenerForTesting processor_listener_; }; @@ -113,7 +118,7 @@ TEST_F(ResourceAdaptationProcessorTest, DisabledByDefault) { SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); processor_.StartResourceAdaptation(); // Adaptation does not happen when disabled. - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); } @@ -124,11 +129,11 @@ TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) { // Adaptation does not happen if input is insufficient. // When frame size is missing (OnFrameSizeObserved not called yet). input_state_provider_.OnHasInputChanged(true); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); // When "has input" is missing. SetInputStates(false, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); // Note: frame rate cannot be missing, if unset it is 0. } @@ -143,7 +148,7 @@ TEST_F(ResourceAdaptationProcessorTest, DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); EXPECT_TRUE( processor_listener_.restrictions().max_pixels_per_frame().has_value()); @@ -155,7 +160,7 @@ TEST_F(ResourceAdaptationProcessorTest, DegradationPreference::MAINTAIN_RESOLUTION); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); EXPECT_TRUE(processor_listener_.restrictions().max_frame_rate().has_value()); } @@ -170,7 +175,7 @@ TEST_F(ResourceAdaptationProcessorTest, // VideoStreamAdapter and default input states. This test requires it to be // achieved within 4 adaptations. for (size_t i = 0; i < 4; ++i) { - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(i + 1, processor_listener_.restrictions_updated_count()); RestrictSource(processor_listener_.restrictions()); } @@ -184,11 +189,11 @@ TEST_F(ResourceAdaptationProcessorTest, AwaitingPreviousAdaptation) { DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); // If we don't restrict the source then adaptation will not happen again due // to "awaiting previous adaptation". This prevents "double-adapt". - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); } @@ -197,7 +202,7 @@ TEST_F(ResourceAdaptationProcessorTest, CannotAdaptUpWhenUnrestricted) { DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kUnderuse); + resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); } @@ -206,10 +211,10 @@ TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) { DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); RestrictSource(processor_listener_.restrictions()); - resource_.set_usage_state(ResourceUsageState::kUnderuse); + resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(2u, processor_listener_.restrictions_updated_count()); EXPECT_EQ(VideoSourceRestrictions(), processor_listener_.restrictions()); } @@ -220,12 +225,12 @@ TEST_F(ResourceAdaptationProcessorTest, ResourcesCanPreventAdaptingUp) { processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); // Adapt down so that we can adapt up. - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); RestrictSource(processor_listener_.restrictions()); // Adapting up is prevented. - resource_.set_is_adaptation_up_allowed(false); - resource_.set_usage_state(ResourceUsageState::kUnderuse); + resource_->set_is_adaptation_up_allowed(false); + resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); } @@ -235,12 +240,12 @@ TEST_F(ResourceAdaptationProcessorTest, DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); RestrictSource(processor_listener_.restrictions()); // Other resource signals under-use - other_resource_.set_usage_state(ResourceUsageState::kUnderuse); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); } @@ -250,18 +255,18 @@ TEST_F(ResourceAdaptationProcessorTest, DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); processor_.ResetVideoSourceRestrictions(); EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); - other_resource_.set_usage_state(ResourceUsageState::kOveruse); + other_resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); // resource_ did not overuse after we reset the restrictions, so adapt up // should be disallowed. - resource_.set_usage_state(ResourceUsageState::kUnderuse); + resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); } @@ -271,28 +276,28 @@ TEST_F(ResourceAdaptationProcessorTest, DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); - other_resource_.set_usage_state(ResourceUsageState::kOveruse); + other_resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); - other_resource_.set_usage_state(ResourceUsageState::kOveruse); + other_resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(3, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); - resource_.set_usage_state(ResourceUsageState::kUnderuse); + resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); // Does not trigger adaptation since resource has no adaptations left. - resource_.set_usage_state(ResourceUsageState::kUnderuse); + resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); - other_resource_.set_usage_state(ResourceUsageState::kUnderuse); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); - other_resource_.set_usage_state(ResourceUsageState::kUnderuse); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); } @@ -302,8 +307,8 @@ TEST_F(ResourceAdaptationProcessorTest, AdaptingTriggersOnAdaptationApplied) { DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, resource_.num_adaptations_applied()); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, resource_->num_adaptations_applied()); } TEST_F(ResourceAdaptationProcessorTest, AdaptingClearsResourceUsageState) { @@ -311,18 +316,18 @@ TEST_F(ResourceAdaptationProcessorTest, AdaptingClearsResourceUsageState) { DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - EXPECT_FALSE(resource_.usage_state().has_value()); + EXPECT_FALSE(resource_->usage_state().has_value()); } TEST_F(ResourceAdaptationProcessorTest, FailingAdaptingAlsoClearsResourceUsageState) { processor_.SetDegradationPreference(DegradationPreference::DISABLED); processor_.StartResourceAdaptation(); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - EXPECT_FALSE(resource_.usage_state().has_value()); + EXPECT_FALSE(resource_->usage_state().has_value()); } TEST_F(ResourceAdaptationProcessorTest, @@ -331,17 +336,17 @@ TEST_F(ResourceAdaptationProcessorTest, DegradationPreference::MAINTAIN_FRAMERATE); processor_.StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - other_resource_.set_usage_state(ResourceUsageState::kUnderuse); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); // Does not trigger adapataion because there's no restriction. EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); - resource_.set_usage_state(ResourceUsageState::kOveruse); + resource_->set_usage_state(ResourceUsageState::kOveruse); // Adapts down even if other resource asked for adapting up. EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); - other_resource_.set_usage_state(ResourceUsageState::kUnderuse); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); // Doesn't adapt up because adaptation is due to another resource. EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); RestrictSource(processor_listener_.restrictions()); diff --git a/call/adaptation/resource_unittest.cc b/call/adaptation/resource_unittest.cc index d864005a72..50a6220f40 100644 --- a/call/adaptation/resource_unittest.cc +++ b/call/adaptation/resource_unittest.cc @@ -10,6 +10,7 @@ #include "call/adaptation/resource.h" +#include "api/scoped_refptr.h" #include "call/adaptation/test/fake_resource.h" #include "test/gmock.h" #include "test/gtest.h" @@ -21,29 +22,33 @@ using ::testing::StrictMock; class MockResourceListener : public ResourceListener { public: - MOCK_METHOD(void, OnResourceUsageStateMeasured, (const Resource& resource)); + MOCK_METHOD(void, + OnResourceUsageStateMeasured, + (rtc::scoped_refptr resource)); }; TEST(ResourceTest, RegisteringListenerReceivesCallbacks) { StrictMock resource_listener; - FakeResource fake_resource("FakeResource"); - fake_resource.SetResourceListener(&resource_listener); + rtc::scoped_refptr fake_resource( + new FakeResource("FakeResource")); + fake_resource->SetResourceListener(&resource_listener); EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)) .Times(1) - .WillOnce([](const Resource& resource) { - EXPECT_EQ(ResourceUsageState::kOveruse, resource.usage_state()); + .WillOnce([](rtc::scoped_refptr resource) { + EXPECT_EQ(ResourceUsageState::kOveruse, resource->usage_state()); }); - fake_resource.set_usage_state(ResourceUsageState::kOveruse); - fake_resource.SetResourceListener(nullptr); + fake_resource->set_usage_state(ResourceUsageState::kOveruse); + fake_resource->SetResourceListener(nullptr); } TEST(ResourceTest, UnregisteringListenerStopsCallbacks) { StrictMock resource_listener; - FakeResource fake_resource("FakeResource"); - fake_resource.SetResourceListener(&resource_listener); - fake_resource.SetResourceListener(nullptr); + rtc::scoped_refptr fake_resource( + new FakeResource("FakeResource")); + fake_resource->SetResourceListener(&resource_listener); + fake_resource->SetResourceListener(nullptr); EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)).Times(0); - fake_resource.set_usage_state(ResourceUsageState::kOveruse); + fake_resource->set_usage_state(ResourceUsageState::kOveruse); } } // namespace webrtc diff --git a/call/adaptation/test/fake_resource.cc b/call/adaptation/test/fake_resource.cc index bd7ad5431f..4c0a129d04 100644 --- a/call/adaptation/test/fake_resource.cc +++ b/call/adaptation/test/fake_resource.cc @@ -15,7 +15,7 @@ namespace webrtc { FakeResource::FakeResource(std::string name) - : Resource(), + : rtc::RefCountedObject(), name_(std::move(name)), is_adaptation_up_allowed_(true), num_adaptations_applied_(0) {} @@ -38,7 +38,7 @@ bool FakeResource::IsAdaptationUpAllowed( const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const { + rtc::scoped_refptr reason_resource) const { return is_adaptation_up_allowed_; } @@ -46,7 +46,7 @@ void FakeResource::OnAdaptationApplied( const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) { + rtc::scoped_refptr reason_resource) { ++num_adaptations_applied_; } diff --git a/call/adaptation/test/fake_resource.h b/call/adaptation/test/fake_resource.h index 0d9b1f46bb..beaca54614 100644 --- a/call/adaptation/test/fake_resource.h +++ b/call/adaptation/test/fake_resource.h @@ -14,11 +14,12 @@ #include #include "call/adaptation/resource.h" +#include "rtc_base/ref_counted_object.h" namespace webrtc { // Fake resource used for testing. -class FakeResource : public Resource { +class FakeResource : public rtc::RefCountedObject { public: explicit FakeResource(std::string name); ~FakeResource() override; @@ -29,14 +30,16 @@ class FakeResource : public Resource { // Resource implementation. std::string name() const override { return name_; } - bool IsAdaptationUpAllowed(const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const override; - void OnAdaptationApplied(const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) override; + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr reason_resource) const override; + void OnAdaptationApplied( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr reason_resource) override; private: const std::string name_; diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index 122458631c..3269d89ac9 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -22,6 +22,7 @@ rtc_library("video_adaptation") { deps = [ "../../api:rtp_parameters", + "../../api:scoped_refptr", "../../api/task_queue:task_queue", "../../api/video:video_adaptation", "../../api/video:video_frame", @@ -35,6 +36,7 @@ rtc_library("video_adaptation") { "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_event", "../../rtc_base:rtc_numerics", + "../../rtc_base:rtc_task_queue", "../../rtc_base:timeutils", "../../rtc_base/experiments:balanced_degradation_settings", "../../rtc_base/experiments:field_trial_parser", @@ -61,6 +63,7 @@ if (rtc_include_tests) { ] deps = [ ":video_adaptation", + "../../api:scoped_refptr", "../../api/video:encoded_image", "../../api/video:video_adaptation", "../../api/video:video_frame_i420", diff --git a/video/adaptation/encode_usage_resource.cc b/video/adaptation/encode_usage_resource.cc index 7a42878aa9..6e2827a9dd 100644 --- a/video/adaptation/encode_usage_resource.cc +++ b/video/adaptation/encode_usage_resource.cc @@ -20,13 +20,27 @@ namespace webrtc { EncodeUsageResource::EncodeUsageResource( std::unique_ptr overuse_detector) - : overuse_detector_(std::move(overuse_detector)), + : rtc::RefCountedObject(), + encoder_queue_(nullptr), + overuse_detector_(std::move(overuse_detector)), is_started_(false), target_frame_rate_(absl::nullopt) { RTC_DCHECK(overuse_detector_); } +void EncodeUsageResource::Initialize(rtc::TaskQueue* encoder_queue) { + RTC_DCHECK(!encoder_queue_); + RTC_DCHECK(encoder_queue); + encoder_queue_ = encoder_queue; +} + +bool EncodeUsageResource::is_started() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return is_started_; +} + void EncodeUsageResource::StartCheckForOveruse(CpuOveruseOptions options) { + RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(!is_started_); overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(), std::move(options), this); @@ -35,12 +49,14 @@ void EncodeUsageResource::StartCheckForOveruse(CpuOveruseOptions options) { } void EncodeUsageResource::StopCheckForOveruse() { + RTC_DCHECK_RUN_ON(encoder_queue_); overuse_detector_->StopCheckForOveruse(); is_started_ = false; } void EncodeUsageResource::SetTargetFrameRate( absl::optional target_frame_rate) { + RTC_DCHECK_RUN_ON(encoder_queue_); if (target_frame_rate == target_frame_rate_) return; target_frame_rate_ = target_frame_rate; @@ -50,6 +66,7 @@ void EncodeUsageResource::SetTargetFrameRate( void EncodeUsageResource::OnEncodeStarted(const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) { + RTC_DCHECK_RUN_ON(encoder_queue_); // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g. // "OnEncodeStarted"?) or revise usage. overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us); @@ -60,6 +77,7 @@ void EncodeUsageResource::OnEncodeCompleted( int64_t time_sent_in_us, int64_t capture_time_us, absl::optional encode_duration_us) { + RTC_DCHECK_RUN_ON(encoder_queue_); // TODO(hbos): Rename FrameSent() to something more appropriate (e.g. // "OnEncodeCompleted"?). overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, @@ -67,14 +85,21 @@ void EncodeUsageResource::OnEncodeCompleted( } void EncodeUsageResource::AdaptUp() { + RTC_DCHECK_RUN_ON(encoder_queue_); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // PostTask the resource usage measurements. OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); } void EncodeUsageResource::AdaptDown() { + RTC_DCHECK_RUN_ON(encoder_queue_); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // PostTask the resource usage measurements. OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); } int EncodeUsageResource::TargetFrameRateAsInt() { + RTC_DCHECK_RUN_ON(encoder_queue_); return target_frame_rate_.has_value() ? static_cast(target_frame_rate_.value()) : std::numeric_limits::max(); diff --git a/video/adaptation/encode_usage_resource.h b/video/adaptation/encode_usage_resource.h index 7147569ffb..2ade816ad7 100644 --- a/video/adaptation/encode_usage_resource.h +++ b/video/adaptation/encode_usage_resource.h @@ -17,6 +17,8 @@ #include "absl/types/optional.h" #include "api/video/video_adaptation_reason.h" #include "call/adaptation/resource.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/task_queue.h" #include "video/adaptation/overuse_frame_detector.h" namespace webrtc { @@ -26,13 +28,17 @@ namespace webrtc { // indirectly by usage in the ResourceAdaptationProcessor (which is only tested // because of its usage in VideoStreamEncoder); all tests are currently in // video_stream_encoder_unittest.cc. -class EncodeUsageResource : public Resource, +class EncodeUsageResource : public rtc::RefCountedObject, public OveruseFrameDetectorObserverInterface { public: explicit EncodeUsageResource( std::unique_ptr overuse_detector); - bool is_started() const { return is_started_; } + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // pass it in here. + void Initialize(rtc::TaskQueue* encoder_queue); + + bool is_started() const; void StartCheckForOveruse(CpuOveruseOptions options); void StopCheckForOveruse(); @@ -54,9 +60,11 @@ class EncodeUsageResource : public Resource, private: int TargetFrameRateAsInt(); - const std::unique_ptr overuse_detector_; - bool is_started_; - absl::optional target_frame_rate_; + rtc::TaskQueue* encoder_queue_; + const std::unique_ptr overuse_detector_ + RTC_GUARDED_BY(encoder_queue_); + bool is_started_ RTC_GUARDED_BY(encoder_queue_); + absl::optional target_frame_rate_ RTC_GUARDED_BY(encoder_queue_); }; } // namespace webrtc diff --git a/video/adaptation/quality_scaler_resource.cc b/video/adaptation/quality_scaler_resource.cc index ca317e5a8c..9fcc58e6be 100644 --- a/video/adaptation/quality_scaler_resource.cc +++ b/video/adaptation/quality_scaler_resource.cc @@ -16,40 +16,61 @@ namespace webrtc { -QualityScalerResource::QualityScalerResource( - ResourceAdaptationProcessorInterface* adaptation_processor) - : adaptation_processor_(adaptation_processor), +QualityScalerResource::QualityScalerResource() + : rtc::RefCountedObject(), + encoder_queue_(nullptr), + adaptation_processor_(nullptr), quality_scaler_(nullptr), pending_qp_usage_callback_(nullptr) {} +void QualityScalerResource::Initialize(rtc::TaskQueue* encoder_queue) { + RTC_DCHECK(!encoder_queue_); + RTC_DCHECK(encoder_queue); + encoder_queue_ = encoder_queue; +} + +QualityScalerResource::~QualityScalerResource() {} + +void QualityScalerResource::SetAdaptationProcessor( + ResourceAdaptationProcessorInterface* adaptation_processor) { + RTC_DCHECK_RUN_ON(encoder_queue_); + adaptation_processor_ = adaptation_processor; +} + bool QualityScalerResource::is_started() const { + RTC_DCHECK_RUN_ON(encoder_queue_); return quality_scaler_.get(); } void QualityScalerResource::StartCheckForOveruse( VideoEncoder::QpThresholds qp_thresholds) { + RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(!is_started()); quality_scaler_ = std::make_unique(this, std::move(qp_thresholds)); } void QualityScalerResource::StopCheckForOveruse() { + RTC_DCHECK_RUN_ON(encoder_queue_); quality_scaler_.reset(); } void QualityScalerResource::SetQpThresholds( VideoEncoder::QpThresholds qp_thresholds) { + RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(is_started()); quality_scaler_->SetQpThresholds(std::move(qp_thresholds)); } bool QualityScalerResource::QpFastFilterLow() { + RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(is_started()); return quality_scaler_->QpFastFilterLow(); } void QualityScalerResource::OnEncodeCompleted(const EncodedImage& encoded_image, int64_t time_sent_in_us) { + RTC_DCHECK_RUN_ON(encoder_queue_); if (quality_scaler_ && encoded_image.qp_ >= 0) { quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us); } else if (!quality_scaler_) { @@ -64,6 +85,7 @@ void QualityScalerResource::OnEncodeCompleted(const EncodedImage& encoded_image, void QualityScalerResource::OnFrameDropped( EncodedImageCallback::DropReason reason) { + RTC_DCHECK_RUN_ON(encoder_queue_); if (!quality_scaler_) return; switch (reason) { @@ -78,7 +100,10 @@ void QualityScalerResource::OnFrameDropped( void QualityScalerResource::OnReportQpUsageHigh( rtc::scoped_refptr callback) { + RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(!pending_qp_usage_callback_); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // PostTask the resource usage measurements. pending_qp_usage_callback_ = std::move(callback); // If this triggers adaptation, OnAdaptationApplied() is called by the // processor where we determine if QP should be cleared and we invoke and null @@ -94,7 +119,10 @@ void QualityScalerResource::OnReportQpUsageHigh( void QualityScalerResource::OnReportQpUsageLow( rtc::scoped_refptr callback) { + RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(!pending_qp_usage_callback_); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // PostTask the resource usage measurements. OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); callback->OnQpUsageHandled(true); } @@ -103,7 +131,10 @@ void QualityScalerResource::OnAdaptationApplied( const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) { + rtc::scoped_refptr reason_resource) { + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // ensure that this is running on it instead. + RTC_DCHECK_RUN_ON(encoder_queue_); // We only clear QP samples on adaptations triggered by the QualityScaler. if (!pending_qp_usage_callback_) return; @@ -119,7 +150,8 @@ void QualityScalerResource::OnAdaptationApplied( // interval whose delay is calculated based on events such as these. Now there // is much dependency on a specific OnReportQpUsageHigh() event and "balanced" // but adaptations happening might not align with QualityScaler's CheckQpTask. - if (adaptation_processor_->effective_degradation_preference() == + if (adaptation_processor_ && + adaptation_processor_->effective_degradation_preference() == DegradationPreference::BALANCED && DidDecreaseFrameRate(restrictions_before, restrictions_after)) { absl::optional min_diff = BalancedDegradationSettings().MinFpsDiff( diff --git a/video/adaptation/quality_scaler_resource.h b/video/adaptation/quality_scaler_resource.h index 30796c3db9..6cec79c4b0 100644 --- a/video/adaptation/quality_scaler_resource.h +++ b/video/adaptation/quality_scaler_resource.h @@ -19,6 +19,8 @@ #include "call/adaptation/resource.h" #include "call/adaptation/resource_adaptation_processor_interface.h" #include "modules/video_coding/utility/quality_scaler.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/task_queue.h" namespace webrtc { @@ -27,10 +29,16 @@ namespace webrtc { // indirectly by usage in the ResourceAdaptationProcessor (which is only tested // because of its usage in VideoStreamEncoder); all tests are currently in // video_stream_encoder_unittest.cc. -class QualityScalerResource : public Resource, +class QualityScalerResource : public rtc::RefCountedObject, public QualityScalerQpUsageHandlerInterface { public: - explicit QualityScalerResource( + QualityScalerResource(); + ~QualityScalerResource() override; + + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // pass it in here. + void Initialize(rtc::TaskQueue* encoder_queue); + void SetAdaptationProcessor( ResourceAdaptationProcessorInterface* adaptation_processor); bool is_started() const; @@ -55,16 +63,21 @@ class QualityScalerResource : public Resource, std::string name() const override { return "QualityScalerResource"; } // Resource implementation. - void OnAdaptationApplied(const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) override; + void OnAdaptationApplied( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr reason_resource) override; private: - ResourceAdaptationProcessorInterface* const adaptation_processor_; - std::unique_ptr quality_scaler_; + rtc::TaskQueue* encoder_queue_; + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // guard the processor by it instead. + ResourceAdaptationProcessorInterface* adaptation_processor_ + RTC_GUARDED_BY(encoder_queue_); + std::unique_ptr quality_scaler_ RTC_GUARDED_BY(encoder_queue_); rtc::scoped_refptr - pending_qp_usage_callback_; + pending_qp_usage_callback_ RTC_GUARDED_BY(encoder_queue_); }; } // namespace webrtc diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index b8179d0f71..f5c812a761 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -68,7 +68,8 @@ VideoAdaptationReason OtherReason(VideoAdaptationReason reason) { class VideoStreamEncoderResourceManager::InitialFrameDropper { public: - explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource) + explicit InitialFrameDropper( + rtc::scoped_refptr quality_scaler_resource) : quality_scaler_resource_(quality_scaler_resource), quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), has_seen_first_bwe_drop_(false), @@ -128,7 +129,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { // achieve desired bitrate. static const int kMaxInitialFramedrop = 4; - const QualityScalerResource* quality_scaler_resource_; + const rtc::scoped_refptr quality_scaler_resource_; const QualityScalerSettings quality_scaler_settings_; bool has_seen_first_bwe_drop_; DataRate set_start_bitrate_; @@ -139,13 +140,16 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts:: PreventAdaptUpDueToActiveCounts(VideoStreamEncoderResourceManager* manager) - : manager_(manager) {} + : rtc::RefCountedObject(), manager_(manager) {} bool VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts:: IsAdaptationUpAllowed(const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const { + rtc::scoped_refptr reason_resource) const { + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // ensure that this is running on it instead. + RTC_DCHECK_RUN_ON(manager_->encoder_queue_); VideoAdaptationReason reason = manager_->GetReasonFromResource(reason_resource); // We can't adapt up if we're already at the highest setting. @@ -155,10 +159,11 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts:: // TODO(hbos): Why would the reason matter? If a particular resource doesn't // want us to go up it should prevent us from doing so itself rather than to // have this catch-all reason- and stats-based approach. - int num_downgrades = FilterVideoAdaptationCountersByDegradationPreference( - manager_->active_counts_[reason], - manager_->effective_degradation_preference_) - .Total(); + int num_downgrades = + FilterVideoAdaptationCountersByDegradationPreference( + manager_->active_counts_[reason], + manager_->adaptation_processor_->effective_degradation_preference()) + .Total(); RTC_DCHECK_GE(num_downgrades, 0); return num_downgrades > 0; } @@ -167,14 +172,17 @@ VideoStreamEncoderResourceManager:: PreventIncreaseResolutionDueToBitrateResource:: PreventIncreaseResolutionDueToBitrateResource( VideoStreamEncoderResourceManager* manager) - : manager_(manager) {} + : rtc::RefCountedObject(), manager_(manager) {} bool VideoStreamEncoderResourceManager:: PreventIncreaseResolutionDueToBitrateResource::IsAdaptationUpAllowed( const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const { + rtc::scoped_refptr reason_resource) const { + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // ensure that this is running on it instead. + RTC_DCHECK_RUN_ON(manager_->encoder_queue_); VideoAdaptationReason reason = manager_->GetReasonFromResource(reason_resource); // If increasing resolution due to kQuality, make sure bitrate limits are not @@ -205,13 +213,16 @@ bool VideoStreamEncoderResourceManager:: VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: PreventAdaptUpInBalancedResource(VideoStreamEncoderResourceManager* manager) - : manager_(manager) {} + : rtc::RefCountedObject(), manager_(manager) {} bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: IsAdaptationUpAllowed(const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const { + rtc::scoped_refptr reason_resource) const { + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // ensure that this is running on it instead. + RTC_DCHECK_RUN_ON(manager_->encoder_queue_); VideoAdaptationReason reason = manager_->GetReasonFromResource(reason_resource); // Don't adapt if BalancedDegradationSettings applies and determines this will @@ -219,7 +230,7 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: // TODO(hbos): Why are we allowing violating balanced settings if adapting due // CPU? Shouldn't this condition be checked regardless of reason? if (reason == VideoAdaptationReason::kQuality && - manager_->effective_degradation_preference_ == + manager_->adaptation_processor_->effective_degradation_preference() == DegradationPreference::BALANCED && !manager_->balanced_settings_.CanAdaptUp( input_state.video_codec_type(), @@ -240,26 +251,29 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, - ResourceAdaptationProcessorInterface* adaptation_processor, VideoStreamEncoderObserver* encoder_stats_observer, Clock* clock, bool experiment_cpu_load_estimator, std::unique_ptr overuse_detector) - : prevent_adapt_up_due_to_active_counts_(this), - prevent_increase_resolution_due_to_bitrate_resource_(this), - prevent_adapt_up_in_balanced_resource_(this), - encode_usage_resource_(std::move(overuse_detector)), - quality_scaler_resource_(adaptation_processor), + : prevent_adapt_up_due_to_active_counts_( + new PreventAdaptUpDueToActiveCounts(this)), + prevent_increase_resolution_due_to_bitrate_resource_( + new PreventIncreaseResolutionDueToBitrateResource(this)), + prevent_adapt_up_in_balanced_resource_( + new PreventAdaptUpInBalancedResource(this)), + encode_usage_resource_( + new EncodeUsageResource(std::move(overuse_detector))), + quality_scaler_resource_(new QualityScalerResource()), + encoder_queue_(nullptr), input_state_provider_(input_state_provider), - adaptation_processor_(adaptation_processor), + adaptation_processor_(nullptr), encoder_stats_observer_(encoder_stats_observer), degradation_preference_(DegradationPreference::DISABLED), - effective_degradation_preference_(DegradationPreference::DISABLED), video_source_restrictions_(), clock_(clock), experiment_cpu_load_estimator_(experiment_cpu_load_estimator), initial_frame_dropper_( - std::make_unique(&quality_scaler_resource_)), + std::make_unique(quality_scaler_resource_)), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), encoder_target_bitrate_bps_(absl::nullopt), quality_rampup_done_(false), @@ -267,43 +281,65 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( encoder_settings_(absl::nullopt), active_counts_() { RTC_DCHECK(encoder_stats_observer_); - MapResourceToReason(&prevent_adapt_up_due_to_active_counts_, + MapResourceToReason(prevent_adapt_up_due_to_active_counts_, VideoAdaptationReason::kQuality); - MapResourceToReason(&prevent_increase_resolution_due_to_bitrate_resource_, + MapResourceToReason(prevent_increase_resolution_due_to_bitrate_resource_, VideoAdaptationReason::kQuality); - MapResourceToReason(&prevent_adapt_up_in_balanced_resource_, + MapResourceToReason(prevent_adapt_up_in_balanced_resource_, VideoAdaptationReason::kQuality); - MapResourceToReason(&encode_usage_resource_, VideoAdaptationReason::kCpu); - MapResourceToReason(&quality_scaler_resource_, + MapResourceToReason(encode_usage_resource_, VideoAdaptationReason::kCpu); + MapResourceToReason(quality_scaler_resource_, VideoAdaptationReason::kQuality); } -VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() { - RTC_DCHECK(!encode_usage_resource_.is_started()); +VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() {} + +void VideoStreamEncoderResourceManager::Initialize( + rtc::TaskQueue* encoder_queue) { + RTC_DCHECK(!encoder_queue_); + RTC_DCHECK(encoder_queue); + encoder_queue_ = encoder_queue; + encode_usage_resource_->Initialize(encoder_queue_); + quality_scaler_resource_->Initialize(encoder_queue_); +} + +void VideoStreamEncoderResourceManager::SetAdaptationProcessor( + ResourceAdaptationProcessorInterface* adaptation_processor) { + RTC_DCHECK_RUN_ON(encoder_queue_); + adaptation_processor_ = adaptation_processor; + quality_scaler_resource_->SetAdaptationProcessor(adaptation_processor); } void VideoStreamEncoderResourceManager::SetDegradationPreferences( - DegradationPreference degradation_preference, - DegradationPreference effective_degradation_preference) { + DegradationPreference degradation_preference) { + RTC_DCHECK_RUN_ON(encoder_queue_); degradation_preference_ = degradation_preference; - effective_degradation_preference_ = effective_degradation_preference; UpdateStatsAdaptationSettings(); } +DegradationPreference +VideoStreamEncoderResourceManager::degradation_preference() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return degradation_preference_; +} + void VideoStreamEncoderResourceManager::StartEncodeUsageResource() { - RTC_DCHECK(!encode_usage_resource_.is_started()); + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK(!encode_usage_resource_->is_started()); RTC_DCHECK(encoder_settings_.has_value()); - encode_usage_resource_.StartCheckForOveruse(GetCpuOveruseOptions()); + encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions()); } void VideoStreamEncoderResourceManager::StopManagedResources() { - encode_usage_resource_.StopCheckForOveruse(); - quality_scaler_resource_.StopCheckForOveruse(); + RTC_DCHECK_RUN_ON(encoder_queue_); + encode_usage_resource_->StopCheckForOveruse(); + quality_scaler_resource_->StopCheckForOveruse(); } void VideoStreamEncoderResourceManager::MapResourceToReason( - Resource* resource, + rtc::scoped_refptr resource, VideoAdaptationReason reason) { + rtc::CritScope crit(&resource_lock_); RTC_DCHECK(resource); RTC_DCHECK(absl::c_find_if(resources_, [resource](const ResourceAndReason& r) { @@ -313,22 +349,25 @@ void VideoStreamEncoderResourceManager::MapResourceToReason( resources_.emplace_back(resource, reason); } -std::vector VideoStreamEncoderResourceManager::MappedResources() - const { - std::vector resources; +std::vector> +VideoStreamEncoderResourceManager::MappedResources() const { + rtc::CritScope crit(&resource_lock_); + std::vector> resources; for (auto const& resource_and_reason : resources_) { resources.push_back(resource_and_reason.resource); } return resources; } -QualityScalerResource* +rtc::scoped_refptr VideoStreamEncoderResourceManager::quality_scaler_resource_for_testing() { - return &quality_scaler_resource_; + rtc::CritScope crit(&resource_lock_); + return quality_scaler_resource_; } void VideoStreamEncoderResourceManager::SetEncoderSettings( EncoderSettings encoder_settings) { + RTC_DCHECK_RUN_ON(encoder_queue_); encoder_settings_ = std::move(encoder_settings); quality_rampup_experiment_.SetMaxBitrate( @@ -339,6 +378,7 @@ void VideoStreamEncoderResourceManager::SetEncoderSettings( void VideoStreamEncoderResourceManager::SetStartBitrate( DataRate start_bitrate) { + RTC_DCHECK_RUN_ON(encoder_queue_); if (!start_bitrate.IsZero()) encoder_target_bitrate_bps_ = start_bitrate.bps(); initial_frame_dropper_->SetStartBitrate(start_bitrate, @@ -347,6 +387,7 @@ void VideoStreamEncoderResourceManager::SetStartBitrate( void VideoStreamEncoderResourceManager::SetTargetBitrate( DataRate target_bitrate) { + RTC_DCHECK_RUN_ON(encoder_queue_); if (!target_bitrate.IsZero()) encoder_target_bitrate_bps_ = target_bitrate.bps(); initial_frame_dropper_->SetTargetBitrate(target_bitrate, @@ -355,10 +396,14 @@ void VideoStreamEncoderResourceManager::SetTargetBitrate( void VideoStreamEncoderResourceManager::SetEncoderRates( const VideoEncoder::RateControlParameters& encoder_rates) { + RTC_DCHECK_RUN_ON(encoder_queue_); encoder_rates_ = encoder_rates; } void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() { + RTC_DCHECK_RUN_ON(encoder_queue_); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // PostTask the request to adapt due to frame drop. adaptation_processor_->TriggerAdaptationDueToFrameDroppedDueToSize( quality_scaler_resource_); initial_frame_dropper_->OnFrameDroppedDueToSize(); @@ -367,51 +412,57 @@ void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() { void VideoStreamEncoderResourceManager::OnEncodeStarted( const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) { - encode_usage_resource_.OnEncodeStarted(cropped_frame, - time_when_first_seen_us); + encode_usage_resource_->OnEncodeStarted(cropped_frame, + time_when_first_seen_us); } void VideoStreamEncoderResourceManager::OnEncodeCompleted( const EncodedImage& encoded_image, int64_t time_sent_in_us, absl::optional encode_duration_us) { + RTC_DCHECK_RUN_ON(encoder_queue_); // Inform |encode_usage_resource_| of the encode completed event. uint32_t timestamp = encoded_image.Timestamp(); int64_t capture_time_us = encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; - encode_usage_resource_.OnEncodeCompleted(timestamp, time_sent_in_us, - capture_time_us, encode_duration_us); + encode_usage_resource_->OnEncodeCompleted( + timestamp, time_sent_in_us, capture_time_us, encode_duration_us); // Inform |quality_scaler_resource_| of the encode completed event. - quality_scaler_resource_.OnEncodeCompleted(encoded_image, time_sent_in_us); + quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); } void VideoStreamEncoderResourceManager::OnFrameDropped( EncodedImageCallback::DropReason reason) { - quality_scaler_resource_.OnFrameDropped(reason); + RTC_DCHECK_RUN_ON(encoder_queue_); + quality_scaler_resource_->OnFrameDropped(reason); } bool VideoStreamEncoderResourceManager::DropInitialFrames() const { + RTC_DCHECK_RUN_ON(encoder_queue_); return initial_frame_dropper_->DropInitialFrames(); } void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() { + RTC_DCHECK_RUN_ON(encoder_queue_); initial_frame_dropper_->OnMaybeEncodeFrame(); MaybePerformQualityRampupExperiment(); } void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings( absl::optional qp_thresholds) { + RTC_DCHECK_RUN_ON(encoder_queue_); if (qp_thresholds.has_value()) { - quality_scaler_resource_.StopCheckForOveruse(); - quality_scaler_resource_.StartCheckForOveruse(qp_thresholds.value()); + quality_scaler_resource_->StopCheckForOveruse(); + quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value()); } else { - quality_scaler_resource_.StopCheckForOveruse(); + quality_scaler_resource_->StopCheckForOveruse(); } initial_frame_dropper_->OnQualityScalerSettingsUpdated(); } void VideoStreamEncoderResourceManager::ConfigureQualityScaler( const VideoEncoder::EncoderInfo& encoder_info) { + RTC_DCHECK_RUN_ON(encoder_queue_); const auto scaling_settings = encoder_info.scaling_settings; const bool quality_scaling_allowed = IsResolutionScalingEnabled(degradation_preference_) && @@ -420,7 +471,7 @@ void VideoStreamEncoderResourceManager::ConfigureQualityScaler( // TODO(https://crbug.com/webrtc/11222): Should this move to // QualityScalerResource? if (quality_scaling_allowed) { - if (!quality_scaler_resource_.is_started()) { + if (!quality_scaler_resource_->is_started()) { // Quality scaler has not already been configured. // Use experimental thresholds if available. @@ -439,26 +490,27 @@ void VideoStreamEncoderResourceManager::ConfigureQualityScaler( // Set the qp-thresholds to the balanced settings if balanced mode. if (degradation_preference_ == DegradationPreference::BALANCED && - quality_scaler_resource_.is_started()) { + quality_scaler_resource_->is_started()) { absl::optional thresholds = balanced_settings_.GetQpThresholds( GetVideoCodecTypeOrGeneric(encoder_settings_), LastInputFrameSizeOrDefault()); if (thresholds) { - quality_scaler_resource_.SetQpThresholds(*thresholds); + quality_scaler_resource_->SetQpThresholds(*thresholds); } } UpdateStatsAdaptationSettings(); } VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource( - const Resource& resource) const { + rtc::scoped_refptr resource) const { + rtc::CritScope crit(&resource_lock_); const auto& registered_resource = absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) { - return r.resource == &resource; + return r.resource == resource; }); RTC_DCHECK(registered_resource != resources_.end()) - << resource.name() << " not found."; + << resource->name() << " not found."; return registered_resource->reason; } @@ -468,6 +520,7 @@ VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource( // remotely cope with the load right now. CpuOveruseOptions VideoStreamEncoderResourceManager::GetCpuOveruseOptions() const { + RTC_DCHECK_RUN_ON(encoder_queue_); // This is already ensured by the only caller of this method: // StartResourceAdaptation(). RTC_DCHECK(encoder_settings_.has_value()); @@ -485,6 +538,7 @@ CpuOveruseOptions VideoStreamEncoderResourceManager::GetCpuOveruseOptions() } int VideoStreamEncoderResourceManager::LastInputFrameSizeOrDefault() const { + RTC_DCHECK_RUN_ON(encoder_queue_); return input_state_provider_->InputState().frame_size_pixels().value_or( kDefaultInputPixelsWidth * kDefaultInputPixelsHeight); } @@ -492,7 +546,11 @@ int VideoStreamEncoderResourceManager::LastInputFrameSizeOrDefault() const { void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - const Resource* reason) { + rtc::scoped_refptr reason) { + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // ensure that this is running on it instead, and PostTask back to the encoder + // queue if need be. + RTC_DCHECK_RUN_ON(encoder_queue_); video_source_restrictions_ = restrictions; VideoAdaptationCounters previous_adaptation_counters = active_counts_[VideoAdaptationReason::kQuality] + @@ -504,7 +562,7 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( // to be updated every time the adaptation counter is incremented or // decremented due to a resource. RTC_DCHECK_EQ(adaptation_counters_total_abs_diff, 1); - VideoAdaptationReason reason_type = GetReasonFromResource(*reason); + VideoAdaptationReason reason_type = GetReasonFromResource(reason); UpdateAdaptationStats(adaptation_counters, reason_type); } else if (adaptation_counters.Total() == 0) { // Adaptation was manually reset - clear the per-reason counters too. @@ -521,6 +579,7 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( } void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() { + RTC_DCHECK_RUN_ON(encoder_queue_); absl::optional codec_max_frame_rate = encoder_settings_.has_value() ? absl::optional( @@ -537,7 +596,7 @@ void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() { codec_max_frame_rate.value() < target_frame_rate.value())) { target_frame_rate = codec_max_frame_rate; } - encode_usage_resource_.SetTargetFrameRate(target_frame_rate); + encode_usage_resource_->SetTargetFrameRate(target_frame_rate); } void VideoStreamEncoderResourceManager::OnAdaptationCountChanged( @@ -606,6 +665,7 @@ void VideoStreamEncoderResourceManager::OnAdaptationCountChanged( void VideoStreamEncoderResourceManager::UpdateAdaptationStats( const VideoAdaptationCounters& total_counts, VideoAdaptationReason reason) { + RTC_DCHECK_RUN_ON(encoder_queue_); // Update active counts VideoAdaptationCounters& active_count = active_counts_[reason]; VideoAdaptationCounters& other_active = active_counts_[OtherReason(reason)]; @@ -618,12 +678,13 @@ void VideoStreamEncoderResourceManager::UpdateAdaptationStats( } void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const { + RTC_DCHECK_RUN_ON(encoder_queue_); VideoStreamEncoderObserver::AdaptationSettings cpu_settings( IsResolutionScalingEnabled(degradation_preference_), IsFramerateScalingEnabled(degradation_preference_)); VideoStreamEncoderObserver::AdaptationSettings quality_settings = - quality_scaler_resource_.is_started() + quality_scaler_resource_->is_started() ? cpu_settings : VideoStreamEncoderObserver::AdaptationSettings(); encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings, @@ -631,7 +692,8 @@ void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const { } void VideoStreamEncoderResourceManager::MaybePerformQualityRampupExperiment() { - if (!quality_scaler_resource_.is_started()) + RTC_DCHECK_RUN_ON(encoder_queue_); + if (!quality_scaler_resource_->is_started()) return; if (quality_rampup_done_) @@ -648,7 +710,7 @@ void VideoStreamEncoderResourceManager::MaybePerformQualityRampupExperiment() { if (encoder_settings_ && encoder_target_bitrate_bps_.value_or(0) == encoder_settings_->video_codec().maxBitrate * 1000 && - quality_scaler_resource_.QpFastFilterLow()) { + quality_scaler_resource_->QpFastFilterLow()) { try_quality_rampup = true; } } @@ -667,12 +729,14 @@ void VideoStreamEncoderResourceManager::MaybePerformQualityRampupExperiment() { } void VideoStreamEncoderResourceManager::ResetActiveCounts() { + RTC_DCHECK_RUN_ON(encoder_queue_); active_counts_.clear(); active_counts_[VideoAdaptationReason::kCpu] = VideoAdaptationCounters(); active_counts_[VideoAdaptationReason::kQuality] = VideoAdaptationCounters(); } std::string VideoStreamEncoderResourceManager::ActiveCountsToString() const { + RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK_EQ(2, active_counts_.size()); rtc::StringBuilder ss; diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index d0e5455529..43364a8fff 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -20,6 +20,7 @@ #include "absl/types/optional.h" #include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" #include "api/video/video_adaptation_counters.h" #include "api/video/video_adaptation_reason.h" #include "api/video/video_frame.h" @@ -35,6 +36,7 @@ #include "rtc_base/experiments/quality_rampup_experiment.h" #include "rtc_base/experiments/quality_scaler_settings.h" #include "rtc_base/strings/string_builder.h" +#include "rtc_base/task_queue.h" #include "system_wrappers/include/clock.h" #include "video/adaptation/encode_usage_resource.h" #include "video/adaptation/overuse_frame_detector.h" @@ -61,16 +63,23 @@ class VideoStreamEncoderResourceManager public: VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, - ResourceAdaptationProcessorInterface* adaptation_processor, VideoStreamEncoderObserver* encoder_stats_observer, Clock* clock, bool experiment_cpu_load_estimator, std::unique_ptr overuse_detector); ~VideoStreamEncoderResourceManager() override; - void SetDegradationPreferences( - DegradationPreference degradation_preference, - DegradationPreference effective_degradation_preference); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // pass it in here. + void Initialize(rtc::TaskQueue* encoder_queue); + void SetAdaptationProcessor( + ResourceAdaptationProcessorInterface* adaptation_processor); + + // TODO(https://crbug.com/webrtc/11563): The degradation preference is a + // setting of the Processor, it does not belong to the Manager - can we get + // rid of this? + void SetDegradationPreferences(DegradationPreference degradation_preference); + DegradationPreference degradation_preference() const; // Starts the encode usage resource. The quality scaler resource is // automatically started on being configured. @@ -103,9 +112,11 @@ class VideoStreamEncoderResourceManager // - Legacy getStats() purposes. // - Preventing adapting up in some circumstances (which may be questionable). // TODO(hbos): Can we get rid of this? - void MapResourceToReason(Resource* resource, VideoAdaptationReason reason); - std::vector MappedResources() const; - QualityScalerResource* quality_scaler_resource_for_testing(); + void MapResourceToReason(rtc::scoped_refptr resource, + VideoAdaptationReason reason); + std::vector> MappedResources() const; + rtc::scoped_refptr + quality_scaler_resource_for_testing(); // If true, the VideoStreamEncoder should eexecute its logic to maybe drop // frames baseed on size and bitrate. bool DropInitialFrames() const; @@ -115,7 +126,7 @@ class VideoStreamEncoderResourceManager void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - const Resource* reason) override; + rtc::scoped_refptr reason) override; // For reasons of adaptation and statistics, we not only count the total // number of adaptations, but we also count the number of adaptations per @@ -132,15 +143,12 @@ class VideoStreamEncoderResourceManager private: class InitialFrameDropper; - VideoAdaptationReason GetReasonFromResource(const Resource& resource) const; + VideoAdaptationReason GetReasonFromResource( + rtc::scoped_refptr resource) const; CpuOveruseOptions GetCpuOveruseOptions() const; int LastInputFrameSizeOrDefault() const; - // Makes |video_source_restrictions_| up-to-date and informs the - // |adaptation_listener_| if restrictions are changed, allowing the listener - // to reconfigure the source accordingly. - void MaybeUpdateVideoSourceRestrictions(const Resource* reason_resource); // Calculates an up-to-date value of the target frame rate and informs the // |encode_usage_resource_| of the new value. void MaybeUpdateTargetFrameRate(); @@ -168,7 +176,8 @@ class VideoStreamEncoderResourceManager // Does not trigger adaptations, only prevents adapting up based on // |active_counts_|. - class PreventAdaptUpDueToActiveCounts final : public Resource { + class PreventAdaptUpDueToActiveCounts final + : public rtc::RefCountedObject { public: explicit PreventAdaptUpDueToActiveCounts( VideoStreamEncoderResourceManager* manager); @@ -182,14 +191,15 @@ class VideoStreamEncoderResourceManager const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const override; + rtc::scoped_refptr reason_resource) const override; private: VideoStreamEncoderResourceManager* manager_; - } prevent_adapt_up_due_to_active_counts_; + }; // Does not trigger adaptations, only prevents adapting up resolution. - class PreventIncreaseResolutionDueToBitrateResource final : public Resource { + class PreventIncreaseResolutionDueToBitrateResource final + : public rtc::RefCountedObject { public: explicit PreventIncreaseResolutionDueToBitrateResource( VideoStreamEncoderResourceManager* manager); @@ -203,14 +213,15 @@ class VideoStreamEncoderResourceManager const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const override; + rtc::scoped_refptr reason_resource) const override; private: VideoStreamEncoderResourceManager* manager_; - } prevent_increase_resolution_due_to_bitrate_resource_; + }; // Does not trigger adaptations, only prevents adapting up in BALANCED. - class PreventAdaptUpInBalancedResource final : public Resource { + class PreventAdaptUpInBalancedResource final + : public rtc::RefCountedObject { public: explicit PreventAdaptUpInBalancedResource( VideoStreamEncoderResourceManager* manager); @@ -224,54 +235,74 @@ class VideoStreamEncoderResourceManager const VideoStreamInputState& input_state, const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, - const Resource& reason_resource) const override; + rtc::scoped_refptr reason_resource) const override; private: VideoStreamEncoderResourceManager* manager_; - } prevent_adapt_up_in_balanced_resource_; + }; - EncodeUsageResource encode_usage_resource_; - QualityScalerResource quality_scaler_resource_; + const rtc::scoped_refptr + prevent_adapt_up_due_to_active_counts_; + const rtc::scoped_refptr + prevent_increase_resolution_due_to_bitrate_resource_; + const rtc::scoped_refptr + prevent_adapt_up_in_balanced_resource_; + const rtc::scoped_refptr encode_usage_resource_; + const rtc::scoped_refptr quality_scaler_resource_; - VideoStreamInputStateProvider* const input_state_provider_; - ResourceAdaptationProcessorInterface* const adaptation_processor_; - VideoStreamEncoderObserver* const encoder_stats_observer_; + rtc::TaskQueue* encoder_queue_; + VideoStreamInputStateProvider* const input_state_provider_ + RTC_GUARDED_BY(encoder_queue_); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // guard the processor by it instead. + ResourceAdaptationProcessorInterface* adaptation_processor_ + RTC_GUARDED_BY(encoder_queue_); + VideoStreamEncoderObserver* const encoder_stats_observer_ + RTC_GUARDED_BY(encoder_queue_); - DegradationPreference degradation_preference_; - DegradationPreference effective_degradation_preference_; - VideoSourceRestrictions video_source_restrictions_; + DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); + VideoSourceRestrictions video_source_restrictions_ + RTC_GUARDED_BY(encoder_queue_); - const BalancedDegradationSettings balanced_settings_; - Clock* clock_; - const bool experiment_cpu_load_estimator_; - const std::unique_ptr initial_frame_dropper_; - const bool quality_scaling_experiment_enabled_; - absl::optional encoder_target_bitrate_bps_; - absl::optional encoder_rates_; - bool quality_rampup_done_; - QualityRampupExperiment quality_rampup_experiment_; - absl::optional encoder_settings_; + const BalancedDegradationSettings balanced_settings_ + RTC_GUARDED_BY(encoder_queue_); + Clock* clock_ RTC_GUARDED_BY(encoder_queue_); + const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_); + const std::unique_ptr initial_frame_dropper_ + RTC_GUARDED_BY(encoder_queue_); + const bool quality_scaling_experiment_enabled_ RTC_GUARDED_BY(encoder_queue_); + absl::optional encoder_target_bitrate_bps_ + RTC_GUARDED_BY(encoder_queue_); + absl::optional encoder_rates_ + RTC_GUARDED_BY(encoder_queue_); + bool quality_rampup_done_ RTC_GUARDED_BY(encoder_queue_); + QualityRampupExperiment quality_rampup_experiment_ + RTC_GUARDED_BY(encoder_queue_); + absl::optional encoder_settings_ + RTC_GUARDED_BY(encoder_queue_); // Ties a resource to a reason for statistical reporting. This AdaptReason is // also used by this module to make decisions about how to adapt up/down. struct ResourceAndReason { - ResourceAndReason(Resource* resource, VideoAdaptationReason reason) + ResourceAndReason(rtc::scoped_refptr resource, + VideoAdaptationReason reason) : resource(resource), reason(reason) {} virtual ~ResourceAndReason() = default; - Resource* const resource; + const rtc::scoped_refptr resource; const VideoAdaptationReason reason; }; - std::vector resources_; + rtc::CriticalSection resource_lock_; + std::vector resources_ RTC_GUARDED_BY(&resource_lock_); // One AdaptationCounter for each reason, tracking the number of times we have // adapted for each reason. The sum of active_counts_ MUST always equal the // total adaptation provided by the VideoSourceRestrictions. - // TODO(https://crbug.com/webrtc/11392): Move all active count logic to - // encoder_stats_observer_; Counters used for deciding if the video resolution - // or framerate is currently restricted, and if so, why, on a per degradation - // preference basis. + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // guard the activec counts by it instead. The |encoder_stats_observer_| is + // thread-safe anyway, and active counts are used by + // PreventAdaptUpDueToActiveCounts to make decisions. std::unordered_map - active_counts_; + active_counts_ RTC_GUARDED_BY(encoder_queue_); }; } // namespace webrtc diff --git a/video/video_source_sink_controller.h b/video/video_source_sink_controller.h index 4811b2866e..68fef3f071 100644 --- a/video/video_source_sink_controller.h +++ b/video/video_source_sink_controller.h @@ -53,9 +53,6 @@ class VideoSourceSinkController { rtc::VideoSinkWants CurrentSettingsToSinkWants() const RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); - // TODO(hbos): If everything is handled on the same sequence (i.e. - // VideoStreamEncoder's encoder queue) then |crit_| can be replaced by - // sequence checker. Investigate if we want to do this. mutable rtc::CriticalSection crit_; rtc::VideoSinkInterface* const sink_; rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(&crit_); diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index c69cf1e731..bb779ff186 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -262,7 +262,6 @@ VideoStreamEncoder::VideoStreamEncoder( &input_state_provider_, encoder_stats_observer)), stream_resource_manager_(&input_state_provider_, - resource_adaptation_processor_.get(), encoder_stats_observer, clock_, settings_.experiment_cpu_load_estimator, @@ -275,13 +274,25 @@ VideoStreamEncoder::VideoStreamEncoder( RTC_DCHECK(encoder_stats_observer); RTC_DCHECK_GE(number_of_cores, 1); - resource_adaptation_processor_->AddAdaptationListener( - &stream_resource_manager_); - resource_adaptation_processor_->AddAdaptationListener(this); + stream_resource_manager_.Initialize(&encoder_queue_); - // Add the stream resource manager's resources to the processor. - for (Resource* resource : stream_resource_manager_.MappedResources()) - resource_adaptation_processor_->AddResource(resource); + rtc::Event initialize_processor_event; + encoder_queue_.PostTask([this, &initialize_processor_event] { + RTC_DCHECK_RUN_ON(&encoder_queue_); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // initialize the processor on it instead. + resource_adaptation_processor_->InitializeOnResourceAdaptationQueue(); + stream_resource_manager_.SetAdaptationProcessor( + resource_adaptation_processor_.get()); + resource_adaptation_processor_->AddAdaptationListener( + &stream_resource_manager_); + resource_adaptation_processor_->AddAdaptationListener(this); + // Add the stream resource manager's resources to the processor. + for (Resource* resource : stream_resource_manager_.MappedResources()) + resource_adaptation_processor_->AddResource(resource); + initialize_processor_event.Set(); + }); + initialize_processor_event.Wait(rtc::Event::kForever); for (auto& state : encoder_buffer_state_) state.fill(std::numeric_limits::max()); @@ -298,14 +309,25 @@ void VideoStreamEncoder::Stop() { video_source_sink_controller_.SetSource(nullptr); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // destroy the processor on it instead. + if (resource_adaptation_processor_) { + resource_adaptation_processor_->StopResourceAdaptation(); + for (Resource* resource : stream_resource_manager_.MappedResources()) { + resource_adaptation_processor_->RemoveResource(resource); + } + resource_adaptation_processor_->RemoveAdaptationListener(this); + resource_adaptation_processor_->RemoveAdaptationListener( + &stream_resource_manager_); + stream_resource_manager_.SetAdaptationProcessor(nullptr); + resource_adaptation_processor_.reset(); + } stream_resource_manager_.StopManagedResources(); - resource_adaptation_processor_->StopResourceAdaptation(); rate_allocator_ = nullptr; bitrate_observer_ = nullptr; ReleaseEncoder(); shutdown_event_.Set(); }); - shutdown_event_.Wait(rtc::Event::kForever); } @@ -336,14 +358,18 @@ void VideoStreamEncoder::SetSource( const DegradationPreference& degradation_preference) { RTC_DCHECK_RUN_ON(&thread_checker_); video_source_sink_controller_.SetSource(source); - encoder_queue_.PostTask([this, source, degradation_preference] { + input_state_provider_.OnHasInputChanged(source); + encoder_queue_.PostTask([this, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); - input_state_provider_.OnHasInputChanged(source); + if (!resource_adaptation_processor_) { + // The VideoStreamEncoder was stopped and the processor destroyed before + // this task had a chance to execute. No action needed. + return; + } resource_adaptation_processor_->SetDegradationPreference( degradation_preference); stream_resource_manager_.SetDegradationPreferences( - resource_adaptation_processor_->degradation_preference(), - resource_adaptation_processor_->effective_degradation_preference()); + resource_adaptation_processor_->degradation_preference()); if (encoder_) { stream_resource_manager_.ConfigureQualityScaler( encoder_->GetEncoderInfo()); @@ -670,10 +696,18 @@ void VideoStreamEncoder::ReconfigureEncoder() { } if (pending_encoder_creation_) { + // TODO(hbos): Stopping and restarting for backwards compatibility reasons. + // We may be able to change this to "EnsureStarted()" if it took care of + // reconfiguring the QualityScaler as well. (ConfigureQualityScaler() is + // invoked later in this method.) stream_resource_manager_.StopManagedResources(); - resource_adaptation_processor_->StopResourceAdaptation(); stream_resource_manager_.StartEncodeUsageResource(); - resource_adaptation_processor_->StartResourceAdaptation(); + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // PostTask ensuring it is started. + if (resource_adaptation_processor_) { + // Ensures started. If already started this is a NO-OP. + resource_adaptation_processor_->StartResourceAdaptation(); + } pending_encoder_creation_ = false; } @@ -755,8 +789,7 @@ void VideoStreamEncoder::OnEncoderSettingsChanged() { resource_adaptation_processor_->SetIsScreenshare( encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen); stream_resource_manager_.SetDegradationPreferences( - resource_adaptation_processor_->degradation_preference(), - resource_adaptation_processor_->effective_degradation_preference()); + resource_adaptation_processor_->degradation_preference()); input_state_provider_.OnEncoderSettingsChanged(encoder_settings); stream_resource_manager_.SetEncoderSettings(encoder_settings); } @@ -1683,7 +1716,9 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - const Resource* reason) { + rtc::scoped_refptr reason) { + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // ensure that this is running on it instead. RTC_DCHECK_RUN_ON(&encoder_queue_); video_source_sink_controller_.SetRestrictions(std::move(restrictions)); video_source_sink_controller_.PushSourceSinkSettings(); @@ -1887,7 +1922,7 @@ void VideoStreamEncoder::CheckForAnimatedContent( if (!automatic_animation_detection_experiment_.enabled || encoder_config_.content_type != VideoEncoderConfig::ContentType::kScreen || - resource_adaptation_processor_->degradation_preference() != + stream_resource_manager_.degradation_preference() != DegradationPreference::BALANCED) { return; } @@ -1952,13 +1987,19 @@ void VideoStreamEncoder::CheckForAnimatedContent( } } void VideoStreamEncoder::InjectAdaptationResource( - Resource* resource, + rtc::scoped_refptr resource, VideoAdaptationReason reason) { - stream_resource_manager_.MapResourceToReason(resource, reason); - resource_adaptation_processor_->AddResource(resource); + rtc::Event inject_resource_event; + encoder_queue_.PostTask([this, resource, reason, &inject_resource_event] { + RTC_DCHECK_RUN_ON(&encoder_queue_); + stream_resource_manager_.MapResourceToReason(resource, reason); + resource_adaptation_processor_->AddResource(resource); + inject_resource_event.Set(); + }); + inject_resource_event.Wait(rtc::Event::kForever); } -QualityScalerResource* +rtc::scoped_refptr VideoStreamEncoder::quality_scaler_resource_for_testing() { RTC_DCHECK_RUN_ON(&encoder_queue_); return stream_resource_manager_.quality_scaler_resource_for_testing(); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index a4a055dbd3..61943551dd 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -110,15 +110,16 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - const Resource* reason) override; + rtc::scoped_refptr reason) override; // Used for injected test resources. // TODO(eshr): Move all adaptation tests out of VideoStreamEncoder tests. - void InjectAdaptationResource(Resource* resource, + void InjectAdaptationResource(rtc::scoped_refptr resource, VideoAdaptationReason reason) RTC_RUN_ON(&encoder_queue_); - QualityScalerResource* quality_scaler_resource_for_testing(); + rtc::scoped_refptr + quality_scaler_resource_for_testing(); private: class VideoFrameInfo { @@ -400,27 +401,30 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_); // Provies video stream input states: current resolution and frame rate. - VideoStreamInputStateProvider input_state_provider_ - RTC_GUARDED_BY(&encoder_queue_); + // This class is thread-safe. + VideoStreamInputStateProvider input_state_provider_; // Responsible for adapting input resolution or frame rate to ensure resources // (e.g. CPU or bandwidth) are not overused. + // This class is single-threaded. + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // guard the processor by it instead. std::unique_ptr resource_adaptation_processor_ RTC_GUARDED_BY(&encoder_queue_); // Handles input, output and stats reporting related to VideoStreamEncoder // specific resources, such as "encode usage percent" measurements and "QP // scaling". Also involved with various mitigations such as inital frame // dropping. + // The manager primarily operates on the |encoder_queue_| but its lifetime is + // tied to the VideoStreamEncoder (which is destroyed off the encoder queue) + // and its resource list is accessible from any thread. + // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, + // remove the RTC_GUARDED_BY to get resources on the adaptation queue. VideoStreamEncoderResourceManager stream_resource_manager_ RTC_GUARDED_BY(&encoder_queue_); // Carries out the VideoSourceRestrictions provided by the // ResourceAdaptationProcessor, i.e. reconfigures the source of video frames // to provide us with different resolution or frame rate. - // - // Used on the |encoder_queue_| with a few exceptions: - // - VideoStreamEncoder::SetSource() invokes SetSource(). - // - VideoStreamEncoder::SetSink() invokes SetRotationApplied() and - // PushSourceSinkSettings(). - // - VideoStreamEncoder::Stop() invokes SetSource(). + // This class is thread-safe. VideoSourceSinkController video_source_sink_controller_; // All public methods are proxied to |encoder_queue_|. It must must be diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index cf9e790fcf..30660bbce6 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -178,13 +178,11 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { overuse_detector_proxy_ = new CpuOveruseDetectorProxy(stats_proxy)), task_queue_factory), - fake_cpu_resource_(std::make_unique("FakeResource[CPU]")), - fake_quality_resource_( - std::make_unique("FakeResource[QP]")) { - InjectAdaptationResource(fake_quality_resource_.get(), + fake_cpu_resource_(new FakeResource("FakeResource[CPU]")), + fake_quality_resource_(new FakeResource("FakeResource[QP]")) { + InjectAdaptationResource(fake_quality_resource_, VideoAdaptationReason::kQuality); - InjectAdaptationResource(fake_cpu_resource_.get(), - VideoAdaptationReason::kCpu); + InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu); } // This is used as a synchronisation mechanism, to make sure that the @@ -248,8 +246,8 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { } CpuOveruseDetectorProxy* overuse_detector_proxy_; - std::unique_ptr fake_cpu_resource_; - std::unique_ptr fake_quality_resource_; + rtc::scoped_refptr fake_cpu_resource_; + rtc::scoped_refptr fake_quality_resource_; }; class VideoStreamFactory