diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/resource_adaptation_processor.cc index 42d1c383cb..0be01cb794 100644 --- a/call/adaptation/resource_adaptation_processor.cc +++ b/call/adaptation/resource_adaptation_processor.cc @@ -71,7 +71,6 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( : resource_adaptation_queue_(nullptr), resource_listener_delegate_( new rtc::RefCountedObject(this)), - is_resource_adaptation_enabled_(false), input_state_provider_(input_state_provider), encoder_stats_observer_(encoder_stats_observer), resources_(), @@ -85,7 +84,6 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - RTC_DCHECK(!is_resource_adaptation_enabled_); RTC_DCHECK(restrictions_listeners_.empty()) << "There are restrictions listener(s) depending on a " << "ResourceAdaptationProcessor being destroyed."; @@ -123,26 +121,6 @@ ResourceAdaptationProcessor::effective_degradation_preference() const { return effective_degradation_preference_; } -void ResourceAdaptationProcessor::StartResourceAdaptation() { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - if (is_resource_adaptation_enabled_) - return; - for (const auto& resource : resources_) { - resource->SetResourceListener(resource_listener_delegate_); - } - is_resource_adaptation_enabled_ = true; -} - -void ResourceAdaptationProcessor::StopResourceAdaptation() { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - if (!is_resource_adaptation_enabled_) - return; - for (const auto& resource : resources_) { - resource->SetResourceListener(nullptr); - } - is_resource_adaptation_enabled_ = false; -} - void ResourceAdaptationProcessor::AddRestrictionsListener( VideoSourceRestrictionsListener* restrictions_listener) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); @@ -164,12 +142,11 @@ void ResourceAdaptationProcessor::RemoveRestrictionsListener( void ResourceAdaptationProcessor::AddResource( rtc::scoped_refptr resource) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - // TODO(hbos): Allow adding resources while |is_resource_adaptation_enabled_| - // by registering as a listener of the resource on adding it. - RTC_DCHECK(!is_resource_adaptation_enabled_); - RTC_DCHECK(std::find(resources_.begin(), resources_.end(), resource) == - resources_.end()); + RTC_DCHECK(resource); + RTC_DCHECK(absl::c_find(resources_, resource) == resources_.end()) + << "Resource \"" << resource->Name() << "\" was already registered."; resources_.push_back(resource); + resource->SetResourceListener(resource_listener_delegate_); } std::vector> @@ -181,13 +158,21 @@ ResourceAdaptationProcessor::GetResources() const { void ResourceAdaptationProcessor::RemoveResource( rtc::scoped_refptr resource) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - // TODO(hbos): Allow removing resources while - // |is_resource_adaptation_enabled_| by unregistering as a listener of the - // resource on removing it. - RTC_DCHECK(!is_resource_adaptation_enabled_); - auto it = std::find(resources_.begin(), resources_.end(), resource); - RTC_DCHECK(it != resources_.end()); + RTC_DCHECK(resource); + RTC_LOG(INFO) << "Removing resource \"" << resource->Name() << "\"."; + auto it = absl::c_find(resources_, resource); + RTC_DCHECK(it != resources_.end()) << "Resource \"" << resource->Name() + << "\" was not a registered resource."; + auto resource_adaptation_limits = + adaptation_limits_by_resources_.find(resource); + if (resource_adaptation_limits != adaptation_limits_by_resources_.end()) { + VideoStreamAdapter::RestrictionsWithCounters adaptation_limits = + resource_adaptation_limits->second; + adaptation_limits_by_resources_.erase(resource_adaptation_limits); + MaybeUpdateResourceLimitationsOnResourceRemoval(adaptation_limits); + } resources_.erase(it); + resource->SetResourceListener(nullptr); } void ResourceAdaptationProcessor::AddAdaptationConstraint( @@ -256,8 +241,7 @@ void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { stream_adapter_->ClearRestrictions(); adaptation_limits_by_resources_.clear(); for (auto restrictions_listener : restrictions_listeners_) { - restrictions_listener->OnResourceLimitationChanged( - nullptr, adaptation_limits_by_resources_); + restrictions_listener->OnResourceLimitationChanged(nullptr, {}); } MaybeUpdateVideoSourceRestrictions(nullptr); } @@ -287,6 +271,13 @@ void ResourceAdaptationProcessor::OnResourceUsageStateMeasured( rtc::scoped_refptr resource, ResourceUsageState usage_state) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_DCHECK(resource); + // |resource| could have been removed after signalling. + if (absl::c_find(resources_, resource) == resources_.end()) { + RTC_LOG(INFO) << "Ignoring signal from removed resource \"" + << resource->Name() << "\"."; + return; + } MitigationResultAndLogMessage result_and_message; switch (usage_state) { case ResourceUsageState::kOveruse: @@ -362,13 +353,10 @@ ResourceAdaptationProcessor::OnResourceUnderuse( VideoSourceRestrictions restrictions_after = peek_restrictions.restrictions; // Check that resource is most limited... std::vector> most_limited_resources; - VideoAdaptationCounters most_limited_restrictions; + VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions; std::tie(most_limited_resources, most_limited_restrictions) = FindMostLimitedResources(); - RTC_DCHECK(!most_limited_resources.empty()) - << "Can not have no limited resources when adaptation status is valid. " - "Should be kLimitReached."; for (const auto* constraint : adaptation_constraints_) { if (!constraint->IsAdaptationUpAllowed(input_state, restrictions_before, restrictions_after, @@ -383,8 +371,9 @@ ResourceAdaptationProcessor::OnResourceUnderuse( } // If the most restricted resource is less limited than current restrictions // then proceed with adapting up. - if (most_limited_restrictions.Total() >= - stream_adapter_->adaptation_counters().Total()) { + if (!most_limited_resources.empty() && + most_limited_restrictions.adaptation_counters.Total() >= + stream_adapter_->adaptation_counters().Total()) { // If |reason_resource| is not one of the most limiting resources then abort // adaptation. if (absl::c_find(most_limited_resources, reason_resource) == @@ -504,20 +493,24 @@ void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize( } } -std::pair>, VideoAdaptationCounters> +std::pair>, + VideoStreamAdapter::RestrictionsWithCounters> ResourceAdaptationProcessor::FindMostLimitedResources() const { std::vector> most_limited_resources; - VideoAdaptationCounters most_limited_restrictions; + VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions{ + VideoSourceRestrictions(), VideoAdaptationCounters()}; for (const auto& resource_and_adaptation_limit_ : adaptation_limits_by_resources_) { - const VideoAdaptationCounters& counters = + const auto& restrictions_with_counters = resource_and_adaptation_limit_.second; - if (counters.Total() > most_limited_restrictions.Total()) { - most_limited_restrictions = counters; + if (restrictions_with_counters.adaptation_counters.Total() > + most_limited_restrictions.adaptation_counters.Total()) { + most_limited_restrictions = restrictions_with_counters; most_limited_resources.clear(); most_limited_resources.push_back(resource_and_adaptation_limit_.first); - } else if (most_limited_restrictions == counters) { + } else if (most_limited_restrictions.adaptation_counters == + restrictions_with_counters.adaptation_counters) { most_limited_resources.push_back(resource_and_adaptation_limit_.first); } } @@ -529,12 +522,55 @@ void ResourceAdaptationProcessor::UpdateResourceLimitations( rtc::scoped_refptr reason_resource, const VideoStreamAdapter::RestrictionsWithCounters& peek_next_restrictions) { - adaptation_limits_by_resources_[reason_resource] = - peek_next_restrictions.adaptation_counters; + adaptation_limits_by_resources_[reason_resource] = peek_next_restrictions; + + std::map, VideoAdaptationCounters> limitations; + for (const auto& p : adaptation_limits_by_resources_) { + limitations.insert(std::make_pair(p.first, p.second.adaptation_counters)); + } for (auto restrictions_listener : restrictions_listeners_) { - restrictions_listener->OnResourceLimitationChanged( - reason_resource, adaptation_limits_by_resources_); + restrictions_listener->OnResourceLimitationChanged(reason_resource, + limitations); + } +} + +void ResourceAdaptationProcessor:: + MaybeUpdateResourceLimitationsOnResourceRemoval( + VideoStreamAdapter::RestrictionsWithCounters removed_limitations) { + if (adaptation_limits_by_resources_.empty()) { + // Only the resource being removed was adapted so reset restrictions. + ResetVideoSourceRestrictions(); + return; + } + + VideoStreamAdapter::RestrictionsWithCounters most_limited = + FindMostLimitedResources().second; + + if (removed_limitations.adaptation_counters.Total() <= + most_limited.adaptation_counters.Total()) { + // The removed limitations were less limited than the most limited resource. + // Don't change the current restrictions. + return; + } + + // Apply the new most limited resource as the next restrictions. + Adaptation adapt_to = stream_adapter_->GetAdaptationTo( + most_limited.adaptation_counters, most_limited.restrictions); + RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid); + stream_adapter_->ApplyAdaptation(adapt_to); + + RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to " + "next most limited restrictions: " + << most_limited.restrictions.ToString() << " with counters " + << most_limited.adaptation_counters.ToString(); + + MaybeUpdateVideoSourceRestrictions(nullptr); + auto input_state = input_state_provider_->InputState(); + for (auto* adaptation_listener : adaptation_listeners_) { + adaptation_listener->OnAdaptationApplied( + input_state, removed_limitations.restrictions, + most_limited.restrictions, nullptr); } } diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h index 6c35e0f4d0..cff50955e7 100644 --- a/call/adaptation/resource_adaptation_processor.h +++ b/call/adaptation/resource_adaptation_processor.h @@ -65,8 +65,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, DegradationPreference degradation_preference() const override; DegradationPreference effective_degradation_preference() const override; - void StartResourceAdaptation() override; - void StopResourceAdaptation() override; void AddRestrictionsListener( VideoSourceRestrictionsListener* restrictions_listener) override; void RemoveRestrictionsListener( @@ -165,13 +163,16 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // resource performing the adaptation is the only most limited resource. This // function returns the list of all most limited resources as well as the // corresponding adaptation of that resource. - std::pair>, VideoAdaptationCounters> + std::pair>, + VideoStreamAdapter::RestrictionsWithCounters> FindMostLimitedResources() const RTC_RUN_ON(resource_adaptation_queue_); + void MaybeUpdateResourceLimitationsOnResourceRemoval( + VideoStreamAdapter::RestrictionsWithCounters removed_limitations) + RTC_RUN_ON(resource_adaptation_queue_); + TaskQueueBase* resource_adaptation_queue_; rtc::scoped_refptr resource_listener_delegate_; - bool is_resource_adaptation_enabled_ - RTC_GUARDED_BY(resource_adaptation_queue_); // Input and output. VideoStreamInputStateProvider* const input_state_provider_ RTC_GUARDED_BY(resource_adaptation_queue_); @@ -186,7 +187,8 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, std::vector adaptation_listeners_ RTC_GUARDED_BY(resource_adaptation_queue_); // Purely used for statistics, does not ensure mapped resources stay alive. - std::map, VideoAdaptationCounters> + std::map, + VideoStreamAdapter::RestrictionsWithCounters> adaptation_limits_by_resources_ RTC_GUARDED_BY(resource_adaptation_queue_); // Adaptation strategy settings. diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h index a2d1a24d40..a97fe8efe4 100644 --- a/call/adaptation/resource_adaptation_processor_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h @@ -74,8 +74,6 @@ class ResourceAdaptationProcessorInterface { // with AddResource() and RemoveResource() instead. When the processor is // multi-stream aware, stream-specific resouces will get added and removed // over time. - virtual void StartResourceAdaptation() = 0; - virtual void StopResourceAdaptation() = 0; virtual void AddRestrictionsListener( VideoSourceRestrictionsListener* restrictions_listener) = 0; virtual void RemoveRestrictionsListener( diff --git a/call/adaptation/resource_adaptation_processor_unittest.cc b/call/adaptation/resource_adaptation_processor_unittest.cc index d7fbc88a44..da1ab1cda1 100644 --- a/call/adaptation/resource_adaptation_processor_unittest.cc +++ b/call/adaptation/resource_adaptation_processor_unittest.cc @@ -45,19 +45,19 @@ class VideoSourceRestrictionsListenerForTesting ~VideoSourceRestrictionsListenerForTesting() override {} size_t restrictions_updated_count() const { - rtc::CritScope crit(&lock_); + RTC_DCHECK_RUN_ON(&sequence_checker_); return restrictions_updated_count_; } VideoSourceRestrictions restrictions() const { - rtc::CritScope crit(&lock_); + RTC_DCHECK_RUN_ON(&sequence_checker_); return restrictions_; } VideoAdaptationCounters adaptation_counters() const { - rtc::CritScope crit(&lock_); + RTC_DCHECK_RUN_ON(&sequence_checker_); return adaptation_counters_; } rtc::scoped_refptr reason() const { - rtc::CritScope crit(&lock_); + RTC_DCHECK_RUN_ON(&sequence_checker_); return reason_; } @@ -66,7 +66,7 @@ class VideoSourceRestrictionsListenerForTesting VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, rtc::scoped_refptr reason) override { - rtc::CritScope crit(&lock_); + RTC_DCHECK_RUN_ON(&sequence_checker_); ++restrictions_updated_count_; restrictions_ = restrictions; adaptation_counters_ = adaptation_counters; @@ -74,11 +74,12 @@ class VideoSourceRestrictionsListenerForTesting } private: - rtc::CriticalSection lock_; - size_t restrictions_updated_count_ RTC_GUARDED_BY(lock_); - VideoSourceRestrictions restrictions_ RTC_GUARDED_BY(lock_); - VideoAdaptationCounters adaptation_counters_ RTC_GUARDED_BY(lock_); - rtc::scoped_refptr reason_ RTC_GUARDED_BY(lock_); + SequenceChecker sequence_checker_; + size_t restrictions_updated_count_ RTC_GUARDED_BY(&sequence_checker_); + VideoSourceRestrictions restrictions_ RTC_GUARDED_BY(&sequence_checker_); + VideoAdaptationCounters adaptation_counters_ + RTC_GUARDED_BY(&sequence_checker_); + rtc::scoped_refptr reason_ RTC_GUARDED_BY(&sequence_checker_); }; class ResourceAdaptationProcessorTest : public ::testing::Test { @@ -121,15 +122,22 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { } void DestroyProcessor() { - processor_->StopResourceAdaptation(); processor_->RemoveRestrictionsListener(&restrictions_listener_); - processor_->RemoveResource(resource_); - processor_->RemoveResource(other_resource_); + if (resource_) { + processor_->RemoveResource(resource_); + } + if (other_resource_) { + processor_->RemoveResource(other_resource_); + } processor_->RemoveAdaptationConstraint(&adaptation_constraint_); processor_->RemoveAdaptationListener(&adaptation_listener_); processor_.reset(); } + static void WaitUntilTaskQueueIdle() { + ASSERT_TRUE(rtc::Thread::Current()->ProcessMessages(0)); + } + protected: FakeFrameRateProvider frame_rate_provider_; VideoStreamInputStateProvider input_state_provider_; @@ -149,7 +157,6 @@ TEST_F(ResourceAdaptationProcessorTest, DisabledByDefault) { EXPECT_EQ(DegradationPreference::DISABLED, processor_->effective_degradation_preference()); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - processor_->StartResourceAdaptation(); // Adaptation does not happen when disabled. resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); @@ -158,7 +165,6 @@ TEST_F(ResourceAdaptationProcessorTest, DisabledByDefault) { TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); // Adaptation does not happen if input is insufficient. // When frame size is missing (OnFrameSizeObserved not called yet). input_state_provider_.OnHasInputChanged(true); @@ -179,7 +185,6 @@ TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingResolutionInMaintainFrameRate) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); @@ -191,7 +196,6 @@ TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingFrameRateInMaintainResolution) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_RESOLUTION); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); @@ -202,7 +206,6 @@ TEST_F(ResourceAdaptationProcessorTest, TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingFrameRateAndResolutionInBalanced) { processor_->SetDegradationPreference(DegradationPreference::BALANCED); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); // Adapting multiple times eventually resticts both frame rate and // resolution. Exactly many times we need to adapt depends on @@ -222,7 +225,6 @@ TEST_F(ResourceAdaptationProcessorTest, TEST_F(ResourceAdaptationProcessorTest, AwaitingPreviousAdaptation) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); @@ -235,7 +237,6 @@ TEST_F(ResourceAdaptationProcessorTest, AwaitingPreviousAdaptation) { TEST_F(ResourceAdaptationProcessorTest, CannotAdaptUpWhenUnrestricted) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kUnderuse); EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); @@ -244,7 +245,6 @@ TEST_F(ResourceAdaptationProcessorTest, CannotAdaptUpWhenUnrestricted) { TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); @@ -257,7 +257,6 @@ TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) { TEST_F(ResourceAdaptationProcessorTest, ResourcesCanPreventAdaptingUp) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); // Adapt down so that we can adapt up. resource_->SetUsageState(ResourceUsageState::kOveruse); @@ -273,7 +272,6 @@ TEST_F(ResourceAdaptationProcessorTest, ResourcesCanNotAdaptUpIfNeverAdaptedDown) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); @@ -288,7 +286,6 @@ TEST_F(ResourceAdaptationProcessorTest, ResourcesCanNotAdaptUpIfNotAdaptedDownAfterReset) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); @@ -308,7 +305,6 @@ TEST_F(ResourceAdaptationProcessorTest, TEST_F(ResourceAdaptationProcessorTest, OnlyMostLimitedResourceMayAdaptUp) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); @@ -339,7 +335,6 @@ TEST_F(ResourceAdaptationProcessorTest, MultipleResourcesCanTriggerMultipleAdaptations) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); @@ -398,7 +393,6 @@ TEST_F(ResourceAdaptationProcessorTest, MostLimitedResourceAdaptationWorksAfterChangingDegradataionPreference) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); // Adapt down until we can't anymore. resource_->SetUsageState(ResourceUsageState::kOveruse); @@ -431,7 +425,6 @@ TEST_F(ResourceAdaptationProcessorTest, TEST_F(ResourceAdaptationProcessorTest, AdaptingTriggersOnAdaptationApplied) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1u, adaptation_listener_.num_adaptations_applied()); @@ -441,7 +434,6 @@ TEST_F(ResourceAdaptationProcessorTest, AdaptsDownWhenOtherResourceIsAlwaysUnderused) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); other_resource_->SetUsageState(ResourceUsageState::kUnderuse); // Does not trigger adapataion because there's no restriction. @@ -463,7 +455,6 @@ TEST_F(ResourceAdaptationProcessorTest, TriggerOveruseNotOnAdaptationTaskQueue) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); TaskQueueForTest resource_task_queue("ResourceTaskQueue"); @@ -478,7 +469,6 @@ TEST_F(ResourceAdaptationProcessorTest, DestroyProcessorWhileResourceListenerDelegateHasTaskInFlight) { processor_->SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); // Wait for |resource_| to signal oversue first so we know that the delegate @@ -499,4 +489,252 @@ TEST_F(ResourceAdaptationProcessorTest, EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); } +TEST_F(ResourceAdaptationProcessorTest, + ResourceOveruseIgnoredWhenSignalledDuringRemoval) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + rtc::Event overuse_event; + TaskQueueForTest resource_task_queue("ResourceTaskQueue"); + // Queues task for |resource_| overuse while |processor_| is still listening. + resource_task_queue.PostTask(ToQueuedTask([&]() { + resource_->SetUsageState(ResourceUsageState::kOveruse); + overuse_event.Set(); + })); + EXPECT_TRUE(overuse_event.Wait(kDefaultTimeoutMs)); + // Once we know the overuse task is queued, remove |resource_| so that + // |processor_| is not listening to it. + processor_->RemoveResource(resource_); + + // Runs the queued task so |processor_| gets signalled kOveruse from + // |resource_| even though |processor_| was not listening. + WaitUntilTaskQueueIdle(); + + // No restrictions should change even though |resource_| signaled |kOveruse|. + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingOnlyAdaptedResourceResetsAdaptation) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + processor_->RemoveResource(resource_); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsAdaptationToNextLimitedLevel) { + processor_->SetDegradationPreference(DegradationPreference::BALANCED); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited |resource_| should revert us back to + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsAdaptationIfInputStateUnchanged) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + // Overuse twice and underuse once. After the underuse we don't restrict the + // source. Normally this would block future underuses. + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited |resource_| should revert us back to, even though we + // did not call RestrictSource() after |resource_| was overused. Normally + // adaptation for MAINTAIN_FRAMERATE would be blocked here but for removal we + // allow this anyways. + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingResourceNotMostLimitedHasNoEffectOnLimitations) { + processor_->SetDegradationPreference(DegradationPreference::BALANCED); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + VideoSourceRestrictions current_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters current_counters = + restrictions_listener_.adaptation_counters(); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited |resource_| should revert us back to + processor_->RemoveResource(other_resource_); + EXPECT_EQ(current_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(current_counters, restrictions_listener_.adaptation_counters()); + + // Delete |other_resource_| for cleanup. + other_resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceAfterSwitchingDegradationPreferences) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_RESOLUTION); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Revert to |other_resource_| when removing |resource_| even though the + // degradation preference was different when it was overused. + processor_->RemoveResource(resource_); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // After switching back to MAINTAIN_FRAMERATE, the next most limited settings + // are restored. + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsNextLimitationsInDisabled) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + processor_->SetDegradationPreference(DegradationPreference::DISABLED); + + // Revert to |other_resource_| when removing |resource_| even though the + // current degradataion preference is disabled. + processor_->RemoveResource(resource_); + + // After switching back to MAINTAIN_FRAMERATE, the next most limited settings + // are restored. + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovedResourceSignalsIgnoredByProcessor) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + processor_->RemoveResource(resource_); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingResourceWhenMultipleMostLimtedHasNoEffect) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + // Adapt |resource_| up and then down so that both resource's are most + // limited at 1 adaptation. + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + // Removing |resource_| has no effect since both |resource_| and + // |other_resource_| are most limited. + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + } // namespace webrtc diff --git a/call/adaptation/video_stream_adapter.cc b/call/adaptation/video_stream_adapter.cc index 7f47128ab3..4bf236fe71 100644 --- a/call/adaptation/video_stream_adapter.cc +++ b/call/adaptation/video_stream_adapter.cc @@ -123,8 +123,15 @@ const char* Adaptation::StatusToString(Adaptation::Status status) { } } -Adaptation::Step::Step(StepType type, int target) - : type(type), target(target) {} +Adaptation::Step::Step(StepType type, int target) : type(type), target(target) { + RTC_DCHECK_NE(type, Adaptation::StepType::kForce); +} + +Adaptation::Step::Step(VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters) + : type(Adaptation::StepType::kForce), + restrictions(restrictions), + counters(counters) {} Adaptation::Adaptation(int validation_id, Step step) : validation_id_(validation_id), @@ -188,6 +195,12 @@ class VideoStreamAdapter::VideoSourceRestrictor { adaptations_ = VideoAdaptationCounters(); } + void ForceRestrictions(const VideoSourceRestrictions& restrictions, + const VideoAdaptationCounters& counters) { + source_restrictions_ = restrictions; + adaptations_ = counters; + } + void set_min_pixels_per_frame(int min_pixels_per_frame) { min_pixels_per_frame_ = min_pixels_per_frame; } @@ -227,13 +240,16 @@ class VideoStreamAdapter::VideoSourceRestrictor { DegradationPreference degradation_preference) { switch (step.type) { case Adaptation::StepType::kIncreaseResolution: - IncreaseResolutionTo(step.target); + RTC_DCHECK(step.target); + IncreaseResolutionTo(step.target.value()); break; case Adaptation::StepType::kDecreaseResolution: - DecreaseResolutionTo(step.target); + RTC_DCHECK(step.target); + DecreaseResolutionTo(step.target.value()); break; case Adaptation::StepType::kIncreaseFrameRate: - IncreaseFrameRateTo(step.target); + RTC_DCHECK(step.target); + IncreaseFrameRateTo(step.target.value()); // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps. // GetAdaptationUp() should tell us the correct value, but BALANCED // logic in DecrementFramerate() makes it hard to predict whether this @@ -247,7 +263,13 @@ class VideoStreamAdapter::VideoSourceRestrictor { } break; case Adaptation::StepType::kDecreaseFrameRate: - DecreaseFrameRateTo(step.target); + RTC_DCHECK(step.target); + DecreaseFrameRateTo(step.target.value()); + break; + case Adaptation::StepType::kForce: + RTC_DCHECK(step.restrictions); + RTC_DCHECK(step.counters); + ForceRestrictions(step.restrictions.value(), step.counters.value()); break; } } @@ -542,4 +564,12 @@ void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) { degradation_preference_); } +Adaptation VideoStreamAdapter::GetAdaptationTo( + const VideoAdaptationCounters& counters, + const VideoSourceRestrictions& restrictions) const { + // Adapts up/down from the current levels so counters are equal. + return Adaptation(adaptation_validation_id_, + Adaptation::Step(restrictions, counters)); +} + } // namespace webrtc diff --git a/call/adaptation/video_stream_adapter.h b/call/adaptation/video_stream_adapter.h index 34b1281146..3a56f4f7c5 100644 --- a/call/adaptation/video_stream_adapter.h +++ b/call/adaptation/video_stream_adapter.h @@ -75,12 +75,22 @@ class Adaptation final { kDecreaseResolution, kIncreaseFrameRate, kDecreaseFrameRate, + kForce }; struct Step { Step(StepType type, int target); + // StepType is kForce + Step(VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters); const StepType type; - const int target; // Pixel or frame rate depending on |type|. + // Pixel or frame rate depending on |type|. + // Only set when |type| is not kForce. + const absl::optional target; + // Only set when |type| is kForce. + const absl::optional restrictions; + // Only set when |type| is kForce. + const absl::optional counters; }; // Constructs with a valid adaptation Step. Status is kValid. @@ -129,6 +139,8 @@ class VideoStreamAdapter { // status code indicating the reason why we cannot adapt. Adaptation GetAdaptationUp() const; Adaptation GetAdaptationDown() const; + Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, + const VideoSourceRestrictions& restrictions) const; struct RestrictionsWithCounters { VideoSourceRestrictions restrictions; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 9c17e30efb..ef02cc2343 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -324,7 +324,6 @@ void VideoStreamEncoder::Stop() { &shutdown_adaptation_processor_event] { RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); if (resource_adaptation_processor_) { - resource_adaptation_processor_->StopResourceAdaptation(); for (auto& resource : stream_resource_manager_.MappedResources()) { resource_adaptation_processor_->RemoveResource(resource); } @@ -399,11 +398,7 @@ void VideoStreamEncoder::AddAdaptationResource( // this task had a chance to execute. No action needed. return; } - // TODO(hbos): When https://webrtc-review.googlesource.com/c/src/+/176406 - // has landed, there is no need to Stop+Start when adding a resource. - resource_adaptation_processor_->StopResourceAdaptation(); resource_adaptation_processor_->AddResource(resource); - resource_adaptation_processor_->StartResourceAdaptation(); add_resource_event.Set(); }); add_resource_event.Wait(rtc::Event::kForever); @@ -788,16 +783,6 @@ void VideoStreamEncoder::ReconfigureEncoder() { // invoked later in this method.) stream_resource_manager_.StopManagedResources(); stream_resource_manager_.StartEncodeUsageResource(); - resource_adaptation_queue_.PostTask([this] { - RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); - if (!resource_adaptation_processor_) { - // The VideoStreamEncoder was stopped and the processor destroyed before - // this task had a chance to execute. No action needed. - return; - } - // Ensures started. If already started this is a NO-OP. - resource_adaptation_processor_->StartResourceAdaptation(); - }); pending_encoder_creation_ = false; }