From 381d10963ae256d3d0bce7d74a7b08f600c63439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Tue, 12 May 2020 18:49:07 +0200 Subject: [PATCH] [Adaptation] Move adaptation logic to a separate task queue. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL unblocks future Call-Level Mitigation strategies by moving the ResourceAdaptationProcessor to a separate task queue. This signifies a major milestone in the new resource adaptation architecture because with this CL the threading model is in place and moving the Processor to the Call and increasing its responsibilities is made possible. In this CL, we still have one Processor per VideoStreamEncoder and the VideoStreamEncoder is responsible for the creation and the destruction of its Processor and that Processor's task queue. But the PostTasks are in place and the decision-making is executed on a separate queue. This CL: - Moves ResourceAdaptationProcessor to an adaptation task queue. It continues to be entirely single-threaded, but now operates on a separate task queue. - Makes Resources thread-safe: Interaction with the Processor, i.e. OnResourceUsageStateMeasured() and IsAdaptationUpAllowed(), happens on the adaptation task queue. State updates are pushed from the encoder task queue with PostTasks. - QualityScalerResource operates on both task queues; the QP usage callbacks are invoked asynchronously. - The VideoStreamEncoderResourceManager operates on the encoder task queue with the following exceptions: 1) Its resources are accessible on any thread (using a mutex). This is OK because resources are reference counted and thread safe. This aids adding and removing resources to the Processor on the adaptation task queue. 2) |active_counts_| is moved to the adaptation task queue. This makes it possible for PreventAdaptUpDueToActiveCounts to run IsAdaptationUpAllowed() on the adaptation task queue. A side-effect of this is that some stats reporting now happen on the adaptation task queue, but that is OK because VideoStreamEncoderObserver is thread-safe. The Manager is updated to take the new threading model into account: - OnFrameDroppedDueToSize() posts to the adaptation task queue to invoke the Processor. - OnVideoSourceRestrictionsUpdated(), now invoked on the adaptation task queue, updates |active_counts_| synchronously but posts to the encoder task queue to update video source restrictions (which it only uses to calculate target frame rate). - MaybePerformQualityRampupExperiment() posts to the adaptation task queue to maybe reset video source restrictions on the Processor. |quality_rampup_done_| is made std::atomic. Bug: webrtc:11542, webrtc:11520 Change-Id: I1cfd76e0cd42f006a6d2527f5aa2aeb5266ba6d6 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/174441 Reviewed-by: Evan Shrubsole Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Henrik Boström Cr-Commit-Position: refs/heads/master@{#31231} --- call/adaptation/BUILD.gn | 4 + call/adaptation/resource.cc | 32 +- call/adaptation/resource.h | 14 +- .../resource_adaptation_processor_unittest.cc | 463 +++++++++++------- call/adaptation/resource_unittest.cc | 65 ++- video/adaptation/encode_usage_resource.cc | 45 +- video/adaptation/encode_usage_resource.h | 11 +- video/adaptation/quality_scaler_resource.cc | 117 +++-- video/adaptation/quality_scaler_resource.h | 34 +- .../quality_scaler_resource_unittest.cc | 13 +- .../video_stream_encoder_resource_manager.cc | 180 ++++--- .../video_stream_encoder_resource_manager.h | 36 +- video/video_stream_encoder.cc | 116 +++-- video/video_stream_encoder.h | 28 +- video/video_stream_encoder_unittest.cc | 334 ++++++++----- 15 files changed, 933 insertions(+), 559 deletions(-) diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index c54665c03e..2a6933ebd5 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -60,10 +60,14 @@ if (rtc_include_tests) { ":resource_adaptation", ":resource_adaptation_test_utilities", "../../api:scoped_refptr", + "../../api/task_queue:default_task_queue_factory", + "../../api/task_queue:task_queue", "../../api/video:video_adaptation", "../../api/video_codecs:video_codecs_api", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_task_queue", + "../../rtc_base:task_queue_for_test", "../../test:field_trial", "../../test:rtc_expect_death", "../../test:test_support", diff --git a/call/adaptation/resource.cc b/call/adaptation/resource.cc index 52343ee644..a546450bc6 100644 --- a/call/adaptation/resource.cc +++ b/call/adaptation/resource.cc @@ -17,14 +17,30 @@ namespace webrtc { ResourceListener::~ResourceListener() {} -Resource::Resource() : usage_state_(absl::nullopt), listener_(nullptr) {} +Resource::Resource() + : encoder_queue_(nullptr), + resource_adaptation_queue_(nullptr), + usage_state_(absl::nullopt), + listener_(nullptr) {} Resource::~Resource() { RTC_DCHECK(!listener_) << "There is a listener depending on a Resource being destroyed."; } +void Resource::Initialize(rtc::TaskQueue* encoder_queue, + rtc::TaskQueue* resource_adaptation_queue) { + RTC_DCHECK(!encoder_queue_); + RTC_DCHECK(encoder_queue); + RTC_DCHECK(!resource_adaptation_queue_); + RTC_DCHECK(resource_adaptation_queue); + encoder_queue_ = encoder_queue; + resource_adaptation_queue_ = resource_adaptation_queue; +} + void Resource::SetResourceListener(ResourceListener* listener) { + RTC_DCHECK(resource_adaptation_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); // If you want to change listener you need to unregister the old listener by // setting it to null first. RTC_DCHECK(!listener_ || !listener) << "A listener is already set"; @@ -32,10 +48,14 @@ void Resource::SetResourceListener(ResourceListener* listener) { } absl::optional Resource::usage_state() const { + RTC_DCHECK(resource_adaptation_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); return usage_state_; } void Resource::ClearUsageState() { + RTC_DCHECK(resource_adaptation_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); usage_state_ = absl::nullopt; } @@ -53,7 +73,17 @@ void Resource::OnAdaptationApplied( const VideoSourceRestrictions& restrictions_after, rtc::scoped_refptr reason_resource) {} +rtc::TaskQueue* Resource::encoder_queue() const { + return encoder_queue_; +} + +rtc::TaskQueue* Resource::resource_adaptation_queue() const { + return resource_adaptation_queue_; +} + void Resource::OnResourceUsageStateMeasured(ResourceUsageState usage_state) { + RTC_DCHECK(resource_adaptation_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); usage_state_ = usage_state; if (!listener_) return; diff --git a/call/adaptation/resource.h b/call/adaptation/resource.h index 6c7af488cf..2ee0c720d2 100644 --- a/call/adaptation/resource.h +++ b/call/adaptation/resource.h @@ -19,6 +19,7 @@ #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state.h" #include "rtc_base/ref_count.h" +#include "rtc_base/task_queue.h" namespace webrtc { @@ -47,6 +48,9 @@ class Resource : public rtc::RefCountInterface { Resource(); ~Resource() override; + void Initialize(rtc::TaskQueue* encoder_queue, + rtc::TaskQueue* resource_adaptation_queue); + void SetResourceListener(ResourceListener* listener); absl::optional usage_state() const; @@ -69,12 +73,18 @@ class Resource : public rtc::RefCountInterface { virtual std::string name() const = 0; protected: + rtc::TaskQueue* encoder_queue() const; + rtc::TaskQueue* resource_adaptation_queue() const; + // Updates the usage state and informs all registered listeners. void OnResourceUsageStateMeasured(ResourceUsageState usage_state); private: - absl::optional usage_state_; - ResourceListener* listener_; + rtc::TaskQueue* encoder_queue_; + rtc::TaskQueue* resource_adaptation_queue_; + absl::optional usage_state_ + RTC_GUARDED_BY(resource_adaptation_queue_); + ResourceListener* listener_ RTC_GUARDED_BY(resource_adaptation_queue_); }; } // namespace webrtc diff --git a/call/adaptation/resource_adaptation_processor_unittest.cc b/call/adaptation/resource_adaptation_processor_unittest.cc index 68dc4ba050..e94b3a99d7 100644 --- a/call/adaptation/resource_adaptation_processor_unittest.cc +++ b/call/adaptation/resource_adaptation_processor_unittest.cc @@ -18,6 +18,8 @@ #include "call/adaptation/test/fake_resource.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state_provider.h" +#include "rtc_base/event.h" +#include "rtc_base/task_queue_for_test.h" #include "test/gtest.h" namespace webrtc { @@ -67,22 +69,38 @@ class ResourceAdaptationProcessorListenerForTesting class ResourceAdaptationProcessorTest : public ::testing::Test { public: ResourceAdaptationProcessorTest() - : frame_rate_provider_(), + : resource_adaptation_queue_("ResourceAdaptationQueue"), + encoder_queue_("EncoderQueue"), + frame_rate_provider_(), input_state_provider_(&frame_rate_provider_), resource_(new FakeResource("FakeResource")), other_resource_(new FakeResource("OtherFakeResource")), - processor_(&input_state_provider_, - /*encoder_stats_observer=*/&frame_rate_provider_) { - processor_.InitializeOnResourceAdaptationQueue(); - processor_.AddAdaptationListener(&processor_listener_); - processor_.AddResource(resource_); - processor_.AddResource(other_resource_); + processor_(std::make_unique( + &input_state_provider_, + /*encoder_stats_observer=*/&frame_rate_provider_)) { + resource_->Initialize(&encoder_queue_, &resource_adaptation_queue_); + other_resource_->Initialize(&encoder_queue_, &resource_adaptation_queue_); + rtc::Event event; + resource_adaptation_queue_.PostTask([this, &event] { + processor_->InitializeOnResourceAdaptationQueue(); + processor_->AddAdaptationListener(&processor_listener_); + processor_->AddResource(resource_); + processor_->AddResource(other_resource_); + event.Set(); + }); + event.Wait(rtc::Event::kForever); } ~ResourceAdaptationProcessorTest() override { - processor_.StopResourceAdaptation(); - processor_.RemoveResource(resource_); - processor_.RemoveResource(other_resource_); - processor_.RemoveAdaptationListener(&processor_listener_); + rtc::Event event; + resource_adaptation_queue_.PostTask([this, &event] { + processor_->StopResourceAdaptation(); + processor_->RemoveResource(resource_); + processor_->RemoveResource(other_resource_); + processor_->RemoveAdaptationListener(&processor_listener_); + processor_.reset(); + event.Set(); + }); + event.Wait(rtc::Event::kForever); } void SetInputStates(bool has_input, int fps, int frame_size) { @@ -100,42 +118,52 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { } protected: + TaskQueueForTest resource_adaptation_queue_; + TaskQueueForTest encoder_queue_; FakeFrameRateProvider frame_rate_provider_; VideoStreamInputStateProvider input_state_provider_; rtc::scoped_refptr resource_; rtc::scoped_refptr other_resource_; - ResourceAdaptationProcessor processor_; + std::unique_ptr processor_; ResourceAdaptationProcessorListenerForTesting processor_listener_; }; } // namespace TEST_F(ResourceAdaptationProcessorTest, DisabledByDefault) { - EXPECT_EQ(DegradationPreference::DISABLED, - processor_.degradation_preference()); - EXPECT_EQ(DegradationPreference::DISABLED, - processor_.effective_degradation_preference()); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - processor_.StartResourceAdaptation(); - // Adaptation does not happen when disabled. - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); + resource_adaptation_queue_.SendTask( + [this] { + EXPECT_EQ(DegradationPreference::DISABLED, + processor_->degradation_preference()); + EXPECT_EQ(DegradationPreference::DISABLED, + processor_->effective_degradation_preference()); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + processor_->StartResourceAdaptation(); + // Adaptation does not happen when disabled. + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - // Adaptation does not happen if input is insufficient. - // When frame size is missing (OnFrameSizeObserved not called yet). - input_state_provider_.OnHasInputChanged(true); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - // When "has input" is missing. - SetInputStates(false, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - // Note: frame rate cannot be missing, if unset it is 0. + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + // Adaptation does not happen if input is insufficient. + // When frame size is missing (OnFrameSizeObserved not called yet). + input_state_provider_.OnHasInputChanged(true); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); + // When "has input" is missing. + SetInputStates(false, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); + // Note: frame rate cannot be missing, if unset it is 0. + }, + RTC_FROM_HERE); } // These tests verify that restrictions are applied, but not exactly how much @@ -144,212 +172,273 @@ TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) { // restrictions. For that, see video_stream_adapter_unittest.cc. TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingResolutionInMaintainFrameRate) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - EXPECT_TRUE( - processor_listener_.restrictions().max_pixels_per_frame().has_value()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + EXPECT_TRUE(processor_listener_.restrictions() + .max_pixels_per_frame() + .has_value()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingFrameRateInMaintainResolution) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_RESOLUTION); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - EXPECT_TRUE(processor_listener_.restrictions().max_frame_rate().has_value()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_RESOLUTION); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + EXPECT_TRUE( + processor_listener_.restrictions().max_frame_rate().has_value()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingFrameRateAndResolutionInBalanced) { - processor_.SetDegradationPreference(DegradationPreference::BALANCED); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - // Adapting multiple times eventually resticts both frame rate and resolution. - // Exactly many times we need to adapt depends on BalancedDegradationSettings, - // VideoStreamAdapter and default input states. This test requires it to be - // achieved within 4 adaptations. - for (size_t i = 0; i < 4; ++i) { - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(i + 1, processor_listener_.restrictions_updated_count()); - RestrictSource(processor_listener_.restrictions()); - } - EXPECT_TRUE( - processor_listener_.restrictions().max_pixels_per_frame().has_value()); - EXPECT_TRUE(processor_listener_.restrictions().max_frame_rate().has_value()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference(DegradationPreference::BALANCED); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adapting multiple times eventually resticts both frame rate and + // resolution. Exactly many times we need to adapt depends on + // BalancedDegradationSettings, VideoStreamAdapter and default input + // states. This test requires it to be achieved within 4 adaptations. + for (size_t i = 0; i < 4; ++i) { + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(i + 1, processor_listener_.restrictions_updated_count()); + RestrictSource(processor_listener_.restrictions()); + } + EXPECT_TRUE(processor_listener_.restrictions() + .max_pixels_per_frame() + .has_value()); + EXPECT_TRUE( + processor_listener_.restrictions().max_frame_rate().has_value()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, AwaitingPreviousAdaptation) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - // If we don't restrict the source then adaptation will not happen again due - // to "awaiting previous adaptation". This prevents "double-adapt". - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + // If we don't restrict the source then adaptation will not happen again + // due to "awaiting previous adaptation". This prevents "double-adapt". + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, CannotAdaptUpWhenUnrestricted) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - RestrictSource(processor_listener_.restrictions()); - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(2u, processor_listener_.restrictions_updated_count()); - EXPECT_EQ(VideoSourceRestrictions(), processor_listener_.restrictions()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + RestrictSource(processor_listener_.restrictions()); + resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(2u, processor_listener_.restrictions_updated_count()); + EXPECT_EQ(VideoSourceRestrictions(), + processor_listener_.restrictions()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, ResourcesCanPreventAdaptingUp) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - // Adapt down so that we can adapt up. - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - RestrictSource(processor_listener_.restrictions()); - // Adapting up is prevented. - resource_->set_is_adaptation_up_allowed(false); - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adapt down so that we can adapt up. + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + RestrictSource(processor_listener_.restrictions()); + // Adapting up is prevented. + resource_->set_is_adaptation_up_allowed(false); + resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, ResourcesCanNotAdaptUpIfNeverAdaptedDown) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - RestrictSource(processor_listener_.restrictions()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + RestrictSource(processor_listener_.restrictions()); - // Other resource signals under-use - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + // Other resource signals under-use + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, ResourcesCanNotAdaptUpIfNotAdaptedDownAfterReset) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - processor_.ResetVideoSourceRestrictions(); - EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); - other_resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); + processor_->ResetVideoSourceRestrictions(); + EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); + other_resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); - // resource_ did not overuse after we reset the restrictions, so adapt up - // should be disallowed. - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); + // resource_ did not overuse after we reset the restrictions, so adapt + // up should be disallowed. + resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, MultipleResourcesCanTriggerMultipleAdaptations) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(3, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); + other_resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); + other_resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(3, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - // Does not trigger adaptation since resource has no adaptations left. - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); + resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); + // Does not trigger adaptation since resource has no adaptations left. + resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); + EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, AdaptingTriggersOnAdaptationApplied) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, resource_->num_adaptations_applied()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, resource_->num_adaptations_applied()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, AdaptingClearsResourceUsageState) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - EXPECT_FALSE(resource_->usage_state().has_value()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); + EXPECT_FALSE(resource_->usage_state().has_value()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, FailingAdaptingAlsoClearsResourceUsageState) { - processor_.SetDegradationPreference(DegradationPreference::DISABLED); - processor_.StartResourceAdaptation(); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - EXPECT_FALSE(resource_->usage_state().has_value()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference(DegradationPreference::DISABLED); + processor_->StartResourceAdaptation(); + resource_->set_usage_state(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); + EXPECT_FALSE(resource_->usage_state().has_value()); + }, + RTC_FROM_HERE); } TEST_F(ResourceAdaptationProcessorTest, AdaptsDownWhenOtherResourceIsAlwaysUnderused) { - processor_.SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_.StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - // Does not trigger adapataion because there's no restriction. - EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); + resource_adaptation_queue_.SendTask( + [this] { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + processor_->StartResourceAdaptation(); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); + // Does not trigger adapataion because there's no restriction. + EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - resource_->set_usage_state(ResourceUsageState::kOveruse); - // Adapts down even if other resource asked for adapting up. - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); + resource_->set_usage_state(ResourceUsageState::kOveruse); + // Adapts down even if other resource asked for adapting up. + EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - // Doesn't adapt up because adaptation is due to another resource. - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); + RestrictSource(processor_listener_.restrictions()); + other_resource_->set_usage_state(ResourceUsageState::kUnderuse); + // Doesn't adapt up because adaptation is due to another resource. + EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); + RestrictSource(processor_listener_.restrictions()); + }, + RTC_FROM_HERE); } } // namespace webrtc diff --git a/call/adaptation/resource_unittest.cc b/call/adaptation/resource_unittest.cc index 50a6220f40..8f3ae32dca 100644 --- a/call/adaptation/resource_unittest.cc +++ b/call/adaptation/resource_unittest.cc @@ -10,8 +10,12 @@ #include "call/adaptation/resource.h" +#include + #include "api/scoped_refptr.h" #include "call/adaptation/test/fake_resource.h" +#include "rtc_base/event.h" +#include "rtc_base/task_queue_for_test.h" #include "test/gmock.h" #include "test/gtest.h" @@ -27,28 +31,49 @@ class MockResourceListener : public ResourceListener { (rtc::scoped_refptr resource)); }; -TEST(ResourceTest, RegisteringListenerReceivesCallbacks) { - StrictMock resource_listener; - rtc::scoped_refptr fake_resource( - new FakeResource("FakeResource")); - fake_resource->SetResourceListener(&resource_listener); - EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)) - .Times(1) - .WillOnce([](rtc::scoped_refptr resource) { - EXPECT_EQ(ResourceUsageState::kOveruse, resource->usage_state()); - }); - fake_resource->set_usage_state(ResourceUsageState::kOveruse); - fake_resource->SetResourceListener(nullptr); +class ResourceTest : public ::testing::Test { + public: + ResourceTest() + : resource_adaptation_queue_("ResourceAdaptationQueue"), + encoder_queue_("EncoderQueue"), + fake_resource_(new FakeResource("FakeResource")) { + fake_resource_->Initialize(&encoder_queue_, &resource_adaptation_queue_); + } + + protected: + const std::unique_ptr task_queue_factory_; + TaskQueueForTest resource_adaptation_queue_; + TaskQueueForTest encoder_queue_; + rtc::scoped_refptr fake_resource_; +}; + +TEST_F(ResourceTest, RegisteringListenerReceivesCallbacks) { + resource_adaptation_queue_.SendTask( + [this] { + StrictMock resource_listener; + fake_resource_->SetResourceListener(&resource_listener); + EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)) + .Times(1) + .WillOnce([](rtc::scoped_refptr resource) { + EXPECT_EQ(ResourceUsageState::kOveruse, resource->usage_state()); + }); + fake_resource_->set_usage_state(ResourceUsageState::kOveruse); + fake_resource_->SetResourceListener(nullptr); + }, + RTC_FROM_HERE); } -TEST(ResourceTest, UnregisteringListenerStopsCallbacks) { - StrictMock resource_listener; - rtc::scoped_refptr fake_resource( - new FakeResource("FakeResource")); - fake_resource->SetResourceListener(&resource_listener); - fake_resource->SetResourceListener(nullptr); - EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)).Times(0); - fake_resource->set_usage_state(ResourceUsageState::kOveruse); +TEST_F(ResourceTest, UnregisteringListenerStopsCallbacks) { + resource_adaptation_queue_.SendTask( + [this] { + StrictMock resource_listener; + fake_resource_->SetResourceListener(&resource_listener); + fake_resource_->SetResourceListener(nullptr); + EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)) + .Times(0); + fake_resource_->set_usage_state(ResourceUsageState::kOveruse); + }, + RTC_FROM_HERE); } } // namespace webrtc diff --git a/video/adaptation/encode_usage_resource.cc b/video/adaptation/encode_usage_resource.cc index 6e2827a9dd..49531a3aa4 100644 --- a/video/adaptation/encode_usage_resource.cc +++ b/video/adaptation/encode_usage_resource.cc @@ -21,26 +21,19 @@ namespace webrtc { EncodeUsageResource::EncodeUsageResource( std::unique_ptr overuse_detector) : rtc::RefCountedObject(), - encoder_queue_(nullptr), overuse_detector_(std::move(overuse_detector)), is_started_(false), target_frame_rate_(absl::nullopt) { RTC_DCHECK(overuse_detector_); } -void EncodeUsageResource::Initialize(rtc::TaskQueue* encoder_queue) { - RTC_DCHECK(!encoder_queue_); - RTC_DCHECK(encoder_queue); - encoder_queue_ = encoder_queue; -} - bool EncodeUsageResource::is_started() const { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); return is_started_; } void EncodeUsageResource::StartCheckForOveruse(CpuOveruseOptions options) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); RTC_DCHECK(!is_started_); overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(), std::move(options), this); @@ -49,14 +42,14 @@ void EncodeUsageResource::StartCheckForOveruse(CpuOveruseOptions options) { } void EncodeUsageResource::StopCheckForOveruse() { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); overuse_detector_->StopCheckForOveruse(); is_started_ = false; } void EncodeUsageResource::SetTargetFrameRate( absl::optional target_frame_rate) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); if (target_frame_rate == target_frame_rate_) return; target_frame_rate_ = target_frame_rate; @@ -66,7 +59,7 @@ void EncodeUsageResource::SetTargetFrameRate( void EncodeUsageResource::OnEncodeStarted(const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g. // "OnEncodeStarted"?) or revise usage. overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us); @@ -77,7 +70,7 @@ void EncodeUsageResource::OnEncodeCompleted( int64_t time_sent_in_us, int64_t capture_time_us, absl::optional encode_duration_us) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); // TODO(hbos): Rename FrameSent() to something more appropriate (e.g. // "OnEncodeCompleted"?). overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, @@ -85,21 +78,29 @@ void EncodeUsageResource::OnEncodeCompleted( } void EncodeUsageResource::AdaptUp() { - RTC_DCHECK_RUN_ON(encoder_queue_); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // PostTask the resource usage measurements. - OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); + RTC_DCHECK_RUN_ON(encoder_queue()); + // Reference counting guarantees that this object is still alive by the time + // the task is executed. + resource_adaptation_queue()->PostTask( + [this_ref = rtc::scoped_refptr(this)] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue()); + this_ref->OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); + }); } void EncodeUsageResource::AdaptDown() { - RTC_DCHECK_RUN_ON(encoder_queue_); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // PostTask the resource usage measurements. - OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); + RTC_DCHECK_RUN_ON(encoder_queue()); + // Reference counting guarantees that this object is still alive by the time + // the task is executed. + resource_adaptation_queue()->PostTask( + [this_ref = rtc::scoped_refptr(this)] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue()); + this_ref->OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); + }); } int EncodeUsageResource::TargetFrameRateAsInt() { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); return target_frame_rate_.has_value() ? static_cast(target_frame_rate_.value()) : std::numeric_limits::max(); diff --git a/video/adaptation/encode_usage_resource.h b/video/adaptation/encode_usage_resource.h index 2ade816ad7..3c6f02b243 100644 --- a/video/adaptation/encode_usage_resource.h +++ b/video/adaptation/encode_usage_resource.h @@ -34,10 +34,6 @@ class EncodeUsageResource : public rtc::RefCountedObject, explicit EncodeUsageResource( std::unique_ptr overuse_detector); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // pass it in here. - void Initialize(rtc::TaskQueue* encoder_queue); - bool is_started() const; void StartCheckForOveruse(CpuOveruseOptions options); @@ -60,11 +56,10 @@ class EncodeUsageResource : public rtc::RefCountedObject, private: int TargetFrameRateAsInt(); - rtc::TaskQueue* encoder_queue_; const std::unique_ptr overuse_detector_ - RTC_GUARDED_BY(encoder_queue_); - bool is_started_ RTC_GUARDED_BY(encoder_queue_); - absl::optional target_frame_rate_ RTC_GUARDED_BY(encoder_queue_); + RTC_GUARDED_BY(encoder_queue()); + bool is_started_ RTC_GUARDED_BY(encoder_queue()); + absl::optional target_frame_rate_ RTC_GUARDED_BY(encoder_queue()); }; } // namespace webrtc diff --git a/video/adaptation/quality_scaler_resource.cc b/video/adaptation/quality_scaler_resource.cc index 631e5b08fd..403f6080ca 100644 --- a/video/adaptation/quality_scaler_resource.cc +++ b/video/adaptation/quality_scaler_resource.cc @@ -18,12 +18,10 @@ namespace webrtc { QualityScalerResource::QualityScalerResource() : rtc::RefCountedObject(), - encoder_queue_(nullptr), - adaptation_processor_(nullptr), quality_scaler_(nullptr), num_handled_callbacks_(0), pending_callbacks_(), - processing_in_progress_(false), + adaptation_processor_(nullptr), clear_qp_samples_(false) {} QualityScalerResource::~QualityScalerResource() { @@ -31,33 +29,27 @@ QualityScalerResource::~QualityScalerResource() { RTC_DCHECK(pending_callbacks_.empty()); } -void QualityScalerResource::Initialize(rtc::TaskQueue* encoder_queue) { - RTC_DCHECK(!encoder_queue_); - RTC_DCHECK(encoder_queue); - encoder_queue_ = encoder_queue; -} - void QualityScalerResource::SetAdaptationProcessor( ResourceAdaptationProcessorInterface* adaptation_processor) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue()); adaptation_processor_ = adaptation_processor; } bool QualityScalerResource::is_started() const { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); return quality_scaler_.get(); } void QualityScalerResource::StartCheckForOveruse( VideoEncoder::QpThresholds qp_thresholds) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); RTC_DCHECK(!is_started()); quality_scaler_ = std::make_unique(this, std::move(qp_thresholds)); } void QualityScalerResource::StopCheckForOveruse() { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); // Ensure we have no pending callbacks. This makes it safe to destroy the // QualityScaler and even task queues with tasks in-flight. AbortPendingCallbacks(); @@ -66,35 +58,41 @@ void QualityScalerResource::StopCheckForOveruse() { void QualityScalerResource::SetQpThresholds( VideoEncoder::QpThresholds qp_thresholds) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); RTC_DCHECK(is_started()); quality_scaler_->SetQpThresholds(std::move(qp_thresholds)); } bool QualityScalerResource::QpFastFilterLow() { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); RTC_DCHECK(is_started()); return quality_scaler_->QpFastFilterLow(); } void QualityScalerResource::OnEncodeCompleted(const EncodedImage& encoded_image, int64_t time_sent_in_us) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); if (quality_scaler_ && encoded_image.qp_ >= 0) { quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us); } else if (!quality_scaler_) { + // Reference counting guarantees that this object is still alive by the time + // the task is executed. // TODO(webrtc:11553): this is a workaround to ensure that all quality // scaler imposed limitations are removed once qualty scaler is disabled // mid call. // Instead it should be done at a higher layer in the same way for all // resources. - OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); + resource_adaptation_queue()->PostTask( + [this_ref = rtc::scoped_refptr(this)] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue()); + this_ref->OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); + }); } } void QualityScalerResource::OnFrameDropped( EncodedImageCallback::DropReason reason) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); if (!quality_scaler_) return; switch (reason) { @@ -109,29 +107,37 @@ void QualityScalerResource::OnFrameDropped( void QualityScalerResource::OnReportQpUsageHigh( rtc::scoped_refptr callback) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); size_t callback_id = QueuePendingCallback(callback); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // PostTask the resource usage measurements. - RTC_DCHECK(!processing_in_progress_); - processing_in_progress_ = true; - clear_qp_samples_ = false; - // If this OnResourceUsageStateMeasured() triggers an adaptation, - // OnAdaptationApplied() will occur between this line and the next. This - // allows modifying |clear_qp_samples_| based on the adaptation. - OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); - HandlePendingCallback(callback_id, clear_qp_samples_); - processing_in_progress_ = false; + // Reference counting guarantees that this object is still alive by the time + // the task is executed. + resource_adaptation_queue()->PostTask( + [this_ref = rtc::scoped_refptr(this), + callback_id] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue()); + this_ref->clear_qp_samples_ = false; + // If this OnResourceUsageStateMeasured() triggers an adaptation, + // OnAdaptationApplied() will occur between this line and the next. This + // allows modifying |clear_qp_samples_| based on the adaptation. + this_ref->OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); + this_ref->HandlePendingCallback(callback_id, + this_ref->clear_qp_samples_); + }); } void QualityScalerResource::OnReportQpUsageLow( rtc::scoped_refptr callback) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); size_t callback_id = QueuePendingCallback(callback); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // PostTask the resource usage measurements. - OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); - HandlePendingCallback(callback_id, true); + // Reference counting guarantees that this object is still alive by the time + // the task is executed. + resource_adaptation_queue()->PostTask( + [this_ref = rtc::scoped_refptr(this), + callback_id] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue()); + this_ref->OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); + this_ref->HandlePendingCallback(callback_id, true); + }); } void QualityScalerResource::OnAdaptationApplied( @@ -139,9 +145,9 @@ void QualityScalerResource::OnAdaptationApplied( const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, rtc::scoped_refptr reason_resource) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue()); // We only clear QP samples on adaptations triggered by the QualityScaler. - if (!processing_in_progress_) + if (reason_resource != this) return; clear_qp_samples_ = true; // If we're in "balanced" and the frame rate before and after adaptation did @@ -173,7 +179,7 @@ void QualityScalerResource::OnAdaptationApplied( size_t QualityScalerResource::QueuePendingCallback( rtc::scoped_refptr callback) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); pending_callbacks_.push(callback); // The ID of a callback is its sequence number (1, 2, 3...). return num_handled_callbacks_ + pending_callbacks_.size(); @@ -181,24 +187,29 @@ size_t QualityScalerResource::QueuePendingCallback( void QualityScalerResource::HandlePendingCallback(size_t callback_id, bool clear_qp_samples) { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // this method would be invoked on the adaptation queue and a PostTask would - // be used to resolve the callback. - RTC_DCHECK_RUN_ON(encoder_queue_); - if (num_handled_callbacks_ >= callback_id) { - // The callback with this ID has already been handled. - // This happens if AbortPendingCallbacks() is called while the task is - // in flight. - return; - } - RTC_DCHECK(!pending_callbacks_.empty()); - pending_callbacks_.front()->OnQpUsageHandled(clear_qp_samples); - ++num_handled_callbacks_; - pending_callbacks_.pop(); + RTC_DCHECK_RUN_ON(resource_adaptation_queue()); + // Reference counting guarantees that this object is still alive by the time + // the task is executed. + encoder_queue()->PostTask( + [this_ref = rtc::scoped_refptr(this), callback_id, + clear_qp_samples] { + RTC_DCHECK_RUN_ON(this_ref->encoder_queue()); + if (this_ref->num_handled_callbacks_ >= callback_id) { + // The callback with this ID has already been handled. + // This happens if AbortPendingCallbacks() is called while the task is + // in flight. + return; + } + RTC_DCHECK(!this_ref->pending_callbacks_.empty()); + this_ref->pending_callbacks_.front()->OnQpUsageHandled( + clear_qp_samples); + ++this_ref->num_handled_callbacks_; + this_ref->pending_callbacks_.pop(); + }); } void QualityScalerResource::AbortPendingCallbacks() { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue()); while (!pending_callbacks_.empty()) { pending_callbacks_.front()->OnQpUsageHandled(false); ++num_handled_callbacks_; diff --git a/video/adaptation/quality_scaler_resource.h b/video/adaptation/quality_scaler_resource.h index 7c55e9bacd..78685823c3 100644 --- a/video/adaptation/quality_scaler_resource.h +++ b/video/adaptation/quality_scaler_resource.h @@ -33,9 +33,6 @@ class QualityScalerResource : public rtc::RefCountedObject, QualityScalerResource(); ~QualityScalerResource() override; - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // pass it in here. - void Initialize(rtc::TaskQueue* encoder_queue); void SetAdaptationProcessor( ResourceAdaptationProcessorInterface* adaptation_processor); @@ -74,25 +71,22 @@ class QualityScalerResource : public rtc::RefCountedObject, void HandlePendingCallback(size_t callback_id, bool clear_qp_samples); void AbortPendingCallbacks(); - rtc::TaskQueue* encoder_queue_; - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // guard the processor by it instead. - ResourceAdaptationProcessorInterface* adaptation_processor_ - RTC_GUARDED_BY(encoder_queue_); - std::unique_ptr quality_scaler_ RTC_GUARDED_BY(encoder_queue_); + // Members accessed on the encoder queue. + std::unique_ptr quality_scaler_ + RTC_GUARDED_BY(encoder_queue()); // Every OnReportQpUsageHigh/Low() operation has a callback that MUST be - // invoked on the |encoder_queue_|. - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // handling a measurement entails a task queue "ping" round-trip between the - // encoder queue and the adaptation queue. Multiple callbacks in-flight would - // then be possible. - size_t num_handled_callbacks_ RTC_GUARDED_BY(encoder_queue_); + // invoked on the |encoder_queue_|. Because usage measurements are reported on + // the |encoder_queue_| but handled by the processor on the the + // |resource_adaptation_queue_|, handling a measurement entails a task queue + // "ping" round-trip. Multiple callbacks in-flight is thus possible. + size_t num_handled_callbacks_ RTC_GUARDED_BY(encoder_queue()); std::queue> - pending_callbacks_ RTC_GUARDED_BY(encoder_queue_); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // guard processing_in_progress_/clear_cp_samples_ by it instead. - bool processing_in_progress_ RTC_GUARDED_BY(encoder_queue_); - bool clear_qp_samples_ RTC_GUARDED_BY(encoder_queue_); + pending_callbacks_ RTC_GUARDED_BY(encoder_queue()); + + // Members accessed on the adaptation queue. + ResourceAdaptationProcessorInterface* adaptation_processor_ + RTC_GUARDED_BY(resource_adaptation_queue()); + bool clear_qp_samples_ RTC_GUARDED_BY(resource_adaptation_queue()); }; } // namespace webrtc diff --git a/video/adaptation/quality_scaler_resource_unittest.cc b/video/adaptation/quality_scaler_resource_unittest.cc index d49addfe8c..66f4e13870 100644 --- a/video/adaptation/quality_scaler_resource_unittest.cc +++ b/video/adaptation/quality_scaler_resource_unittest.cc @@ -68,11 +68,15 @@ class QualityScalerResourceTest : public ::testing::Test { public: QualityScalerResourceTest() : task_queue_factory_(CreateDefaultTaskQueueFactory()), + resource_adaptation_queue_(task_queue_factory_->CreateTaskQueue( + "ResourceAdaptationQueue", + TaskQueueFactory::Priority::NORMAL)), encoder_queue_(task_queue_factory_->CreateTaskQueue( "EncoderQueue", TaskQueueFactory::Priority::NORMAL)), quality_scaler_resource_(new QualityScalerResource()) { - quality_scaler_resource_->Initialize(&encoder_queue_); + quality_scaler_resource_->Initialize(&encoder_queue_, + &resource_adaptation_queue_); rtc::Event event; encoder_queue_.PostTask([this, &event] { quality_scaler_resource_->StartCheckForOveruse( @@ -93,6 +97,7 @@ class QualityScalerResourceTest : public ::testing::Test { protected: const std::unique_ptr task_queue_factory_; + rtc::TaskQueue resource_adaptation_queue_; rtc::TaskQueue encoder_queue_; rtc::scoped_refptr quality_scaler_resource_; }; @@ -115,9 +120,6 @@ TEST_F(QualityScalerResourceTest, ReportQpLow) { callback->qp_usage_handled_event()->Wait(kDefaultTimeout); } -// TODO(https://crbug.com/webrtc/11542): Callbacks are currently resolved -// immediately, but when we have an adaptation queue this test will ensure we -// can have multiple callbacks pending at the same time. TEST_F(QualityScalerResourceTest, MultipleCallbacksInFlight) { rtc::scoped_refptr callback1 = new FakeQualityScalerQpUsageHandlerCallback(&encoder_queue_); @@ -135,9 +137,6 @@ TEST_F(QualityScalerResourceTest, MultipleCallbacksInFlight) { callback3->qp_usage_handled_event()->Wait(kDefaultTimeout); } -// TODO(https://crbug.com/webrtc/11542): Callbacks are currently resolved -// immediately, but when we have an adaptation queue this test will ensure we -// can abort pending callbacks. TEST_F(QualityScalerResourceTest, AbortPendingCallbacksAndStartAgain) { rtc::scoped_refptr callback1 = new FakeQualityScalerQpUsageHandlerCallback(&encoder_queue_); diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index c0103adbd6..b309dd3455 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -147,6 +147,7 @@ VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts:: void VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts:: SetAdaptationProcessor( ResourceAdaptationProcessorInterface* adaptation_processor) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue()); adaptation_processor_ = adaptation_processor; } @@ -155,26 +156,31 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpDueToActiveCounts:: const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, rtc::scoped_refptr reason_resource) const { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // ensure that this is running on it instead. - RTC_DCHECK_RUN_ON(manager_->encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue()); RTC_DCHECK(adaptation_processor_); VideoAdaptationReason reason = manager_->GetReasonFromResource(reason_resource); - // We can't adapt up if we're already at the highest setting. - // Note that this only includes counts relevant to the current degradation - // preference. e.g. we previously adapted resolution, now prefer adpating fps, - // only count the fps adaptations and not the previous resolution adaptations. - // TODO(hbos): Why would the reason matter? If a particular resource doesn't - // want us to go up it should prevent us from doing so itself rather than to - // have this catch-all reason- and stats-based approach. - int num_downgrades = - FilterVideoAdaptationCountersByDegradationPreference( - manager_->active_counts_[reason], - adaptation_processor_->effective_degradation_preference()) - .Total(); - RTC_DCHECK_GE(num_downgrades, 0); - return num_downgrades > 0; + { + // This is the same as |resource_adaptation_queue_|, but need to + // RTC_DCHECK_RUN_ON() both to avoid compiler error when accessing + // |manager_->active_counts_|. + RTC_DCHECK_RUN_ON(manager_->resource_adaptation_queue_); + // We can't adapt up if we're already at the highest setting. + // Note that this only includes counts relevant to the current degradation + // preference. e.g. we previously adapted resolution, now prefer adpating + // fps, only count the fps adaptations and not the previous resolution + // adaptations. + // TODO(hbos): Why would the reason matter? If a particular resource doesn't + // want us to go up it should prevent us from doing so itself rather than to + // have this catch-all reason- and stats-based approach. + int num_downgrades = + FilterVideoAdaptationCountersByDegradationPreference( + manager_->active_counts_[reason], + adaptation_processor_->effective_degradation_preference()) + .Total(); + RTC_DCHECK_GE(num_downgrades, 0); + return num_downgrades > 0; + } } VideoStreamEncoderResourceManager:: @@ -189,18 +195,30 @@ VideoStreamEncoderResourceManager:: void VideoStreamEncoderResourceManager:: PreventIncreaseResolutionDueToBitrateResource::OnEncoderSettingsUpdated( absl::optional encoder_settings) { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // update the state in a PostTask instead. - encoder_settings_ = std::move(encoder_settings); + RTC_DCHECK_RUN_ON(encoder_queue()); + resource_adaptation_queue()->PostTask( + [this_ref = + rtc::scoped_refptr( + this), + encoder_settings] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue()); + this_ref->encoder_settings_ = std::move(encoder_settings); + }); } void VideoStreamEncoderResourceManager:: PreventIncreaseResolutionDueToBitrateResource:: OnEncoderTargetBitrateUpdated( absl::optional encoder_target_bitrate_bps) { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // update the state in a PostTask instead. - encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; + RTC_DCHECK_RUN_ON(encoder_queue()); + resource_adaptation_queue()->PostTask( + [this_ref = + rtc::scoped_refptr( + this), + encoder_target_bitrate_bps] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue()); + this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; + }); } bool VideoStreamEncoderResourceManager:: @@ -209,9 +227,7 @@ bool VideoStreamEncoderResourceManager:: const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, rtc::scoped_refptr reason_resource) const { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // ensure that this is running on it instead. - RTC_DCHECK_RUN_ON(manager_->encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue()); VideoAdaptationReason reason = manager_->GetReasonFromResource(reason_resource); // If increasing resolution due to kQuality, make sure bitrate limits are not @@ -250,15 +266,20 @@ VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: void VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: SetAdaptationProcessor( ResourceAdaptationProcessorInterface* adaptation_processor) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue()); adaptation_processor_ = adaptation_processor; } void VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: OnEncoderTargetBitrateUpdated( absl::optional encoder_target_bitrate_bps) { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // update the state in a PostTask instead. - encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; + RTC_DCHECK_RUN_ON(encoder_queue()); + resource_adaptation_queue()->PostTask( + [this_ref = rtc::scoped_refptr(this), + encoder_target_bitrate_bps] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue()); + this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; + }); } bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: @@ -266,9 +287,7 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: const VideoSourceRestrictions& restrictions_before, const VideoSourceRestrictions& restrictions_after, rtc::scoped_refptr reason_resource) const { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // ensure that this is running on it instead. - RTC_DCHECK_RUN_ON(manager_->encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue()); RTC_DCHECK(adaptation_processor_); VideoAdaptationReason reason = manager_->GetReasonFromResource(reason_resource); @@ -312,6 +331,7 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( new EncodeUsageResource(std::move(overuse_detector))), quality_scaler_resource_(new QualityScalerResource()), encoder_queue_(nullptr), + resource_adaptation_queue_(nullptr), input_state_provider_(input_state_provider), adaptation_processor_(nullptr), encoder_stats_observer_(encoder_stats_observer), @@ -342,17 +362,29 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() {} void VideoStreamEncoderResourceManager::Initialize( - rtc::TaskQueue* encoder_queue) { + rtc::TaskQueue* encoder_queue, + rtc::TaskQueue* resource_adaptation_queue) { RTC_DCHECK(!encoder_queue_); RTC_DCHECK(encoder_queue); + RTC_DCHECK(!resource_adaptation_queue_); + RTC_DCHECK(resource_adaptation_queue); encoder_queue_ = encoder_queue; - encode_usage_resource_->Initialize(encoder_queue_); - quality_scaler_resource_->Initialize(encoder_queue_); + resource_adaptation_queue_ = resource_adaptation_queue; + prevent_adapt_up_due_to_active_counts_->Initialize( + encoder_queue_, resource_adaptation_queue_); + prevent_increase_resolution_due_to_bitrate_resource_->Initialize( + encoder_queue_, resource_adaptation_queue_); + prevent_adapt_up_in_balanced_resource_->Initialize( + encoder_queue_, resource_adaptation_queue_); + encode_usage_resource_->Initialize(encoder_queue_, + resource_adaptation_queue_); + quality_scaler_resource_->Initialize(encoder_queue_, + resource_adaptation_queue_); } void VideoStreamEncoderResourceManager::SetAdaptationProcessor( ResourceAdaptationProcessorInterface* adaptation_processor) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); adaptation_processor_ = adaptation_processor; prevent_adapt_up_due_to_active_counts_->SetAdaptationProcessor( adaptation_processor); @@ -465,16 +497,27 @@ void VideoStreamEncoderResourceManager::SetEncoderRates( void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() { RTC_DCHECK_RUN_ON(encoder_queue_); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // PostTask the request to adapt due to frame drop. - adaptation_processor_->TriggerAdaptationDueToFrameDroppedDueToSize( - quality_scaler_resource_); + // The VideoStreamEncoder makes the manager outlive the adaptation queue. This + // means that if the task gets executed, |this| has not been freed yet. + // TODO(https://crbug.com/webrtc/11565): When the manager no longer outlives + // the adaptation queue, add logic to prevent use-after-free on |this|. + resource_adaptation_queue_->PostTask([this] { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + if (!adaptation_processor_) { + // The processor nulled before this task had a chance to execute. This + // happens if the processor is destroyed. No action needed. + return; + } + adaptation_processor_->TriggerAdaptationDueToFrameDroppedDueToSize( + quality_scaler_resource_); + }); initial_frame_dropper_->OnFrameDroppedDueToSize(); } void VideoStreamEncoderResourceManager::OnEncodeStarted( const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) { + RTC_DCHECK_RUN_ON(encoder_queue_); encode_usage_resource_->OnEncodeStarted(cropped_frame, time_when_first_seen_us); } @@ -610,11 +653,7 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, rtc::scoped_refptr reason) { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // ensure that this is running on it instead, and PostTask back to the encoder - // queue if need be. - RTC_DCHECK_RUN_ON(encoder_queue_); - video_source_restrictions_ = restrictions; + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); VideoAdaptationCounters previous_adaptation_counters = active_counts_[VideoAdaptationReason::kQuality] + active_counts_[VideoAdaptationReason::kCpu]; @@ -638,7 +677,14 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( RTC_DCHECK_EQ(adaptation_counters_total_abs_diff, 0); } RTC_LOG(LS_INFO) << ActiveCountsToString(); - MaybeUpdateTargetFrameRate(); + + // The VideoStreamEncoder makes the manager outlive the encoder queue. This + // means that if the task gets executed, |this| has not been freed yet. + encoder_queue_->PostTask([this, restrictions] { + RTC_DCHECK_RUN_ON(encoder_queue_); + video_source_restrictions_ = restrictions; + MaybeUpdateTargetFrameRate(); + }); } void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() { @@ -728,7 +774,7 @@ void VideoStreamEncoderResourceManager::OnAdaptationCountChanged( void VideoStreamEncoderResourceManager::UpdateAdaptationStats( const VideoAdaptationCounters& total_counts, VideoAdaptationReason reason) { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); // Update active counts VideoAdaptationCounters& active_count = active_counts_[reason]; VideoAdaptationCounters& other_active = active_counts_[OtherReason(reason)]; @@ -777,29 +823,43 @@ void VideoStreamEncoderResourceManager::MaybePerformQualityRampupExperiment() { try_quality_rampup = true; } } - // TODO(https://crbug.com/webrtc/11392): See if we can rely on the total - // counts or the stats, and not the active counts. - const VideoAdaptationCounters& qp_counts = - active_counts_[VideoAdaptationReason::kQuality]; - const VideoAdaptationCounters& cpu_counts = - active_counts_[VideoAdaptationReason::kCpu]; - if (try_quality_rampup && qp_counts.resolution_adaptations > 0 && - cpu_counts.Total() == 0) { - RTC_LOG(LS_INFO) << "Reset quality limitations."; - adaptation_processor_->ResetVideoSourceRestrictions(); - quality_rampup_done_ = true; + if (try_quality_rampup) { + // The VideoStreamEncoder makes the manager outlive the adaptation queue. + // This means that if the task gets executed, |this| has not been freed yet. + // TODO(https://crbug.com/webrtc/11565): When the manager no longer outlives + // the adaptation queue, add logic to prevent use-after-free on |this|. + resource_adaptation_queue_->PostTask([this] { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + if (!adaptation_processor_) { + // The processor nulled before this task had a chance to execute. This + // happens if the processor is destroyed. No action needed. + return; + } + // TODO(https://crbug.com/webrtc/11392): See if we can rely on the total + // counts or the stats, and not the active counts. + const VideoAdaptationCounters& qp_counts = + active_counts_[VideoAdaptationReason::kQuality]; + const VideoAdaptationCounters& cpu_counts = + active_counts_[VideoAdaptationReason::kCpu]; + if (!quality_rampup_done_ && qp_counts.resolution_adaptations > 0 && + cpu_counts.Total() == 0) { + RTC_LOG(LS_INFO) << "Reset quality limitations."; + adaptation_processor_->ResetVideoSourceRestrictions(); + quality_rampup_done_ = true; + } + }); } } void VideoStreamEncoderResourceManager::ResetActiveCounts() { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); active_counts_.clear(); active_counts_[VideoAdaptationReason::kCpu] = VideoAdaptationCounters(); active_counts_[VideoAdaptationReason::kQuality] = VideoAdaptationCounters(); } std::string VideoStreamEncoderResourceManager::ActiveCountsToString() const { - RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK_EQ(2, active_counts_.size()); rtc::StringBuilder ss; diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index ac20670727..d028e5049a 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -11,6 +11,7 @@ #ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ #define VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ +#include #include #include #include @@ -33,6 +34,7 @@ #include "call/adaptation/resource_adaptation_processor_interface.h" #include "call/adaptation/video_stream_adapter.h" #include "call/adaptation/video_stream_input_state_provider.h" +#include "rtc_base/critical_section.h" #include "rtc_base/experiments/quality_rampup_experiment.h" #include "rtc_base/experiments/quality_scaler_settings.h" #include "rtc_base/strings/string_builder.h" @@ -69,9 +71,8 @@ class VideoStreamEncoderResourceManager std::unique_ptr overuse_detector); ~VideoStreamEncoderResourceManager() override; - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // pass it in here. - void Initialize(rtc::TaskQueue* encoder_queue); + void Initialize(rtc::TaskQueue* encoder_queue, + rtc::TaskQueue* resource_adaptation_queue); void SetAdaptationProcessor( ResourceAdaptationProcessorInterface* adaptation_processor); @@ -200,7 +201,8 @@ class VideoStreamEncoderResourceManager // The |manager_| must be alive as long as this resource is added to the // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called. VideoStreamEncoderResourceManager* const manager_; - ResourceAdaptationProcessorInterface* adaptation_processor_; + ResourceAdaptationProcessorInterface* adaptation_processor_ + RTC_GUARDED_BY(resource_adaptation_queue()); }; // Does not trigger adaptations, only prevents adapting up resolution. @@ -230,8 +232,10 @@ class VideoStreamEncoderResourceManager // The |manager_| must be alive as long as this resource is added to the // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called. VideoStreamEncoderResourceManager* const manager_; - absl::optional encoder_settings_; - absl::optional encoder_target_bitrate_bps_; + absl::optional encoder_settings_ + RTC_GUARDED_BY(resource_adaptation_queue()); + absl::optional encoder_target_bitrate_bps_ + RTC_GUARDED_BY(resource_adaptation_queue()); }; // Does not trigger adaptations, only prevents adapting up in BALANCED. @@ -261,8 +265,10 @@ class VideoStreamEncoderResourceManager // The |manager_| must be alive as long as this resource is added to the // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called. VideoStreamEncoderResourceManager* const manager_; - ResourceAdaptationProcessorInterface* adaptation_processor_; - absl::optional encoder_target_bitrate_bps_; + ResourceAdaptationProcessorInterface* adaptation_processor_ + RTC_GUARDED_BY(resource_adaptation_queue()); + absl::optional encoder_target_bitrate_bps_ + RTC_GUARDED_BY(resource_adaptation_queue()); }; const rtc::scoped_refptr @@ -275,14 +281,13 @@ class VideoStreamEncoderResourceManager const rtc::scoped_refptr quality_scaler_resource_; rtc::TaskQueue* encoder_queue_; + rtc::TaskQueue* resource_adaptation_queue_; VideoStreamInputStateProvider* const input_state_provider_ RTC_GUARDED_BY(encoder_queue_); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // guard the processor by it instead. ResourceAdaptationProcessorInterface* adaptation_processor_ - RTC_GUARDED_BY(encoder_queue_); - VideoStreamEncoderObserver* const encoder_stats_observer_ - RTC_GUARDED_BY(encoder_queue_); + RTC_GUARDED_BY(resource_adaptation_queue_); + // Thread-safe. + VideoStreamEncoderObserver* const encoder_stats_observer_; DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); VideoSourceRestrictions video_source_restrictions_ @@ -298,7 +303,8 @@ class VideoStreamEncoderResourceManager RTC_GUARDED_BY(encoder_queue_); absl::optional encoder_rates_ RTC_GUARDED_BY(encoder_queue_); - bool quality_rampup_done_ RTC_GUARDED_BY(encoder_queue_); + // Used on both the encoder queue and resource adaptation queue. + std::atomic quality_rampup_done_; QualityRampupExperiment quality_rampup_experiment_ RTC_GUARDED_BY(encoder_queue_); absl::optional encoder_settings_ @@ -325,7 +331,7 @@ class VideoStreamEncoderResourceManager // thread-safe anyway, and active counts are used by // PreventAdaptUpDueToActiveCounts to make decisions. std::unordered_map - active_counts_ RTC_GUARDED_BY(encoder_queue_); + active_counts_ RTC_GUARDED_BY(resource_adaptation_queue_); }; } // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index bb779ff186..2684c4a61e 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -268,19 +268,21 @@ VideoStreamEncoder::VideoStreamEncoder( std::move(overuse_detector)), video_source_sink_controller_(/*sink=*/this, /*source=*/nullptr), + resource_adaptation_queue_(task_queue_factory->CreateTaskQueue( + "ResourceAdaptationQueue", + TaskQueueFactory::Priority::NORMAL)), encoder_queue_(task_queue_factory->CreateTaskQueue( "EncoderQueue", TaskQueueFactory::Priority::NORMAL)) { RTC_DCHECK(encoder_stats_observer); RTC_DCHECK_GE(number_of_cores, 1); - stream_resource_manager_.Initialize(&encoder_queue_); + stream_resource_manager_.Initialize(&encoder_queue_, + &resource_adaptation_queue_); rtc::Event initialize_processor_event; - encoder_queue_.PostTask([this, &initialize_processor_event] { - RTC_DCHECK_RUN_ON(&encoder_queue_); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // initialize the processor on it instead. + resource_adaptation_queue_.PostTask([this, &initialize_processor_event] { + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); resource_adaptation_processor_->InitializeOnResourceAdaptationQueue(); stream_resource_manager_.SetAdaptationProcessor( resource_adaptation_processor_.get()); @@ -307,10 +309,11 @@ VideoStreamEncoder::~VideoStreamEncoder() { void VideoStreamEncoder::Stop() { RTC_DCHECK_RUN_ON(&thread_checker_); video_source_sink_controller_.SetSource(nullptr); - encoder_queue_.PostTask([this] { - RTC_DCHECK_RUN_ON(&encoder_queue_); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // destroy the processor on it instead. + + rtc::Event shutdown_adaptation_processor_event; + resource_adaptation_queue_.PostTask([this, + &shutdown_adaptation_processor_event] { + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); if (resource_adaptation_processor_) { resource_adaptation_processor_->StopResourceAdaptation(); for (Resource* resource : stream_resource_manager_.MappedResources()) { @@ -322,6 +325,11 @@ void VideoStreamEncoder::Stop() { stream_resource_manager_.SetAdaptationProcessor(nullptr); resource_adaptation_processor_.reset(); } + shutdown_adaptation_processor_event.Set(); + }); + shutdown_adaptation_processor_event.Wait(rtc::Event::kForever); + encoder_queue_.PostTask([this] { + RTC_DCHECK_RUN_ON(&encoder_queue_); stream_resource_manager_.StopManagedResources(); rate_allocator_ = nullptr; bitrate_observer_ = nullptr; @@ -359,8 +367,10 @@ void VideoStreamEncoder::SetSource( RTC_DCHECK_RUN_ON(&thread_checker_); video_source_sink_controller_.SetSource(source); input_state_provider_.OnHasInputChanged(source); - encoder_queue_.PostTask([this, degradation_preference] { - RTC_DCHECK_RUN_ON(&encoder_queue_); + + // Set the degradation preference on the adaptation queue. + resource_adaptation_queue_.PostTask([this, degradation_preference] { + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); if (!resource_adaptation_processor_) { // The VideoStreamEncoder was stopped and the processor destroyed before // this task had a chance to execute. No action needed. @@ -368,8 +378,11 @@ void VideoStreamEncoder::SetSource( } resource_adaptation_processor_->SetDegradationPreference( degradation_preference); - stream_resource_manager_.SetDegradationPreferences( - resource_adaptation_processor_->degradation_preference()); + }); + // This may trigger reconfiguring the QualityScaler on the encoder queue. + encoder_queue_.PostTask([this, degradation_preference] { + RTC_DCHECK_RUN_ON(&encoder_queue_); + stream_resource_manager_.SetDegradationPreferences(degradation_preference); if (encoder_) { stream_resource_manager_.ConfigureQualityScaler( encoder_->GetEncoderInfo()); @@ -702,12 +715,16 @@ void VideoStreamEncoder::ReconfigureEncoder() { // invoked later in this method.) stream_resource_manager_.StopManagedResources(); stream_resource_manager_.StartEncodeUsageResource(); - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // PostTask ensuring it is started. - if (resource_adaptation_processor_) { + resource_adaptation_queue_.PostTask([this] { + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); + if (!resource_adaptation_processor_) { + // The VideoStreamEncoder was stopped and the processor destroyed before + // this task had a chance to execute. No action needed. + return; + } // Ensures started. If already started this is a NO-OP. resource_adaptation_processor_->StartResourceAdaptation(); - } + }); pending_encoder_creation_ = false; } @@ -786,12 +803,19 @@ void VideoStreamEncoder::ReconfigureEncoder() { void VideoStreamEncoder::OnEncoderSettingsChanged() { EncoderSettings encoder_settings(encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_); - resource_adaptation_processor_->SetIsScreenshare( - encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen); - stream_resource_manager_.SetDegradationPreferences( - resource_adaptation_processor_->degradation_preference()); - input_state_provider_.OnEncoderSettingsChanged(encoder_settings); stream_resource_manager_.SetEncoderSettings(encoder_settings); + input_state_provider_.OnEncoderSettingsChanged(encoder_settings); + bool is_screenshare = encoder_settings.encoder_config().content_type == + VideoEncoderConfig::ContentType::kScreen; + resource_adaptation_queue_.PostTask([this, is_screenshare] { + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); + if (!resource_adaptation_processor_) { + // The VideoStreamEncoder was stopped and the processor destroyed before + // this task had a chance to execute. No action needed. + return; + } + resource_adaptation_processor_->SetIsScreenshare(is_screenshare); + }); } void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { @@ -1717,9 +1741,7 @@ void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, rtc::scoped_refptr reason) { - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // ensure that this is running on it instead. - RTC_DCHECK_RUN_ON(&encoder_queue_); + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); video_source_sink_controller_.SetRestrictions(std::move(restrictions)); video_source_sink_controller_.PushSourceSinkSettings(); } @@ -1989,14 +2011,23 @@ void VideoStreamEncoder::CheckForAnimatedContent( void VideoStreamEncoder::InjectAdaptationResource( rtc::scoped_refptr resource, VideoAdaptationReason reason) { - rtc::Event inject_resource_event; - encoder_queue_.PostTask([this, resource, reason, &inject_resource_event] { + rtc::Event map_resource_event; + encoder_queue_.PostTask([this, resource, reason, &map_resource_event] { RTC_DCHECK_RUN_ON(&encoder_queue_); stream_resource_manager_.MapResourceToReason(resource, reason); - resource_adaptation_processor_->AddResource(resource); - inject_resource_event.Set(); + map_resource_event.Set(); + }); + map_resource_event.Wait(rtc::Event::kForever); + + resource_adaptation_queue_.PostTask([this, resource] { + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); + if (!resource_adaptation_processor_) { + // The VideoStreamEncoder was stopped and the processor destroyed before + // this task had a chance to execute. No action needed. + return; + } + resource_adaptation_processor_->AddResource(resource); }); - inject_resource_event.Wait(rtc::Event::kForever); } rtc::scoped_refptr @@ -2005,4 +2036,29 @@ VideoStreamEncoder::quality_scaler_resource_for_testing() { return stream_resource_manager_.quality_scaler_resource_for_testing(); } +void VideoStreamEncoder::AddAdaptationListenerForTesting( + ResourceAdaptationProcessorListener* adaptation_listener) { + rtc::Event event; + resource_adaptation_queue_.PostTask([this, adaptation_listener, &event] { + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); + RTC_DCHECK(resource_adaptation_processor_); + resource_adaptation_processor_->AddAdaptationListener(adaptation_listener); + event.Set(); + }); + event.Wait(rtc::Event::kForever); +} + +void VideoStreamEncoder::RemoveAdaptationListenerForTesting( + ResourceAdaptationProcessorListener* adaptation_listener) { + rtc::Event event; + resource_adaptation_queue_.PostTask([this, adaptation_listener, &event] { + RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); + RTC_DCHECK(resource_adaptation_processor_); + resource_adaptation_processor_->RemoveAdaptationListener( + adaptation_listener); + event.Set(); + }); + event.Wait(rtc::Event::kForever); +} + } // namespace webrtc diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 61943551dd..5c72167964 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -106,6 +106,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // Used for testing. For example the |ScalingObserverInterface| methods must // be called on |encoder_queue_|. rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } + rtc::TaskQueue* resource_adaptation_queue() { + return &resource_adaptation_queue_; + } void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, @@ -121,6 +124,11 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, rtc::scoped_refptr quality_scaler_resource_for_testing(); + void AddAdaptationListenerForTesting( + ResourceAdaptationProcessorListener* adaptation_listener); + void RemoveAdaptationListenerForTesting( + ResourceAdaptationProcessorListener* adaptation_listener); + private: class VideoFrameInfo { public: @@ -405,11 +413,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, VideoStreamInputStateProvider input_state_provider_; // Responsible for adapting input resolution or frame rate to ensure resources // (e.g. CPU or bandwidth) are not overused. - // This class is single-threaded. - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // guard the processor by it instead. + // This class is single-threaded on the resource adaptation queue. std::unique_ptr - resource_adaptation_processor_ RTC_GUARDED_BY(&encoder_queue_); + resource_adaptation_processor_ + RTC_GUARDED_BY(&resource_adaptation_queue_); // Handles input, output and stats reporting related to VideoStreamEncoder // specific resources, such as "encode usage percent" measurements and "QP // scaling". Also involved with various mitigations such as inital frame @@ -417,18 +424,19 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // The manager primarily operates on the |encoder_queue_| but its lifetime is // tied to the VideoStreamEncoder (which is destroyed off the encoder queue) // and its resource list is accessible from any thread. - // TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue, - // remove the RTC_GUARDED_BY to get resources on the adaptation queue. - VideoStreamEncoderResourceManager stream_resource_manager_ - RTC_GUARDED_BY(&encoder_queue_); + VideoStreamEncoderResourceManager stream_resource_manager_; // Carries out the VideoSourceRestrictions provided by the // ResourceAdaptationProcessor, i.e. reconfigures the source of video frames // to provide us with different resolution or frame rate. // This class is thread-safe. VideoSourceSinkController video_source_sink_controller_; - // All public methods are proxied to |encoder_queue_|. It must must be - // destroyed first to make sure no tasks are run that use other members. + // Public methods are proxied to the task queues. The queues must be destroyed + // first to make sure no tasks run that use other members. + // TODO(https://crbug.com/webrtc/11172): Move ownership of the + // ResourceAdaptationProcessor and its task queue to Call when processors are + // multi-stream aware. + rtc::TaskQueue resource_adaptation_queue_; rtc::TaskQueue encoder_queue_; RTC_DISALLOW_COPY_AND_ASSIGN(VideoStreamEncoder); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 30660bbce6..5123d45bdd 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -125,13 +125,16 @@ class CpuOveruseDetectorProxy : public OveruseFrameDetector { public: explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer) : OveruseFrameDetector(metrics_observer), - last_target_framerate_fps_(-1) {} + last_target_framerate_fps_(-1), + framerate_updated_event_(true /* manual_reset */, + false /* initially_signaled */) {} virtual ~CpuOveruseDetectorProxy() {} void OnTargetFramerateUpdated(int framerate_fps) override { rtc::CritScope cs(&lock_); last_target_framerate_fps_ = framerate_fps; OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps); + framerate_updated_event_.Set(); } int GetLastTargetFramerate() { @@ -141,9 +144,12 @@ class CpuOveruseDetectorProxy : public OveruseFrameDetector { CpuOveruseOptions GetOptions() { return options_; } + rtc::Event* framerate_updated_event() { return &framerate_updated_event_; } + private: rtc::CriticalSection lock_; int last_target_framerate_fps_ RTC_GUARDED_BY(lock_); + rtc::Event framerate_updated_event_; }; class FakeQualityScalerQpUsageHandlerCallback @@ -165,6 +171,33 @@ class FakeQualityScalerQpUsageHandlerCallback absl::optional clear_qp_samples_result_; }; +class VideoSourceRestrictionsUpdatedListener + : public ResourceAdaptationProcessorListener { + public: + VideoSourceRestrictionsUpdatedListener() + : was_restrictions_updated_(false), restrictions_updated_event_() {} + ~VideoSourceRestrictionsUpdatedListener() override { + RTC_DCHECK(was_restrictions_updated_); + } + + rtc::Event* restrictions_updated_event() { + return &restrictions_updated_event_; + } + + // ResourceAdaptationProcessorListener implementation. + void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason) override { + was_restrictions_updated_ = true; + restrictions_updated_event_.Set(); + } + + private: + bool was_restrictions_updated_; + rtc::Event restrictions_updated_event_; +}; + class VideoStreamEncoderUnderTest : public VideoStreamEncoder { public: VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy, @@ -180,11 +213,55 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { task_queue_factory), fake_cpu_resource_(new FakeResource("FakeResource[CPU]")), fake_quality_resource_(new FakeResource("FakeResource[QP]")) { + fake_cpu_resource_->Initialize(encoder_queue(), + resource_adaptation_queue()); + fake_quality_resource_->Initialize(encoder_queue(), + resource_adaptation_queue()); InjectAdaptationResource(fake_quality_resource_, VideoAdaptationReason::kQuality); InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu); } + void SetSourceAndWaitForRestrictionsUpdated( + rtc::VideoSourceInterface* source, + const DegradationPreference& degradation_preference) { + VideoSourceRestrictionsUpdatedListener listener; + AddAdaptationListenerForTesting(&listener); + SetSource(source, degradation_preference); + listener.restrictions_updated_event()->Wait(5000); + RemoveAdaptationListenerForTesting(&listener); + } + + void SetSourceAndWaitForFramerateUpdated( + rtc::VideoSourceInterface* source, + const DegradationPreference& degradation_preference) { + overuse_detector_proxy_->framerate_updated_event()->Reset(); + SetSource(source, degradation_preference); + overuse_detector_proxy_->framerate_updated_event()->Wait(5000); + } + + void OnBitrateUpdatedAndWaitForManagedResources( + DataRate target_bitrate, + DataRate stable_target_bitrate, + DataRate link_allocation, + uint8_t fraction_lost, + int64_t round_trip_time_ms, + double cwnd_reduce_ratio) { + OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation, + fraction_lost, round_trip_time_ms, cwnd_reduce_ratio); + // Bitrate is updated on the encoder queue. + WaitUntilTaskQueueIsIdle(); + // Give the managed resources time to react to the new bitrate. + // TODO(hbos): Can we await an appropriate event instead? + WaitUntilAdaptationTaskQueueIsIdle(); + } + + void WaitUntilAdaptationTaskQueueIsIdle() { + rtc::Event event; + resource_adaptation_queue()->PostTask([&event] { event.Set(); }); + ASSERT_TRUE(event.Wait(5000)); + } + // This is used as a synchronisation mechanism, to make sure that the // encoder queue is not blocked before we start sending it frames. void WaitUntilTaskQueueIsIdle() { @@ -196,7 +273,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { // Triggers resource usage measurements on the fake CPU resource. void TriggerCpuOveruse() { rtc::Event event; - encoder_queue()->PostTask([this, &event] { + resource_adaptation_queue()->PostTask([this, &event] { fake_cpu_resource_->set_usage_state(ResourceUsageState::kOveruse); event.Set(); }); @@ -204,7 +281,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { } void TriggerCpuUnderuse() { rtc::Event event; - encoder_queue()->PostTask([this, &event] { + resource_adaptation_queue()->PostTask([this, &event] { fake_cpu_resource_->set_usage_state(ResourceUsageState::kUnderuse); event.Set(); }); @@ -214,7 +291,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { // Triggers resource usage measurements on the fake quality resource. void TriggerQualityLow() { rtc::Event event; - encoder_queue()->PostTask([this, &event] { + resource_adaptation_queue()->PostTask([this, &event] { fake_quality_resource_->set_usage_state(ResourceUsageState::kOveruse); event.Set(); }); @@ -222,7 +299,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { } void TriggerQualityHigh() { rtc::Event event; - encoder_queue()->PostTask([this, &event] { + resource_adaptation_queue()->PostTask([this, &event] { fake_quality_resource_->set_usage_state(ResourceUsageState::kUnderuse); event.Set(); }); @@ -237,8 +314,15 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { rtc::scoped_refptr callback = new FakeQualityScalerQpUsageHandlerCallback(); encoder_queue()->PostTask([this, &event, callback] { + // This should post a usage measurement to the adaptation processor. quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback); - event.Set(); + // Give the processor a chance to react and trigger adaptation on the + // adaptation queue. + resource_adaptation_queue()->PostTask([this, &event] { + // Finally, give the QualityScalerResource time to resolve the callback + // on the encoder queue. + encoder_queue()->PostTask([&event] { event.Set(); }); + }); }); EXPECT_TRUE(event.Wait(5000)); EXPECT_TRUE(callback->clear_qp_samples_result().has_value()); @@ -593,7 +677,7 @@ class VideoStreamEncoderTest : public ::testing::Test { EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate)) .Times(1); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1207,7 +1291,7 @@ class VideoStreamEncoderTest : public ::testing::Test { }; TEST_F(VideoStreamEncoderTest, EncodeOneFrame) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1228,7 +1312,7 @@ TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) { video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr)); EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs)); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1242,23 +1326,23 @@ TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) { } TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); WaitForEncodedFrame(1); - video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0), - DataRate::BitsPerSec(0), - DataRate::BitsPerSec(0), 0, 0, 0); + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), + 0, 0, 0); // The encoder will cache up to one frame for a short duration. Adding two // frames means that the first frame will be dropped and the second frame will // be sent when the encoder is resumed. video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr)); video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr)); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1269,7 +1353,7 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) { } TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1285,7 +1369,7 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) { } TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1301,7 +1385,7 @@ TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) { } TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1320,7 +1404,7 @@ TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) { } TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1342,7 +1426,7 @@ TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) { video_stream_encoder_->WaitUntilTaskQueueIsIdle(); // Capture a frame at codec_width_/codec_height_. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1364,14 +1448,14 @@ TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) { } TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); WaitForEncodedFrame(1); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5); @@ -1390,7 +1474,7 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) { TEST_F(VideoStreamEncoderTest, ConfigureEncoderTriggersOnEncoderConfigurationChanged) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1419,7 +1503,7 @@ TEST_F(VideoStreamEncoderTest, } TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1447,7 +1531,7 @@ TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) { TEST_F(VideoStreamEncoderTest, EncoderInstanceDestroyedBeforeAnotherInstanceCreated) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1471,7 +1555,7 @@ TEST_F(VideoStreamEncoderTest, } TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1518,7 +1602,7 @@ TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) { TEST_F(VideoStreamEncoderTest, IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1566,7 +1650,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1602,7 +1686,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1672,7 +1756,7 @@ TEST_F(VideoStreamEncoderTest, } TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1738,7 +1822,7 @@ TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) { constexpr int kRequestedResolutionAlignment = 7; video_source_.set_adaptation_enabled(true); fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1772,7 +1856,7 @@ TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) { video_source_.set_adaptation_enabled(true); // Enable BALANCED preference, no initial limitation. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1858,7 +1942,7 @@ TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) { } TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -1890,7 +1974,7 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { // Set new source, switch to maintain-resolution. test::FrameForwarder new_video_source; - video_stream_encoder_->SetSource( + video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated( &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION); // Give the encoder queue time to process the change in degradation preference // by waiting for an encoded frame. @@ -1921,8 +2005,8 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps); // Turn off degradation completely. - video_stream_encoder_->SetSource(&new_video_source, - webrtc::DegradationPreference::DISABLED); + video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated( + &new_video_source, webrtc::DegradationPreference::DISABLED); // Give the encoder queue time to process the change in degradation preference // by waiting for an encoded frame. new_video_source.IncomingCapturedFrame( @@ -1941,7 +2025,7 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { VerifyFpsMaxResolutionMax(new_video_source.sink_wants()); // Calling SetSource with resolution scaling enabled apply the old SinkWants. - video_stream_encoder_->SetSource( + video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated( &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); // Give the encoder queue time to process the change in degradation preference // by waiting for an encoded frame. @@ -1955,7 +2039,7 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps); // Calling SetSource with framerate scaling enabled apply the old SinkWants. - video_stream_encoder_->SetSource( + video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated( &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION); // Give the encoder queue time to process the change in degradation preference // by waiting for an encoded frame. @@ -1972,7 +2056,7 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { } TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2008,7 +2092,7 @@ TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) { } TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2044,7 +2128,7 @@ TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) { } TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2115,7 +2199,7 @@ TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) { } TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2178,7 +2262,7 @@ TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) { TEST_F(VideoStreamEncoderTest, QualityAdaptationStatsAreResetWhenScalerIsDisabled) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2236,7 +2320,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2303,7 +2387,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2442,7 +2526,7 @@ TEST_F(VideoStreamEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2493,7 +2577,7 @@ TEST_F(VideoStreamEncoderTest, SkipsSameAdaptDownRequest_MaintainFramerateMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2528,7 +2612,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2571,7 +2655,7 @@ TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_MaintainFramerateMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2600,7 +2684,7 @@ TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_MaintainResolutionMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2628,7 +2712,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2658,7 +2742,7 @@ TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) { TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2689,7 +2773,7 @@ TEST_F(VideoStreamEncoderTest, AdaptsResolutionForLowQuality_MaintainFramerateMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2729,7 +2813,7 @@ TEST_F(VideoStreamEncoderTest, const int kWidth = 1280; const int kHeight = 720; const int kInputFps = 30; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2777,7 +2861,7 @@ TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) { const int kHeight = 720; const size_t kNumFrames = 10; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2815,7 +2899,7 @@ TEST_F(VideoStreamEncoderTest, AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2876,7 +2960,7 @@ TEST_F(VideoStreamEncoderTest, AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -2937,7 +3021,7 @@ TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) { fake_encoder_.SetResolutionBitrateLimits( {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p}); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, @@ -2955,7 +3039,7 @@ TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) { WaitForEncodedFrame(1280, 720); // Reduce bitrate and trigger adapt down. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, @@ -2973,7 +3057,7 @@ TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) { VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720); // Increase bitrate. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, @@ -2991,7 +3075,7 @@ TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) { {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p}); // Set bitrate equal to min bitrate of 540p. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, @@ -3031,7 +3115,7 @@ class BalancedDegradationTest : public VideoStreamEncoderTest { } void OnBitrateUpdated(int bitrate_bps) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps), 0, 0, 0); } @@ -3323,7 +3407,7 @@ TEST_F(VideoStreamEncoderTest, AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) { const int kWidth = 1280; const int kHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -3463,7 +3547,7 @@ TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) { const int kWidth = 640; const int kHeight = 360; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -3492,7 +3576,7 @@ TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) { TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsNotReportedForDisabledDegradation) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -3527,7 +3611,7 @@ TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) { EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate)) .Times(1); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kLowTargetBitrateBps), DataRate::BitsPerSec(kLowTargetBitrateBps), DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); @@ -3636,7 +3720,7 @@ TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) { const int kFrameHeight = 720; const int kFramerate = 24; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -3699,7 +3783,7 @@ TEST_F(VideoStreamEncoderTest, const int kLowFramerate = 15; const int kHighFramerate = 25; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -3765,7 +3849,7 @@ TEST_F(VideoStreamEncoderTest, const int kFrameHeight = 720; const int kFramerate = 24; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -3801,9 +3885,8 @@ TEST_F(VideoStreamEncoderTest, // Change degradation preference to not enable framerate scaling. Target // framerate should be changed to codec defined limit. - video_stream_encoder_->SetSource( + video_stream_encoder_->SetSourceAndWaitForFramerateUpdated( &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); EXPECT_EQ( video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), kFramerate); @@ -3813,7 +3896,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) { const int kTooLowBitrateForFrameSizeBps = 10000; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0); @@ -3845,7 +3928,7 @@ TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) { TEST_F(VideoStreamEncoderTest, NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) { const int kTooLowBitrateForFrameSizeBps = 10000; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0); @@ -3872,7 +3955,7 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWithMaintainResolutionPreference) { const int kWidth = 640; const int kHeight = 360; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kLowTargetBitrateBps), DataRate::BitsPerSec(kLowTargetBitrateBps), DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); @@ -3899,7 +3982,7 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) { video_encoder_config.video_format.parameters["foo"] = "foo"; video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), kMaxPayloadLength); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kLowTargetBitrateBps), DataRate::BitsPerSec(kLowTargetBitrateBps), DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); @@ -3927,7 +4010,7 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) { const int kWidth = 640; const int kHeight = 360; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -3935,7 +4018,7 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) { // Frame should not be dropped. WaitForEncodedFrame(1); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0); @@ -3943,7 +4026,7 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) { // Frame should not be dropped. WaitForEncodedFrame(2); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0); @@ -3975,10 +4058,10 @@ TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { // Start at low bitrate. const int kLowBitrateBps = 200000; - video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(kLowBitrateBps), - DataRate::BitsPerSec(kLowBitrateBps), - DataRate::BitsPerSec(kLowBitrateBps), - 0, 0, 0); + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kLowBitrateBps), + DataRate::BitsPerSec(kLowBitrateBps), + DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0); // Expect first frame to be dropped and resolution to be limited. const int kWidth = 1280; @@ -3990,8 +4073,8 @@ TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight); // Increase bitrate to encoder max. - video_stream_encoder_->OnBitrateUpdated(max_bitrate, max_bitrate, max_bitrate, - 0, 0, 0); + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + max_bitrate, max_bitrate, max_bitrate, 0, 0, 0); // Insert frames and advance |min_duration_ms|. for (size_t i = 1; i <= 10; i++) { @@ -4008,6 +4091,9 @@ TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { timestamp_ms += kFrameIntervalMs; source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight)); WaitForEncodedFrame(timestamp_ms); + // The ramp-up code involves the adaptation queue, give it time to execute. + // TODO(hbos): Can we await an appropriate event instead? + video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle(); VerifyFpsMaxResolutionMax(source.sink_wants()); // Frame should not be adapted. @@ -4023,7 +4109,7 @@ TEST_F(VideoStreamEncoderTest, ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) { const int kTooSmallWidth = 10; const int kTooSmallHeight = 10; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4051,7 +4137,7 @@ TEST_F(VideoStreamEncoderTest, const int kTooSmallWidth = 10; const int kTooSmallHeight = 10; const int kFpsLimit = 7; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4087,7 +4173,7 @@ TEST_F(VideoStreamEncoderTest, TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) { fake_encoder_.ForceInitEncodeFailure(true); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4103,7 +4189,7 @@ TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) { // TODO(sprang): Extend this with fps throttling and any "balanced" extensions. TEST_F(VideoStreamEncoderTest, AdaptsResolutionOnOveruse_MaintainFramerateMode) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4139,7 +4225,7 @@ TEST_F(VideoStreamEncoderTest, const int kFrameWidth = 1280; const int kFrameHeight = 720; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4243,7 +4329,7 @@ TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) { // disable frame dropping and make testing easier. ResetEncoder("VP8", 1, 2, 1, true); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4284,7 +4370,7 @@ TEST_F(VideoStreamEncoderTest, const int kHeight = 720; const int64_t kFrameIntervalMs = 150; int64_t timestamp_ms = kFrameIntervalMs; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4467,7 +4553,7 @@ TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) { const int kHeight = 720; const int64_t kFrameIntervalMs = 150; int64_t timestamp_ms = kFrameIntervalMs; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4583,7 +4669,7 @@ TEST_F(VideoStreamEncoderTest, const int kFpsLimit = 15; const int64_t kFrameIntervalMs = 150; int64_t timestamp_ms = kFrameIntervalMs; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4674,7 +4760,7 @@ TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) { const int kAdaptedFrameHeight = 808; const int kFramerate = 24; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4710,7 +4796,7 @@ TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) { const int kLowFps = 2; const int kHighFps = 30; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4727,7 +4813,7 @@ TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) { } // Make sure encoder is updated with new target. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4767,7 +4853,7 @@ TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) { MockBitrateObserver bitrate_observer; video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4781,9 +4867,9 @@ TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) { WaitForEncodedFrame(timestamp_ms); // Next, simulate video suspension due to pacer queue overrun. - video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0), - DataRate::BitsPerSec(0), - DataRate::BitsPerSec(0), 0, 1, 0); + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), + 0, 1, 0); // Skip ahead until a new periodic parameter update should have occured. timestamp_ms += kProcessIntervalMs; @@ -4803,7 +4889,7 @@ TEST_F(VideoStreamEncoderTest, const int kFrameWidth = 1280; const int kFrameHeight = 720; const CpuOveruseOptions default_options; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4828,7 +4914,7 @@ TEST_F(VideoStreamEncoderTest, hardware_options.high_encode_usage_threshold_percent = 200; fake_encoder_.SetIsHardwareAccelerated(true); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4851,7 +4937,7 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) { const int kTargetBitrateBps = 120000; const int kNumFramesInRun = kFps * 5; // Runs of five seconds. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4888,7 +4974,7 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) { overshoot_factor *= 2; } fake_encoder_.SimulateOvershoot(overshoot_factor); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps + 1000), DataRate::BitsPerSec(kTargetBitrateBps + 1000), DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0); @@ -4903,7 +4989,7 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) { timestamp_ms += 1000 / kFps; } - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4928,7 +5014,7 @@ TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) { int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec; max_framerate_ = kActualInputFps; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4949,7 +5035,7 @@ TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) { TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) { VideoFrame::UpdateRect rect; - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -4995,7 +5081,7 @@ TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) { } TEST_F(VideoStreamEncoderTest, SetsFrameTypes) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -5028,7 +5114,7 @@ TEST_F(VideoStreamEncoderTest, SetsFrameTypes) { TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) { // Setup simulcast with three streams. ResetEncoder("VP8", 3, 1, 1, false); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kSimulcastTargetBitrateBps), DataRate::BitsPerSec(kSimulcastTargetBitrateBps), DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0); @@ -5071,7 +5157,7 @@ TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) { // Configure internal source factory and setup test again. encoder_factory_.SetHasInternalSource(true); ResetEncoder("VP8", 1, 1, 1, false); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -5110,7 +5196,7 @@ TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) { // Configure internal source factory and setup test again. encoder_factory_.SetHasInternalSource(true); ResetEncoder("VP8", 1, 1, 1, false); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -5203,7 +5289,7 @@ TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) { const int kFrameHeight = 720; const int kTargetBitrateBps = 300000; // To low for HD resolution. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -5244,7 +5330,7 @@ TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) { const int kFrameHeight = 180; // Initial rate. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/DataRate::KilobitsPerSec(300), /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300), /*link_allocation=*/DataRate::KilobitsPerSec(300), @@ -5263,7 +5349,7 @@ TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) { VideoCodec codec_config = fake_encoder_.codec_config(); DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate); DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/target_rate, /*stable_target_bitrate=*/target_rate, /*link_allocation=*/target_rate, @@ -5284,7 +5370,7 @@ TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) { } TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) { - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); @@ -5358,7 +5444,7 @@ TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) { Field(&Config::param, "ping"), Field(&Config::value, "pong"))))); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/DataRate::KilobitsPerSec(50), /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare), /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare), @@ -5393,7 +5479,7 @@ TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) { EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher(_))) .Times(0); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/DataRate::KilobitsPerSec(0), /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0), /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare), @@ -5425,7 +5511,7 @@ TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) { // The VideoStreamEncoder needs some bitrate before it can start encoding, // setting some bitrate so that subsequent calls to WaitForEncodedFrame does // not fail. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop), /*stable_target_bitrate=*/ DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop), @@ -5493,7 +5579,7 @@ TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) { RequestEncoderSwitch(Matcher( Field(&SdpVideoFormat::name, "AV1")))); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/DataRate::KilobitsPerSec(50), /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare), /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare), @@ -5523,7 +5609,7 @@ TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) { // The VideoStreamEncoder needs some bitrate before it can start encoding, // setting some bitrate so that subsequent calls to WaitForEncodedFrame does // not fail. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop), /*stable_target_bitrate=*/ DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop), @@ -5564,7 +5650,7 @@ TEST_F(VideoStreamEncoderTest, // Set initial rate. auto rate = DataRate::KilobitsPerSec(100); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/rate, /*stable_target_bitrate=*/rate, /*link_allocation=*/rate, @@ -5582,7 +5668,7 @@ TEST_F(VideoStreamEncoderTest, // Change of target bitrate propagates to the encoder. auto new_stable_rate = rate - DataRate::KilobitsPerSec(5); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/new_stable_rate, /*stable_target_bitrate=*/new_stable_rate, /*link_allocation=*/rate, @@ -5601,7 +5687,7 @@ TEST_F(VideoStreamEncoderTest, // Set initial rate. auto rate = DataRate::KilobitsPerSec(100); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/rate, /*stable_target_bitrate=*/rate, /*link_allocation=*/rate, @@ -5620,7 +5706,7 @@ TEST_F(VideoStreamEncoderTest, // Set a higher target rate without changing the link_allocation. Should not // reset encoder's rate. auto new_stable_rate = rate - DataRate::KilobitsPerSec(5); - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( /*target_bitrate=*/rate, /*stable_target_bitrate=*/new_stable_rate, /*link_allocation=*/rate, @@ -5647,7 +5733,7 @@ TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) { video_source_.set_adaptation_enabled(true); // BALANCED degradation preference is required for this feature. - video_stream_encoder_->OnBitrateUpdated( + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);