diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/resource_adaptation_processor.cc index 4ec643f46a..b3095ed857 100644 --- a/call/adaptation/resource_adaptation_processor.cc +++ b/call/adaptation/resource_adaptation_processor.cc @@ -88,9 +88,6 @@ ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { RTC_DCHECK(resources_.empty()) << "There are resource(s) attached to a ResourceAdaptationProcessor " << "being destroyed."; - RTC_DCHECK(adaptation_constraints_.empty()) - << "There are constaint(s) attached to a ResourceAdaptationProcessor " - << "being destroyed."; stream_adapter_->RemoveRestrictionsListener(this); resource_listener_delegate_->OnProcessorDestroyed(); } @@ -201,24 +198,6 @@ void ResourceAdaptationProcessor::RemoveLimitationsImposedByResource( } } -void ResourceAdaptationProcessor::AddAdaptationConstraint( - AdaptationConstraint* adaptation_constraint) { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - RTC_DCHECK(std::find(adaptation_constraints_.begin(), - adaptation_constraints_.end(), - adaptation_constraint) == adaptation_constraints_.end()); - adaptation_constraints_.push_back(adaptation_constraint); -} - -void ResourceAdaptationProcessor::RemoveAdaptationConstraint( - AdaptationConstraint* adaptation_constraint) { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - auto it = std::find(adaptation_constraints_.begin(), - adaptation_constraints_.end(), adaptation_constraint); - RTC_DCHECK(it != adaptation_constraints_.end()); - adaptation_constraints_.erase(it); -} - void ResourceAdaptationProcessor::OnResourceUsageStateMeasured( rtc::scoped_refptr resource, ResourceUsageState usage_state) { @@ -268,7 +247,7 @@ ResourceAdaptationProcessor::OnResourceUnderuse( RTC_DCHECK(!processing_in_progress_); processing_in_progress_ = true; // How can this stream be adapted up? - Adaptation adaptation = stream_adapter_->GetAdaptationUp(); + Adaptation adaptation = stream_adapter_->GetAdaptationUp(reason_resource); if (adaptation.status() != Adaptation::Status::kValid) { processing_in_progress_ = false; rtc::StringBuilder message; @@ -277,27 +256,12 @@ ResourceAdaptationProcessor::OnResourceUnderuse( return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter, message.Release()); } - VideoSourceRestrictions restrictions_before = - stream_adapter_->source_restrictions(); - VideoSourceRestrictions restrictions_after = adaptation.restrictions(); - // Check that resource is most limited... + // Check that resource is most limited. std::vector> most_limited_resources; VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions; std::tie(most_limited_resources, most_limited_restrictions) = FindMostLimitedResources(); - for (const auto* constraint : adaptation_constraints_) { - if (!constraint->IsAdaptationUpAllowed( - adaptation.input_state(), restrictions_before, restrictions_after, - reason_resource)) { - processing_in_progress_ = false; - rtc::StringBuilder message; - message << "Not adapting up because constraint \"" << constraint->Name() - << "\" disallowed it"; - return MitigationResultAndLogMessage( - MitigationResult::kRejectedByConstraint, message.Release()); - } - } // If the most restricted resource is less limited than current restrictions // then proceed with adapting up. if (!most_limited_resources.empty() && diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h index c10b3f6426..7ba871e104 100644 --- a/call/adaptation/resource_adaptation_processor.h +++ b/call/adaptation/resource_adaptation_processor.h @@ -25,7 +25,6 @@ #include "api/video/video_adaptation_counters.h" #include "api/video/video_frame.h" #include "api/video/video_stream_encoder_observer.h" -#include "call/adaptation/adaptation_constraint.h" #include "call/adaptation/resource_adaptation_processor_interface.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_adapter.h" @@ -71,10 +70,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, void AddResource(rtc::scoped_refptr resource) override; std::vector> GetResources() const override; void RemoveResource(rtc::scoped_refptr resource) override; - void AddAdaptationConstraint( - AdaptationConstraint* adaptation_constraint) override; - void RemoveAdaptationConstraint( - AdaptationConstraint* adaptation_constraint) override; // ResourceListener implementation. // Triggers OnResourceUnderuse() or OnResourceOveruse(). @@ -114,7 +109,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, kNotMostLimitedResource, kSharedMostLimitedResource, kRejectedByAdapter, - kRejectedByConstraint, kAdaptationApplied, }; @@ -160,8 +154,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, RTC_GUARDED_BY(resources_lock_); std::vector resource_limitations_listeners_ RTC_GUARDED_BY(resource_adaptation_queue_); - std::vector adaptation_constraints_ - RTC_GUARDED_BY(resource_adaptation_queue_); // Purely used for statistics, does not ensure mapped resources stay alive. std::map, VideoStreamAdapter::RestrictionsWithCounters> diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h index 59d2323715..6b9afccf3f 100644 --- a/call/adaptation/resource_adaptation_processor_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h @@ -64,10 +64,6 @@ class ResourceAdaptationProcessorInterface { virtual void AddResource(rtc::scoped_refptr resource) = 0; virtual std::vector> GetResources() const = 0; virtual void RemoveResource(rtc::scoped_refptr resource) = 0; - virtual void AddAdaptationConstraint( - AdaptationConstraint* adaptation_constraint) = 0; - virtual void RemoveAdaptationConstraint( - AdaptationConstraint* adaptation_constraint) = 0; }; } // namespace webrtc diff --git a/call/adaptation/resource_adaptation_processor_unittest.cc b/call/adaptation/resource_adaptation_processor_unittest.cc index 42fb497159..69b224e711 100644 --- a/call/adaptation/resource_adaptation_processor_unittest.cc +++ b/call/adaptation/resource_adaptation_processor_unittest.cc @@ -14,7 +14,6 @@ #include "api/scoped_refptr.h" #include "api/video/video_adaptation_counters.h" #include "call/adaptation/resource_adaptation_processor_interface.h" -#include "call/adaptation/test/fake_adaptation_constraint.h" #include "call/adaptation/test/fake_adaptation_listener.h" #include "call/adaptation/test/fake_frame_rate_provider.h" #include "call/adaptation/test/fake_resource.h" @@ -90,7 +89,6 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { input_state_provider_(&frame_rate_provider_), resource_(FakeResource::Create("FakeResource")), other_resource_(FakeResource::Create("OtherFakeResource")), - adaptation_constraint_("FakeAdaptationConstraint"), adaptation_listener_(), video_stream_adapter_( std::make_unique(&input_state_provider_)), @@ -101,7 +99,6 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { video_stream_adapter_->AddRestrictionsListener(&restrictions_listener_); processor_->AddResource(resource_); processor_->AddResource(other_resource_); - processor_->AddAdaptationConstraint(&adaptation_constraint_); video_stream_adapter_->AddAdaptationListener(&adaptation_listener_); } ~ResourceAdaptationProcessorTest() override { @@ -131,7 +128,6 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { if (other_resource_) { processor_->RemoveResource(other_resource_); } - processor_->RemoveAdaptationConstraint(&adaptation_constraint_); video_stream_adapter_->RemoveAdaptationListener(&adaptation_listener_); video_stream_adapter_->RemoveRestrictionsListener(&restrictions_listener_); processor_.reset(); @@ -146,7 +142,6 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { VideoStreamInputStateProvider input_state_provider_; rtc::scoped_refptr resource_; rtc::scoped_refptr other_resource_; - FakeAdaptationConstraint adaptation_constraint_; FakeAdaptationListener adaptation_listener_; std::unique_ptr video_stream_adapter_; std::unique_ptr processor_; @@ -255,20 +250,6 @@ TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) { EXPECT_EQ(VideoSourceRestrictions(), restrictions_listener_.restrictions()); } -TEST_F(ResourceAdaptationProcessorTest, ResourcesCanPreventAdaptingUp) { - video_stream_adapter_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - // Adapt down so that we can adapt up. - resource_->SetUsageState(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); - RestrictSource(restrictions_listener_.restrictions()); - // Adapting up is prevented. - adaptation_constraint_.set_is_adaptation_up_allowed(false); - resource_->SetUsageState(ResourceUsageState::kUnderuse); - EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); -} - TEST_F(ResourceAdaptationProcessorTest, ResourcesCanNotAdaptUpIfNeverAdaptedDown) { video_stream_adapter_->SetDegradationPreference( diff --git a/call/adaptation/video_stream_adapter.cc b/call/adaptation/video_stream_adapter.cc index 438b64e883..ec80a13a08 100644 --- a/call/adaptation/video_stream_adapter.cc +++ b/call/adaptation/video_stream_adapter.cc @@ -153,6 +153,8 @@ const char* Adaptation::StatusToString(Adaptation::Status status) { return "kInsufficientInput"; case Status::kAdaptationDisabled: return "kAdaptationDisabled"; + case Status::kRejectedByConstraint: + return "kRejectedByConstraint"; } } @@ -214,6 +216,9 @@ VideoStreamAdapter::~VideoStreamAdapter() { RTC_DCHECK(adaptation_listeners_.empty()) << "There are listener(s) attached to a VideoStreamAdapter being " "destroyed."; + RTC_DCHECK(adaptation_constraints_.empty()) + << "There are constaint(s) attached to a VideoStreamAdapter being " + "destroyed."; } VideoSourceRestrictions VideoStreamAdapter::source_restrictions() const { @@ -274,6 +279,24 @@ void VideoStreamAdapter::RemoveAdaptationListener( adaptation_listeners_.erase(it); } +void VideoStreamAdapter::AddAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(std::find(adaptation_constraints_.begin(), + adaptation_constraints_.end(), + adaptation_constraint) == adaptation_constraints_.end()); + adaptation_constraints_.push_back(adaptation_constraint); +} + +void VideoStreamAdapter::RemoveAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + auto it = std::find(adaptation_constraints_.begin(), + adaptation_constraints_.end(), adaptation_constraint); + RTC_DCHECK(it != adaptation_constraints_.end()); + adaptation_constraints_.erase(it); +} + void VideoStreamAdapter::SetDegradationPreference( DegradationPreference degradation_preference) { RTC_DCHECK_RUN_ON(&sequence_checker_); @@ -324,16 +347,33 @@ Adaptation VideoStreamAdapter::RestrictionsOrStateToAdaptation( } Adaptation VideoStreamAdapter::GetAdaptationUp( - const VideoStreamInputState& input_state) const { - return RestrictionsOrStateToAdaptation(GetAdaptationUpStep(input_state), - input_state); + const VideoStreamInputState& input_state, + rtc::scoped_refptr resource) const { + RestrictionsOrState step = GetAdaptationUpStep(input_state); + // If an adaptation proposed, check with the constraints that it is ok. + if (absl::holds_alternative(step)) { + RestrictionsWithCounters restrictions = + absl::get(step); + for (const auto* constraint : adaptation_constraints_) { + if (!constraint->IsAdaptationUpAllowed( + input_state, current_restrictions_.restrictions, + restrictions.restrictions, resource)) { + RTC_LOG(INFO) << "Not adapting up because constraint \"" + << constraint->Name() << "\" disallowed it"; + step = Adaptation::Status::kRejectedByConstraint; + } + } + } + return RestrictionsOrStateToAdaptation(step, input_state); } -Adaptation VideoStreamAdapter::GetAdaptationUp() { +Adaptation VideoStreamAdapter::GetAdaptationUp( + rtc::scoped_refptr resource) { RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(resource); VideoStreamInputState input_state = input_state_provider_->InputState(); ++adaptation_validation_id_; - Adaptation adaptation = GetAdaptationUp(input_state); + Adaptation adaptation = GetAdaptationUp(input_state, resource); return adaptation; } diff --git a/call/adaptation/video_stream_adapter.h b/call/adaptation/video_stream_adapter.h index f91be3cead..27699e6aa8 100644 --- a/call/adaptation/video_stream_adapter.h +++ b/call/adaptation/video_stream_adapter.h @@ -20,6 +20,7 @@ #include "api/adaptation/resource.h" #include "api/rtp_parameters.h" #include "api/video/video_adaptation_counters.h" +#include "call/adaptation/adaptation_constraint.h" #include "call/adaptation/adaptation_listener.h" #include "call/adaptation/degradation_preference_provider.h" #include "call/adaptation/video_source_restrictions.h" @@ -77,6 +78,8 @@ class Adaptation final { kInsufficientInput, // Adaptation disabled via degradation preference. kAdaptationDisabled, + // Adaptation up was rejected by a VideoAdaptationConstraint. + kRejectedByConstraint, }; static const char* StatusToString(Status status); @@ -138,6 +141,8 @@ class VideoStreamAdapter { VideoSourceRestrictionsListener* restrictions_listener); void AddAdaptationListener(AdaptationListener* adaptation_listener); void RemoveAdaptationListener(AdaptationListener* adaptation_listener); + void AddAdaptationConstraint(AdaptationConstraint* adaptation_constraint); + void RemoveAdaptationConstraint(AdaptationConstraint* adaptation_constraint); // TODO(hbos): Setting the degradation preference should not clear // restrictions! This is not defined in the spec and is unexpected, there is a @@ -146,7 +151,9 @@ class VideoStreamAdapter { // Returns an adaptation that we are guaranteed to be able to apply, or a // status code indicating the reason why we cannot adapt. - Adaptation GetAdaptationUp(); + // TODO(https://crbug.com/webrtc/11771) |resource| is needed by the + // AdaptationConstraint resources. Remove this parameter when it's removed. + Adaptation GetAdaptationUp(rtc::scoped_refptr resource); Adaptation GetAdaptationDown(); Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, const VideoSourceRestrictions& restrictions); @@ -185,7 +192,10 @@ class VideoStreamAdapter { const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); - Adaptation GetAdaptationUp(const VideoStreamInputState& input_state) const + // TODO(https://crbug.com/webrtc/11771) |resource| is needed by the + // AdaptationConstraint resources. Remove this parameter when it's removed. + Adaptation GetAdaptationUp(const VideoStreamInputState& input_state, + rtc::scoped_refptr resource) const RTC_RUN_ON(&sequence_checker_); Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); @@ -250,6 +260,8 @@ class VideoStreamAdapter { RTC_GUARDED_BY(&sequence_checker_); std::vector adaptation_listeners_ RTC_GUARDED_BY(&sequence_checker_); + std::vector adaptation_constraints_ + RTC_GUARDED_BY(&sequence_checker_); RestrictionsWithCounters current_restrictions_ RTC_GUARDED_BY(&sequence_checker_); diff --git a/call/adaptation/video_stream_adapter_unittest.cc b/call/adaptation/video_stream_adapter_unittest.cc index cdda03cc3e..a6c8f6ece3 100644 --- a/call/adaptation/video_stream_adapter_unittest.cc +++ b/call/adaptation/video_stream_adapter_unittest.cc @@ -14,13 +14,16 @@ #include #include "absl/types/optional.h" +#include "api/scoped_refptr.h" #include "api/video/video_adaptation_reason.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_config.h" +#include "call/adaptation/adaptation_constraint.h" #include "call/adaptation/adaptation_listener.h" #include "call/adaptation/encoder_settings.h" #include "call/adaptation/test/fake_adaptation_listener.h" +#include "call/adaptation/test/fake_resource.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state.h" #include "rtc_base/string_encode.h" @@ -33,6 +36,7 @@ namespace webrtc { using ::testing::_; using ::testing::DoAll; +using ::testing::Return; using ::testing::SaveArg; namespace { @@ -162,6 +166,20 @@ class MockAdaptationListener : public AdaptationListener { (override)); }; +class MockAdaptationConstraint : public AdaptationConstraint { + public: + MOCK_METHOD(bool, + IsAdaptationUpAllowed, + (const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr reason_resource), + (const, override)); + + // MOCK_METHOD(std::string, Name, (), (const, override)); + std::string Name() const override { return "MockAdaptationConstraint"; } +}; + } // namespace class VideoStreamAdapterTest : public ::testing::Test { @@ -169,11 +187,13 @@ class VideoStreamAdapterTest : public ::testing::Test { VideoStreamAdapterTest() : field_trials_(BalancedFieldTrialConfig()), input_state_provider_(), + resource_(FakeResource::Create("FakeResource")), adapter_(&input_state_provider_) {} protected: webrtc::test::ScopedFieldTrials field_trials_; FakeVideoStreamInputStateProvider input_state_provider_; + rtc::scoped_refptr resource_; VideoStreamAdapter adapter_; }; @@ -227,7 +247,7 @@ TEST_F(VideoStreamAdapterTest, MaintainFramerate_IncreasePixelsToFiveThirds) { int input_pixels = fake_stream.input_pixels(); // Go up once. The target is 5/3 and the max is 12/5 of the target. const int target = (input_pixels * 5) / 3; - fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp(resource_)); EXPECT_EQ(static_cast((target * 12) / 5), adapter_.source_restrictions().max_pixels_per_frame()); EXPECT_EQ(static_cast(target), @@ -242,11 +262,11 @@ TEST_F(VideoStreamAdapterTest, MaintainFramerate_IncreasePixelsToUnrestricted) { kDefaultMinPixelsPerFrame); // We are unrestricted by default and should not be able to adapt up. EXPECT_EQ(Adaptation::Status::kLimitReached, - adapter_.GetAdaptationUp().status()); + adapter_.GetAdaptationUp(resource_).status()); // If we go down once and then back up we should not have any restrictions. fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); - fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp(resource_)); EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); EXPECT_EQ(0, adapter_.adaptation_counters().Total()); } @@ -296,7 +316,7 @@ TEST_F(VideoStreamAdapterTest, MaintainResolution_IncreaseFpsToThreeHalves) { EXPECT_EQ(2, adapter_.adaptation_counters().fps_adaptations); int input_fps = fake_stream.input_fps(); // Go up once. The target is 3/2 of the input. - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(absl::nullopt, @@ -314,11 +334,11 @@ TEST_F(VideoStreamAdapterTest, MaintainResolution_IncreaseFpsToUnrestricted) { kDefaultMinPixelsPerFrame); // We are unrestricted by default and should not be able to adapt up. EXPECT_EQ(Adaptation::Status::kLimitReached, - adapter_.GetAdaptationUp().status()); + adapter_.GetAdaptationUp(resource_).status()); // If we go down once and then back up we should not have any restrictions. fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); - fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp(resource_)); EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); EXPECT_EQ(0, adapter_.adaptation_counters().Total()); } @@ -465,7 +485,7 @@ TEST_F(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { // the next resolution configuration up ("high") is kBalancedHighFrameRateFps // and "balanced" prefers adapting frame rate if not already applied. { - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(static_cast(kBalancedHighFrameRateFps), @@ -481,7 +501,7 @@ TEST_F(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { constexpr size_t kReducedPixelsSecondStepUp = (kReducedPixelsThirdStep * 5) / 3; { - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(kReducedPixelsSecondStepUp, @@ -492,7 +512,7 @@ TEST_F(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { // Now that our resolution is back in the high-range, the next frame rate to // try out is "unlimited". { - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(absl::nullopt, adapter_.source_restrictions().max_frame_rate()); @@ -503,7 +523,7 @@ TEST_F(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { constexpr size_t kReducedPixelsFirstStepUp = (kReducedPixelsSecondStepUp * 5) / 3; { - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(kReducedPixelsFirstStepUp, @@ -513,7 +533,7 @@ TEST_F(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { } // The last step up should make us entirely unrestricted. { - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); @@ -528,7 +548,7 @@ TEST_F(VideoStreamAdapterTest, Balanced_LimitReached) { kBalancedLowFrameRateFps, kDefaultMinPixelsPerFrame); // Attempting to adapt up while unrestricted should result in kLimitReached. EXPECT_EQ(Adaptation::Status::kLimitReached, - adapter_.GetAdaptationUp().status()); + adapter_.GetAdaptationUp(resource_).status()); // Adapting down once result in restricted frame rate, in this case we reach // the lowest possible frame rate immediately: kBalancedLowFrameRateFps. fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); @@ -590,12 +610,12 @@ TEST_F(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationUp) { fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); EXPECT_EQ(2, adapter_.adaptation_counters().resolution_adaptations); // Adapt up once, but don't update the input. - adapter_.ApplyAdaptation(adapter_.GetAdaptationUp(), nullptr); + adapter_.ApplyAdaptation(adapter_.GetAdaptationUp(resource_), nullptr); EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); { // Having performed the adaptation, but not updated the input based on the // new restrictions, adapting again in the same direction will not work. - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, adaptation.status()); } @@ -612,16 +632,16 @@ TEST_F(VideoStreamAdapterTest, adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); - fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp(resource_)); EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); EXPECT_EQ(0, adapter_.adaptation_counters().resolution_adaptations); // We should be able to adapt in framerate one last time after the change of // degradation preference. adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); - fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp(resource_)); EXPECT_EQ(0, adapter_.adaptation_counters().fps_adaptations); } @@ -636,16 +656,16 @@ TEST_F(VideoStreamAdapterTest, adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); - fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp(resource_)); EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); EXPECT_EQ(0, adapter_.adaptation_counters().fps_adaptations); // We should be able to adapt in framerate one last time after the change of // degradation preference. adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); - fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp(resource_)); EXPECT_EQ(0, adapter_.adaptation_counters().resolution_adaptations); } @@ -660,12 +680,12 @@ TEST_F(VideoStreamAdapterTest, adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); // Apply adaptation up but don't update input. - adapter_.ApplyAdaptation(adapter_.GetAdaptationUp(), nullptr); + adapter_.ApplyAdaptation(adapter_.GetAdaptationUp(resource_), nullptr); EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, - adapter_.GetAdaptationUp().status()); + adapter_.GetAdaptationUp(resource_).status()); adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); } @@ -728,7 +748,7 @@ TEST_F(VideoStreamAdapterTest, RestrictionBroadcasted) { kDefaultMinPixelsPerFrame); // Not broadcast on invalid ApplyAdaptation. { - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); adapter_.ApplyAdaptation(adaptation, nullptr); EXPECT_EQ(0, listener.calls()); } @@ -754,7 +774,7 @@ TEST_F(VideoStreamAdapterTest, AdaptationHasNextRestrcitions) { kDefaultMinPixelsPerFrame); // When adaptation is not possible. { - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kLimitReached, adaptation.status()); EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions()); EXPECT_EQ(0, adaptation.counters().Total()); @@ -769,7 +789,7 @@ TEST_F(VideoStreamAdapterTest, AdaptationHasNextRestrcitions) { } // When we adapt up. { - Adaptation adaptation = adapter_.GetAdaptationUp(); + Adaptation adaptation = adapter_.GetAdaptationUp(resource_); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); fake_stream.ApplyAdaptation(adaptation); EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions()); @@ -882,7 +902,7 @@ TEST_F(VideoStreamAdapterTest, EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, adapter_.GetAdaptationDown().status()); EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, - adapter_.GetAdaptationUp().status()); + adapter_.GetAdaptationUp(resource_).status()); EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, adapter_.GetAdaptDownResolution().status()); } @@ -907,6 +927,48 @@ TEST_F(VideoStreamAdapterTest, AdaptationListenerReceivesSignalOnAdaptation) { adapter_.RemoveAdaptationListener(&adaptation_listener); } +TEST_F(VideoStreamAdapterTest, AdaptationConstraintAllowsAdaptationsUp) { + testing::StrictMock adaptation_constraint; + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + adapter_.AddAdaptationConstraint(&adaptation_constraint); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down once so we can adapt up later. + auto first_adaptation = adapter_.GetAdaptationDown(); + fake_stream.ApplyAdaptation(first_adaptation); + + EXPECT_CALL( + adaptation_constraint, + IsAdaptationUpAllowed(_, first_adaptation.restrictions(), _, resource_)) + .WillOnce(Return(true)); + EXPECT_EQ(Adaptation::Status::kValid, + adapter_.GetAdaptationUp(resource_).status()); + adapter_.RemoveAdaptationConstraint(&adaptation_constraint); +} + +TEST_F(VideoStreamAdapterTest, AdaptationConstraintDisallowsAdaptationsUp) { + testing::StrictMock adaptation_constraint; + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + adapter_.AddAdaptationConstraint(&adaptation_constraint); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down once so we can adapt up later. + auto first_adaptation = adapter_.GetAdaptationDown(); + fake_stream.ApplyAdaptation(first_adaptation); + + EXPECT_CALL( + adaptation_constraint, + IsAdaptationUpAllowed(_, first_adaptation.restrictions(), _, resource_)) + .WillOnce(Return(false)); + EXPECT_EQ(Adaptation::Status::kRejectedByConstraint, + adapter_.GetAdaptationUp(resource_).status()); + adapter_.RemoveAdaptationConstraint(&adaptation_constraint); +} + // Death tests. // Disabled on Android because death tests misbehave on Android, see // base/test/gtest_util.h. diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 098568881d..dc19e1c787 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -173,8 +173,9 @@ bool VideoStreamEncoderResourceManager::BitrateConstraint:: manager_->GetReasonFromResource(reason_resource); // If increasing resolution due to kQuality, make sure bitrate limits are not // violated. - // TODO(hbos): Why are we allowing violating bitrate constraints if adapting - // due to CPU? Shouldn't this condition be checked regardless of reason? + // TODO(https://crbug.com/webrtc/11771): Why are we allowing violating bitrate + // constraints if adapting due to CPU? Shouldn't this condition be checked + // regardless of reason? if (reason == VideoAdaptationReason::kQuality && DidIncreaseResolution(restrictions_before, restrictions_after)) { uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0); @@ -235,8 +236,9 @@ bool VideoStreamEncoderResourceManager::BalancedConstraint:: manager_->GetReasonFromResource(reason_resource); // Don't adapt if BalancedDegradationSettings applies and determines this will // exceed bitrate constraints. - // TODO(hbos): Why are we allowing violating balanced settings if adapting due - // CPU? Shouldn't this condition be checked regardless of reason? + // TODO(https://crbug.com/webrtc/11771): Why are we allowing violating + // balanced settings if adapting due CPU? Shouldn't this condition be checked + // regardless of reason? if (reason == VideoAdaptationReason::kQuality && degradation_preference_provider_->degradation_preference() == DegradationPreference::BALANCED && diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 9776e06593..c0fcd1f7c3 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -390,7 +390,7 @@ VideoStreamEncoder::VideoStreamEncoder( resource_adaptation_processor_->AddResource(resource); } for (auto* constraint : adaptation_constraints_) { - resource_adaptation_processor_->AddAdaptationConstraint(constraint); + video_stream_adapter_->AddAdaptationConstraint(constraint); } for (auto* listener : stream_resource_manager_.AdaptationListeners()) { video_stream_adapter_->AddAdaptationListener(listener); @@ -424,7 +424,7 @@ void VideoStreamEncoder::Stop() { // adaptation_constraints_ and adaptation_listeners_ fields are guarded by // this queue. for (auto* constraint : adaptation_constraints_) { - resource_adaptation_processor_->RemoveAdaptationConstraint(constraint); + video_stream_adapter_->RemoveAdaptationConstraint(constraint); } for (auto* listener : stream_resource_manager_.AdaptationListeners()) { video_stream_adapter_->RemoveAdaptationListener(listener); @@ -2111,8 +2111,7 @@ void VideoStreamEncoder::InjectAdaptationConstraint( return; } adaptation_constraints_.push_back(adaptation_constraint); - resource_adaptation_processor_->AddAdaptationConstraint( - adaptation_constraint); + video_stream_adapter_->AddAdaptationConstraint(adaptation_constraint); event.Set(); }); event.Wait(rtc::Event::kForever);