diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index 95df9c5887..eab01cd290 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -10,6 +10,10 @@ import("../../webrtc.gni") rtc_library("video_adaptation") { sources = [ + "balanced_constraint.cc", + "balanced_constraint.h", + "bitrate_constraint.cc", + "bitrate_constraint.h", "encode_usage_resource.cc", "encode_usage_resource.h", "overuse_frame_detector.cc", diff --git a/video/adaptation/balanced_constraint.cc b/video/adaptation/balanced_constraint.cc new file mode 100644 index 0000000000..a4df474fd0 --- /dev/null +++ b/video/adaptation/balanced_constraint.cc @@ -0,0 +1,65 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "rtc_base/task_utils/to_queued_task.h" +#include "video/adaptation/balanced_constraint.h" + +namespace webrtc { + +BalancedConstraint::BalancedConstraint( + DegradationPreferenceProvider* degradation_preference_provider) + : resource_adaptation_queue_(nullptr), + encoder_target_bitrate_bps_(absl::nullopt), + degradation_preference_provider_(degradation_preference_provider) { + RTC_DCHECK(degradation_preference_provider_); +} + +void BalancedConstraint::SetAdaptationQueue( + TaskQueueBase* resource_adaptation_queue) { + resource_adaptation_queue_ = resource_adaptation_queue; +} + +void BalancedConstraint::OnEncoderTargetBitrateUpdated( + absl::optional encoder_target_bitrate_bps) { + resource_adaptation_queue_->PostTask( + ToQueuedTask([this_ref = rtc::scoped_refptr(this), + encoder_target_bitrate_bps] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_); + this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; + })); +} + +bool BalancedConstraint::IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + // Don't adapt if BalancedDegradationSettings applies and determines this will + // exceed bitrate constraints. + if (degradation_preference_provider_->degradation_preference() == + DegradationPreference::BALANCED && + !balanced_settings_.CanAdaptUp(input_state.video_codec_type(), + input_state.frame_size_pixels().value(), + encoder_target_bitrate_bps_.value_or(0))) { + return false; + } + if (DidIncreaseResolution(restrictions_before, restrictions_after) && + !balanced_settings_.CanAdaptUpResolution( + input_state.video_codec_type(), + input_state.frame_size_pixels().value(), + encoder_target_bitrate_bps_.value_or(0))) { + return false; + } + return true; +} + +} // namespace webrtc diff --git a/video/adaptation/balanced_constraint.h b/video/adaptation/balanced_constraint.h new file mode 100644 index 0000000000..c7e866d009 --- /dev/null +++ b/video/adaptation/balanced_constraint.h @@ -0,0 +1,52 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_BALANCED_CONSTRAINT_H_ +#define VIDEO_ADAPTATION_BALANCED_CONSTRAINT_H_ + +#include + +#include "absl/types/optional.h" +#include "api/task_queue/task_queue_base.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/degradation_preference_provider.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" + +namespace webrtc { + +class BalancedConstraint : public rtc::RefCountInterface, + public AdaptationConstraint { + public: + explicit BalancedConstraint( + DegradationPreferenceProvider* degradation_preference_provider); + ~BalancedConstraint() override = default; + + void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue); + void OnEncoderTargetBitrateUpdated( + absl::optional encoder_target_bitrate_bps); + + // AdaptationConstraint implementation. + std::string Name() const override { return "BalancedConstraint"; } + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const override; + + private: + TaskQueueBase* resource_adaptation_queue_; + absl::optional encoder_target_bitrate_bps_ + RTC_GUARDED_BY(resource_adaptation_queue_); + BalancedDegradationSettings balanced_settings_; + DegradationPreferenceProvider* degradation_preference_provider_; +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_BALANCED_CONSTRAINT_H_ diff --git a/video/adaptation/bitrate_constraint.cc b/video/adaptation/bitrate_constraint.cc new file mode 100644 index 0000000000..b986957685 --- /dev/null +++ b/video/adaptation/bitrate_constraint.cc @@ -0,0 +1,77 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "call/adaptation/video_stream_adapter.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "video/adaptation/bitrate_constraint.h" + +namespace webrtc { + +BitrateConstraint::BitrateConstraint() + : resource_adaptation_queue_(nullptr), + encoder_settings_(absl::nullopt), + encoder_target_bitrate_bps_(absl::nullopt) {} + +void BitrateConstraint::SetAdaptationQueue( + TaskQueueBase* resource_adaptation_queue) { + resource_adaptation_queue_ = resource_adaptation_queue; +} + +void BitrateConstraint::OnEncoderSettingsUpdated( + absl::optional encoder_settings) { + resource_adaptation_queue_->PostTask( + ToQueuedTask([this_ref = rtc::scoped_refptr(this), + encoder_settings] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_); + this_ref->encoder_settings_ = std::move(encoder_settings); + })); +} + +void BitrateConstraint::OnEncoderTargetBitrateUpdated( + absl::optional encoder_target_bitrate_bps) { + resource_adaptation_queue_->PostTask( + ToQueuedTask([this_ref = rtc::scoped_refptr(this), + encoder_target_bitrate_bps] { + RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_); + this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; + })); +} + +bool BitrateConstraint::IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + // Make sure bitrate limits are not violated. + if (DidIncreaseResolution(restrictions_before, restrictions_after)) { + uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0); + absl::optional bitrate_limits = + encoder_settings_.has_value() + ? encoder_settings_->encoder_info() + .GetEncoderBitrateLimitsForResolution( + // Need some sort of expected resulting pixels to be used + // instead of unrestricted. + GetHigherResolutionThan( + input_state.frame_size_pixels().value())) + : absl::nullopt; + if (bitrate_limits.has_value() && bitrate_bps != 0) { + RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, + input_state.frame_size_pixels().value()); + return bitrate_bps >= + static_cast(bitrate_limits->min_start_bitrate_bps); + } + } + return true; +} + +} // namespace webrtc diff --git a/video/adaptation/bitrate_constraint.h b/video/adaptation/bitrate_constraint.h new file mode 100644 index 0000000000..731b4961f5 --- /dev/null +++ b/video/adaptation/bitrate_constraint.h @@ -0,0 +1,56 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_BITRATE_CONSTRAINT_H_ +#define VIDEO_ADAPTATION_BITRATE_CONSTRAINT_H_ + +#include + +#include "absl/types/optional.h" +#include "api/task_queue/task_queue_base.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" + +namespace webrtc { + +class BitrateConstraint : public rtc::RefCountInterface, + public AdaptationConstraint { + public: + BitrateConstraint(); + ~BitrateConstraint() override = default; + + void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue); + void OnEncoderSettingsUpdated( + absl::optional encoder_settings); + void OnEncoderTargetBitrateUpdated( + absl::optional encoder_target_bitrate_bps); + + // AdaptationConstraint implementation. + std::string Name() const override { return "BitrateConstraint"; } + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const override; + + private: + // The |manager_| must be alive as long as this resource is added to the + // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called. + TaskQueueBase* resource_adaptation_queue_; + absl::optional encoder_settings_ + RTC_GUARDED_BY(resource_adaptation_queue_); + absl::optional encoder_target_bitrate_bps_ + RTC_GUARDED_BY(resource_adaptation_queue_); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_BITRATE_CONSTRAINT_H_ diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index c68851fa7e..ad7d88a602 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -131,123 +131,6 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { int initial_framedrop_; }; -VideoStreamEncoderResourceManager::BitrateConstraint::BitrateConstraint( - VideoStreamEncoderResourceManager* manager) - : manager_(manager), - resource_adaptation_queue_(nullptr), - encoder_settings_(absl::nullopt), - encoder_target_bitrate_bps_(absl::nullopt) {} - -void VideoStreamEncoderResourceManager::BitrateConstraint::SetAdaptationQueue( - TaskQueueBase* resource_adaptation_queue) { - resource_adaptation_queue_ = resource_adaptation_queue; -} - -void VideoStreamEncoderResourceManager::BitrateConstraint:: - OnEncoderSettingsUpdated(absl::optional encoder_settings) { - RTC_DCHECK_RUN_ON(manager_->encoder_queue_); - resource_adaptation_queue_->PostTask( - ToQueuedTask([this_ref = rtc::scoped_refptr(this), - encoder_settings] { - RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_); - this_ref->encoder_settings_ = std::move(encoder_settings); - })); -} - -void VideoStreamEncoderResourceManager::BitrateConstraint:: - OnEncoderTargetBitrateUpdated( - absl::optional encoder_target_bitrate_bps) { - RTC_DCHECK_RUN_ON(manager_->encoder_queue_); - resource_adaptation_queue_->PostTask( - ToQueuedTask([this_ref = rtc::scoped_refptr(this), - encoder_target_bitrate_bps] { - RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_); - this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; - })); -} - -bool VideoStreamEncoderResourceManager::BitrateConstraint:: - IsAdaptationUpAllowed( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after) const { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - // Make sure bitrate limits are not violated. - if (DidIncreaseResolution(restrictions_before, restrictions_after)) { - uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0); - absl::optional bitrate_limits = - encoder_settings_.has_value() - ? encoder_settings_->encoder_info() - .GetEncoderBitrateLimitsForResolution( - // Need some sort of expected resulting pixels to be used - // instead of unrestricted. - GetHigherResolutionThan( - input_state.frame_size_pixels().value())) - : absl::nullopt; - if (bitrate_limits.has_value() && bitrate_bps != 0) { - RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, - input_state.frame_size_pixels().value()); - return bitrate_bps >= - static_cast(bitrate_limits->min_start_bitrate_bps); - } - } - return true; -} - -VideoStreamEncoderResourceManager::BalancedConstraint::BalancedConstraint( - VideoStreamEncoderResourceManager* manager, - DegradationPreferenceProvider* degradation_preference_provider) - : manager_(manager), - resource_adaptation_queue_(nullptr), - encoder_target_bitrate_bps_(absl::nullopt), - degradation_preference_provider_(degradation_preference_provider) { - RTC_DCHECK(manager_); - RTC_DCHECK(degradation_preference_provider_); -} - -void VideoStreamEncoderResourceManager::BalancedConstraint::SetAdaptationQueue( - TaskQueueBase* resource_adaptation_queue) { - resource_adaptation_queue_ = resource_adaptation_queue; -} - -void VideoStreamEncoderResourceManager::BalancedConstraint:: - OnEncoderTargetBitrateUpdated( - absl::optional encoder_target_bitrate_bps) { - RTC_DCHECK_RUN_ON(manager_->encoder_queue_); - resource_adaptation_queue_->PostTask( - ToQueuedTask([this_ref = rtc::scoped_refptr(this), - encoder_target_bitrate_bps] { - RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_); - this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; - })); -} - -bool VideoStreamEncoderResourceManager::BalancedConstraint:: - IsAdaptationUpAllowed( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after) const { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - // Don't adapt if BalancedDegradationSettings applies and determines this will - // exceed bitrate constraints. - if (degradation_preference_provider_->degradation_preference() == - DegradationPreference::BALANCED && - !manager_->balanced_settings_.CanAdaptUp( - input_state.video_codec_type(), - input_state.frame_size_pixels().value(), - encoder_target_bitrate_bps_.value_or(0))) { - return false; - } - if (DidIncreaseResolution(restrictions_before, restrictions_after) && - !manager_->balanced_settings_.CanAdaptUpResolution( - input_state.video_codec_type(), - input_state.frame_size_pixels().value(), - encoder_target_bitrate_bps_.value_or(0))) { - return false; - } - return true; -} - VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, VideoStreamEncoderObserver* encoder_stats_observer, @@ -256,9 +139,8 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( std::unique_ptr overuse_detector, DegradationPreferenceProvider* degradation_preference_provider) : degradation_preference_provider_(degradation_preference_provider), - bitrate_constraint_(new rtc::RefCountedObject(this)), + bitrate_constraint_(new rtc::RefCountedObject()), balanced_constraint_(new rtc::RefCountedObject( - this, degradation_preference_provider_)), encode_usage_resource_( EncodeUsageResource::Create(std::move(overuse_detector))), diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index c2e14acde5..471dd13324 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -42,6 +42,8 @@ #include "rtc_base/task_queue.h" #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/clock.h" +#include "video/adaptation/balanced_constraint.h" +#include "video/adaptation/bitrate_constraint.h" #include "video/adaptation/encode_usage_resource.h" #include "video/adaptation/overuse_frame_detector.h" #include "video/adaptation/quality_rampup_experiment_helper.h" @@ -162,68 +164,6 @@ class VideoStreamEncoderResourceManager const std::map& active_counts); - // TODO(hbos): Add tests for manager's constraints. - // Does not trigger adaptations, only prevents adapting up resolution. - class BitrateConstraint : public rtc::RefCountInterface, - public AdaptationConstraint { - public: - explicit BitrateConstraint(VideoStreamEncoderResourceManager* manager); - ~BitrateConstraint() override = default; - - void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue); - void OnEncoderSettingsUpdated( - absl::optional encoder_settings); - void OnEncoderTargetBitrateUpdated( - absl::optional encoder_target_bitrate_bps); - - // AdaptationConstraint implementation. - std::string Name() const override { return "BitrateConstraint"; } - bool IsAdaptationUpAllowed( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after) const override; - - private: - // The |manager_| must be alive as long as this resource is added to the - // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called. - VideoStreamEncoderResourceManager* const manager_; - TaskQueueBase* resource_adaptation_queue_; - absl::optional encoder_settings_ - RTC_GUARDED_BY(resource_adaptation_queue_); - absl::optional encoder_target_bitrate_bps_ - RTC_GUARDED_BY(resource_adaptation_queue_); - }; - - // Does not trigger adaptations, only prevents adapting up in BALANCED. - class BalancedConstraint : public rtc::RefCountInterface, - public AdaptationConstraint { - public: - BalancedConstraint( - VideoStreamEncoderResourceManager* manager, - DegradationPreferenceProvider* degradation_preference_provider); - ~BalancedConstraint() override = default; - - void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue); - void OnEncoderTargetBitrateUpdated( - absl::optional encoder_target_bitrate_bps); - - // AdaptationConstraint implementation. - std::string Name() const override { return "BalancedConstraint"; } - bool IsAdaptationUpAllowed( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after) const override; - - private: - // The |manager_| must be alive as long as this resource is added to the - // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called. - VideoStreamEncoderResourceManager* const manager_; - TaskQueueBase* resource_adaptation_queue_; - absl::optional encoder_target_bitrate_bps_ - RTC_GUARDED_BY(resource_adaptation_queue_); - DegradationPreferenceProvider* degradation_preference_provider_; - }; - DegradationPreferenceProvider* const degradation_preference_provider_; const rtc::scoped_refptr bitrate_constraint_; const rtc::scoped_refptr balanced_constraint_;