diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h index 3cd1869d44..2bf10b955c 100644 --- a/call/adaptation/resource_adaptation_processor.h +++ b/call/adaptation/resource_adaptation_processor.h @@ -54,14 +54,11 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // Triggers OnResourceUnderuse() or OnResourceOveruse(). void OnResourceUsageStateMeasured(const Resource& resource) override; - // May trigger 1-2 adaptations. It is meant to reduce resolution - useful if a - // frame was dropped due to its size - but if you look at the implementation - // you'll find that this is not guaranteed. It could adapt frame rate, which - // does not address the issue. - // TODO(hbos): Can we replace this by something which actually satisfies the - // resolution constraints, or get rid of it altogether? + // May trigger 1-2 adaptations. It is meant to reduce resolution but this is + // not guaranteed. It may adapt frame rate, which does not address the issue. + // TODO(hbos): Can we get rid of this? void TriggerAdaptationDueToFrameDroppedDueToSize( - const Resource& reason_resource); + const Resource& reason_resource) override; private: bool HasSufficientInputForAdaptation( diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h index 89ad903b7d..6984273a29 100644 --- a/call/adaptation/resource_adaptation_processor_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h @@ -66,6 +66,15 @@ class ResourceAdaptationProcessorInterface { DegradationPreference degradation_preference) = 0; virtual void SetIsScreenshare(bool is_screenshare) = 0; virtual void ResetVideoSourceRestrictions() = 0; + + // May trigger one or more adaptations. It is meant to reduce resolution - + // useful if a frame was dropped due to its size - however, the implementation + // may not guarantee this (see resource_adaptation_processor.h). + // TODO(hbos): This is only part of the interface for backwards-compatiblity + // reasons. Can we replace this by something which actually satisfies the + // resolution constraints or get rid of it altogether? + virtual void TriggerAdaptationDueToFrameDroppedDueToSize( + const Resource& reason_resource) = 0; }; } // namespace webrtc diff --git a/video/adaptation/quality_scaler_resource.cc b/video/adaptation/quality_scaler_resource.cc index 3f32fa2ce6..ac35d613cd 100644 --- a/video/adaptation/quality_scaler_resource.cc +++ b/video/adaptation/quality_scaler_resource.cc @@ -12,12 +12,12 @@ #include -#include "call/adaptation/resource_adaptation_processor.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" namespace webrtc { QualityScalerResource::QualityScalerResource( - ResourceAdaptationProcessor* adaptation_processor) + ResourceAdaptationProcessorInterface* adaptation_processor) : adaptation_processor_(adaptation_processor), quality_scaler_(nullptr), pending_qp_usage_callback_(nullptr) {} diff --git a/video/adaptation/quality_scaler_resource.h b/video/adaptation/quality_scaler_resource.h index 8b9d6030e7..30796c3db9 100644 --- a/video/adaptation/quality_scaler_resource.h +++ b/video/adaptation/quality_scaler_resource.h @@ -17,12 +17,11 @@ #include "api/video/video_adaptation_reason.h" #include "api/video_codecs/video_encoder.h" #include "call/adaptation/resource.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" #include "modules/video_coding/utility/quality_scaler.h" namespace webrtc { -class ResourceAdaptationProcessor; - // Handles interaction with the QualityScaler. // TODO(hbos): Add unittests specific to this class, it is currently only tested // indirectly by usage in the ResourceAdaptationProcessor (which is only tested @@ -32,7 +31,7 @@ class QualityScalerResource : public Resource, public QualityScalerQpUsageHandlerInterface { public: explicit QualityScalerResource( - ResourceAdaptationProcessor* adaptation_processor); + ResourceAdaptationProcessorInterface* adaptation_processor); bool is_started() const; @@ -62,7 +61,7 @@ class QualityScalerResource : public Resource, const Resource& reason_resource) override; private: - ResourceAdaptationProcessor* const adaptation_processor_; + ResourceAdaptationProcessorInterface* const adaptation_processor_; std::unique_ptr quality_scaler_; rtc::scoped_refptr pending_qp_usage_callback_; diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 4d0a04b250..b8179d0f71 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -240,7 +240,7 @@ bool VideoStreamEncoderResourceManager::PreventAdaptUpInBalancedResource:: VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, - ResourceAdaptationProcessor* adaptation_processor, + ResourceAdaptationProcessorInterface* adaptation_processor, VideoStreamEncoderObserver* encoder_stats_observer, Clock* clock, bool experiment_cpu_load_estimator, diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 61b1506627..d0e5455529 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -29,7 +29,6 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_config.h" #include "call/adaptation/resource.h" -#include "call/adaptation/resource_adaptation_processor.h" #include "call/adaptation/resource_adaptation_processor_interface.h" #include "call/adaptation/video_stream_adapter.h" #include "call/adaptation/video_stream_input_state_provider.h" @@ -62,7 +61,7 @@ class VideoStreamEncoderResourceManager public: VideoStreamEncoderResourceManager( VideoStreamInputStateProvider* input_state_provider, - ResourceAdaptationProcessor* adaptation_processor, + ResourceAdaptationProcessorInterface* adaptation_processor, VideoStreamEncoderObserver* encoder_stats_observer, Clock* clock, bool experiment_cpu_load_estimator, @@ -235,7 +234,7 @@ class VideoStreamEncoderResourceManager QualityScalerResource quality_scaler_resource_; VideoStreamInputStateProvider* const input_state_provider_; - ResourceAdaptationProcessor* const adaptation_processor_; + ResourceAdaptationProcessorInterface* const adaptation_processor_; VideoStreamEncoderObserver* const encoder_stats_observer_; DegradationPreference degradation_preference_; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 532f2cfba7..fbd60b8659 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -25,6 +25,7 @@ #include "api/video/video_bitrate_allocator_factory.h" #include "api/video/video_codec_constants.h" #include "api/video_codecs/video_encoder.h" +#include "call/adaptation/resource_adaptation_processor.h" #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" #include "modules/video_coding/include/video_codec_initializer.h" #include "rtc_base/arraysize.h" @@ -256,10 +257,12 @@ VideoStreamEncoder::VideoStreamEncoder( ParseAutomatincAnimationDetectionFieldTrial()), encoder_switch_requested_(false), input_state_provider_(encoder_stats_observer), - resource_adaptation_processor_(&input_state_provider_, - encoder_stats_observer), + resource_adaptation_processor_( + std::make_unique( + &input_state_provider_, + encoder_stats_observer)), stream_resource_manager_(&input_state_provider_, - &resource_adaptation_processor_, + resource_adaptation_processor_.get(), encoder_stats_observer, clock_, settings_.experiment_cpu_load_estimator, @@ -272,13 +275,13 @@ VideoStreamEncoder::VideoStreamEncoder( RTC_DCHECK(encoder_stats_observer); RTC_DCHECK_GE(number_of_cores, 1); - resource_adaptation_processor_.AddAdaptationListener( + resource_adaptation_processor_->AddAdaptationListener( &stream_resource_manager_); - resource_adaptation_processor_.AddAdaptationListener(this); + resource_adaptation_processor_->AddAdaptationListener(this); // Add the stream resource manager's resources to the processor. for (Resource* resource : stream_resource_manager_.MappedResources()) - resource_adaptation_processor_.AddResource(resource); + resource_adaptation_processor_->AddResource(resource); for (auto& state : encoder_buffer_state_) state.fill(std::numeric_limits::max()); @@ -296,7 +299,7 @@ void VideoStreamEncoder::Stop() { encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); stream_resource_manager_.StopManagedResources(); - resource_adaptation_processor_.StopResourceAdaptation(); + resource_adaptation_processor_->StopResourceAdaptation(); rate_allocator_ = nullptr; bitrate_observer_ = nullptr; ReleaseEncoder(); @@ -336,11 +339,11 @@ void VideoStreamEncoder::SetSource( encoder_queue_.PostTask([this, source, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); input_state_provider_.OnHasInputChanged(source); - resource_adaptation_processor_.SetDegradationPreference( + resource_adaptation_processor_->SetDegradationPreference( degradation_preference); stream_resource_manager_.SetDegradationPreferences( - resource_adaptation_processor_.degradation_preference(), - resource_adaptation_processor_.effective_degradation_preference()); + resource_adaptation_processor_->degradation_preference(), + resource_adaptation_processor_->effective_degradation_preference()); if (encoder_) { stream_resource_manager_.ConfigureQualityScaler( encoder_->GetEncoderInfo()); @@ -668,9 +671,9 @@ void VideoStreamEncoder::ReconfigureEncoder() { if (pending_encoder_creation_) { stream_resource_manager_.StopManagedResources(); - resource_adaptation_processor_.StopResourceAdaptation(); + resource_adaptation_processor_->StopResourceAdaptation(); stream_resource_manager_.StartEncodeUsageResource(); - resource_adaptation_processor_.StartResourceAdaptation(); + resource_adaptation_processor_->StartResourceAdaptation(); pending_encoder_creation_ = false; } @@ -749,11 +752,11 @@ void VideoStreamEncoder::ReconfigureEncoder() { void VideoStreamEncoder::OnEncoderSettingsChanged() { EncoderSettings encoder_settings(encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_); - resource_adaptation_processor_.SetIsScreenshare( + resource_adaptation_processor_->SetIsScreenshare( encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen); stream_resource_manager_.SetDegradationPreferences( - resource_adaptation_processor_.degradation_preference(), - resource_adaptation_processor_.effective_degradation_preference()); + resource_adaptation_processor_->degradation_preference(), + resource_adaptation_processor_->effective_degradation_preference()); input_state_provider_.OnEncoderSettingsChanged(encoder_settings); stream_resource_manager_.SetEncoderSettings(encoder_settings); } @@ -1879,7 +1882,7 @@ void VideoStreamEncoder::CheckForAnimatedContent( if (!automatic_animation_detection_experiment_.enabled || encoder_config_.content_type != VideoEncoderConfig::ContentType::kScreen || - resource_adaptation_processor_.degradation_preference() != + resource_adaptation_processor_->degradation_preference() != DegradationPreference::BALANCED) { return; } @@ -1947,7 +1950,7 @@ void VideoStreamEncoder::InjectAdaptationResource( Resource* resource, VideoAdaptationReason reason) { stream_resource_manager_.MapResourceToReason(resource, reason); - resource_adaptation_processor_.AddResource(resource); + resource_adaptation_processor_->AddResource(resource); } QualityScalerResource* diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 915b0a34ce..a4a055dbd3 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -404,8 +404,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, RTC_GUARDED_BY(&encoder_queue_); // Responsible for adapting input resolution or frame rate to ensure resources // (e.g. CPU or bandwidth) are not overused. - ResourceAdaptationProcessor resource_adaptation_processor_ - RTC_GUARDED_BY(&encoder_queue_); + std::unique_ptr + resource_adaptation_processor_ RTC_GUARDED_BY(&encoder_queue_); // Handles input, output and stats reporting related to VideoStreamEncoder // specific resources, such as "encode usage percent" measurements and "QP // scaling". Also involved with various mitigations such as inital frame