diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h index e73adce131..d71ffe8e2d 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -61,6 +61,10 @@ class ResourceAdaptationModuleInterface { virtual void SetHasInputVideo(bool has_input_video) = 0; virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; + // Removes all restrictions; the module will need to adapt all over again. + // TODO(hbos): It's not clear why anybody should be able to tell the module to + // reset like this; can we get rid of this method? + virtual void ResetVideoSourceRestrictions() = 0; }; } // namespace webrtc diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index 20c17092a6..ea082a5e66 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -355,7 +355,9 @@ OveruseFrameDetectorResourceAdaptationModule:: last_frame_pixel_count_(absl::nullopt), source_restrictor_(std::make_unique()), overuse_detector_(std::move(overuse_detector)), - codec_max_framerate_(-1), + overuse_detector_is_started_(false), + codec_max_frame_rate_(absl::nullopt), + target_frame_rate_(absl::nullopt), encoder_start_bitrate_bps_(0), is_quality_scaler_enabled_(false), encoder_config_(), @@ -378,6 +380,7 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoder( void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( ResourceAdaptationModuleListener* adaptation_listener) { RTC_DCHECK(encoder_); + RTC_DCHECK(!overuse_detector_is_started_); // TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside // the interval between StartCheckForOveruse() and StopCheckForOveruse(), // support configuring which |adaptation_listener_| to use on the fly. It is @@ -388,10 +391,16 @@ void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( overuse_detector_->StartCheckForOveruse( TaskQueueBase::Current(), video_stream_encoder_->GetCpuOveruseOptions(), this); + overuse_detector_is_started_ = true; + overuse_detector_->OnTargetFramerateUpdated( + target_frame_rate_.has_value() + ? static_cast(target_frame_rate_.value()) + : std::numeric_limits::max()); } void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { overuse_detector_->StopCheckForOveruse(); + overuse_detector_is_started_ = false; } void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( @@ -418,6 +427,14 @@ void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( MaybeUpdateVideoSourceRestrictions(); } +void OveruseFrameDetectorResourceAdaptationModule:: + ResetVideoSourceRestrictions() { + last_adaptation_request_.reset(); + source_restrictor_->ClearRestrictions(); + adapt_counters_.clear(); + MaybeUpdateVideoSourceRestrictions(); +} + void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured( const VideoFrame& frame, int64_t time_when_first_seen_us) { @@ -443,9 +460,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig( encoder_config_ = std::move(encoder_config); } -void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFramerate( - int codec_max_framerate) { - codec_max_framerate_ = codec_max_framerate; +void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFrameRate( + absl::optional codec_max_frame_rate) { + RTC_DCHECK(!codec_max_frame_rate.has_value() || + codec_max_frame_rate.value() > 0.0); + codec_max_frame_rate_ = codec_max_frame_rate; + MaybeUpdateTargetFrameRate(); } void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps( @@ -458,31 +478,6 @@ void OveruseFrameDetectorResourceAdaptationModule::SetIsQualityScalerEnabled( is_quality_scaler_enabled_ = is_quality_scaler_enabled; } -void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() { - absl::optional restricted_frame_rate = - ApplyDegradationPreference(source_restrictor_->source_restrictions(), - degradation_preference_) - .max_frame_rate(); - // Get the current target framerate, ie the maximum framerate as specified by - // the current codec configuration, or any limit imposed by cpu adaption in - // maintain-resolution or balanced mode. This is used to make sure overuse - // detection doesn't needlessly trigger in low and/or variable framerate - // scenarios. - int target_framerate = - std::min(codec_max_framerate_, - restricted_frame_rate.has_value() - ? static_cast(restricted_frame_rate.value()) - : std::numeric_limits::max()); - overuse_detector_->OnTargetFramerateUpdated(target_framerate); -} - -void OveruseFrameDetectorResourceAdaptationModule::ResetAdaptationCounters() { - last_adaptation_request_.reset(); - source_restrictor_->ClearRestrictions(); - adapt_counters_.clear(); - MaybeUpdateVideoSourceRestrictions(); -} - void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { if (!has_input_video_) return; @@ -575,11 +570,8 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { const int requested_framerate = source_restrictor_->RequestHigherFramerateThan(fps); if (requested_framerate == -1) { - overuse_detector_->OnTargetFramerateUpdated(codec_max_framerate_); return; } - overuse_detector_->OnTargetFramerateUpdated( - std::min(codec_max_framerate_, requested_framerate)); GetAdaptCounter().DecrementFramerate(reason); break; } @@ -683,9 +675,6 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( adaptation_request.framerate_fps_); if (requested_framerate == -1) return true; - RTC_DCHECK_NE(codec_max_framerate_, -1); - overuse_detector_->OnTargetFramerateUpdated( - std::min(codec_max_framerate_, requested_framerate)); GetAdaptCounter().IncrementFramerate(reason); break; } @@ -713,6 +702,33 @@ void OveruseFrameDetectorResourceAdaptationModule:: video_source_restrictions_ = std::move(new_restrictions); adaptation_listener_->OnVideoSourceRestrictionsUpdated( video_source_restrictions_); + MaybeUpdateTargetFrameRate(); + } +} + +void OveruseFrameDetectorResourceAdaptationModule:: + MaybeUpdateTargetFrameRate() { + // The current target framerate is the maximum frame rate as specified by + // the current codec configuration or any limit imposed by the adaptation + // module. This is used to make sure overuse detection doesn't needlessly + // trigger in low and/or variable framerate scenarios. + absl::optional target_frame_rate = + ApplyDegradationPreference(source_restrictor_->source_restrictions(), + degradation_preference_) + .max_frame_rate(); + if (!target_frame_rate.has_value() || + (codec_max_frame_rate_.has_value() && + codec_max_frame_rate_.value() < target_frame_rate.value())) { + target_frame_rate = codec_max_frame_rate_; + } + if (target_frame_rate != target_frame_rate_) { + target_frame_rate_ = target_frame_rate; + if (overuse_detector_is_started_) { + overuse_detector_->OnTargetFramerateUpdated( + target_frame_rate_.has_value() + ? static_cast(target_frame_rate_.value()) + : std::numeric_limits::max()); + } } } diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 82f0c27a56..d3b88e5aeb 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -77,6 +77,7 @@ class OveruseFrameDetectorResourceAdaptationModule void SetHasInputVideo(bool has_input_video) override; void SetDegradationPreference( DegradationPreference degradation_preference) override; + void ResetVideoSourceRestrictions() override; // Input to the OveruseFrameDetector, which are required for this module to // function. These map to OveruseFrameDetector methods. @@ -95,7 +96,7 @@ class OveruseFrameDetectorResourceAdaptationModule // need this many public methods? void SetLastFramePixelCount(absl::optional last_frame_pixel_count); void SetEncoderConfig(VideoEncoderConfig encoder_config); - void SetCodecMaxFramerate(int codec_max_framerate); + void SetCodecMaxFrameRate(absl::optional codec_max_frame_rate); void SetEncoderStartBitrateBps(uint32_t encoder_start_bitrate_bps); // Inform the detector whether or not the quality scaler is enabled. This // helps GetActiveCounts() return absl::nullopt when appropriate. @@ -104,11 +105,6 @@ class OveruseFrameDetectorResourceAdaptationModule // method is called incorrectly. void SetIsQualityScalerEnabled(bool is_quality_scaler_enabled); - // TODO(hbos): Can we get rid of this? Seems we should know whether the frame - // rate has updated. - void RefreshTargetFramerate(); - void ResetAdaptationCounters(); - class AdaptCounter final { public: AdaptCounter(); @@ -189,6 +185,10 @@ class OveruseFrameDetectorResourceAdaptationModule // |adaptation_listener_| if restrictions are changed, allowing the listener // to reconfigure the source accordingly. void MaybeUpdateVideoSourceRestrictions(); + // Calculates an up-to-date value of |target_frame_rate_| and informs the + // |overuse_detector_| of the new value if it changed and the detector is + // started. + void MaybeUpdateTargetFrameRate(); void UpdateAdaptationStats(AdaptReason reason); DegradationPreference EffectiveDegradataionPreference(); @@ -216,7 +216,9 @@ class OveruseFrameDetectorResourceAdaptationModule // Keeps track of source restrictions that this adaptation module outputs. const std::unique_ptr source_restrictor_; const std::unique_ptr overuse_detector_; - int codec_max_framerate_; + bool overuse_detector_is_started_; + absl::optional codec_max_frame_rate_; + absl::optional target_frame_rate_; uint32_t encoder_start_bitrate_bps_; bool is_quality_scaler_enabled_; VideoEncoderConfig encoder_config_; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 6d603f32cf..52dc53f432 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -78,11 +78,6 @@ bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { degradation_preference == DegradationPreference::BALANCED; } -bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { - return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION || - degradation_preference == DegradationPreference::BALANCED; -} - bool RequiresEncoderReset(const VideoCodec& prev_send_codec, const VideoCodec& new_send_codec, bool was_encode_called_since_last_initialization) { @@ -393,13 +388,6 @@ void VideoStreamEncoder::SetSource( degradation_preference); if (encoder_) ConfigureQualityScaler(encoder_->GetEncoderInfo()); - - if (!IsFramerateScalingEnabled(degradation_preference) && - max_framerate_ != -1) { - // If frame rate scaling is no longer allowed, remove any potential - // allowance for longer frame intervals. - resource_adaptation_module_->RefreshTargetFramerate(); - } }); } @@ -599,7 +587,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { // Make sure the start bit rate is sane... RTC_DCHECK_LE(codec.startBitrate, 1000000); max_framerate_ = codec.maxFramerate; - resource_adaptation_module_->SetCodecMaxFramerate(max_framerate_); + resource_adaptation_module_->SetCodecMaxFrameRate(max_framerate_); // Inform source about max configured framerate. int max_framerate = 0; @@ -750,8 +738,6 @@ void VideoStreamEncoder::ReconfigureEncoder() { std::move(streams), encoder_config_.content_type, encoder_config_.min_transmit_bitrate_bps); - resource_adaptation_module_->RefreshTargetFramerate(); - ConfigureQualityScaler(info); } @@ -1137,7 +1123,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, resource_adaptation_module_->GetConstAdaptCounter().TotalCount( AdaptationObserverInterface::AdaptReason::kCpu) == 0) { RTC_LOG(LS_INFO) << "Reset quality limitations."; - resource_adaptation_module_->ResetAdaptationCounters(); + resource_adaptation_module_->ResetVideoSourceRestrictions(); quality_rampup_done_ = true; }