diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h index 94968426f9..4efb7ac6cc 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -86,8 +86,11 @@ class ResourceAdaptationModuleInterface { virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; virtual void SetEncoderSettings(EncoderSettings encoder_settings) = 0; - virtual void SetEncoderTargetBitrate( - absl::optional target_bitrate_bps) = 0; + // TODO(bugs.webrtc.org/11222): This function shouldn't be needed, start + // bitrates should be apart of the constructor ideally. See the comment on + // VideoStreamEncoderInterface::SetStartBitrate. + virtual void SetStartBitrate(DataRate start_bitrate) = 0; + virtual void SetTargetBitrate(DataRate target_bitrate) = 0; // Removes all restrictions; the module will need to adapt all over again. // TODO(hbos): It's not clear why anybody should be able to tell the module to // reset like this; can we get rid of this method? diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index cf93309e75..f0f981558a 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -346,11 +346,13 @@ OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ToString( OveruseFrameDetectorResourceAdaptationModule:: OveruseFrameDetectorResourceAdaptationModule( + Clock* clock, bool experiment_cpu_load_estimator, std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer, ResourceAdaptationModuleListener* adaptation_listener) : adaptation_listener_(adaptation_listener), + clock_(clock), experiment_cpu_load_estimator_(experiment_cpu_load_estimator), has_input_video_(false), degradation_preference_(DegradationPreference::DISABLED), @@ -362,9 +364,10 @@ OveruseFrameDetectorResourceAdaptationModule:: overuse_detector_is_started_(false), last_input_frame_size_(absl::nullopt), target_frame_rate_(absl::nullopt), - target_bitrate_bps_(absl::nullopt), + encoder_target_bitrate_bps_(absl::nullopt), quality_scaler_(nullptr), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), + quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), encoder_settings_(absl::nullopt), encoder_stats_observer_(encoder_stats_observer), initial_framedrop_(0) { @@ -432,9 +435,37 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings( MaybeUpdateTargetFrameRate(); } -void OveruseFrameDetectorResourceAdaptationModule::SetEncoderTargetBitrate( - absl::optional target_bitrate_bps) { - target_bitrate_bps_ = target_bitrate_bps; +void OveruseFrameDetectorResourceAdaptationModule::SetStartBitrate( + DataRate start_bitrate) { + if (!start_bitrate.IsZero()) + encoder_target_bitrate_bps_ = start_bitrate.bps(); + start_bitrate_.set_start_bitrate_ = start_bitrate; + start_bitrate_.set_start_bitrate_time_ms_ = clock_->TimeInMicroseconds(); +} + +void OveruseFrameDetectorResourceAdaptationModule::SetTargetBitrate( + DataRate target_bitrate) { + if (!target_bitrate.IsZero()) + encoder_target_bitrate_bps_ = target_bitrate.bps(); + + // Check for bwe drop experiment + if (start_bitrate_.set_start_bitrate_ > DataRate::Zero() && + !start_bitrate_.has_seen_first_bwe_drop_ && quality_scaler_ && + quality_scaler_settings_.InitialBitrateIntervalMs() && + quality_scaler_settings_.InitialBitrateFactor()) { + int64_t diff_ms = clock_->TimeInMilliseconds() - + start_bitrate_.set_start_bitrate_time_ms_; + if (diff_ms < quality_scaler_settings_.InitialBitrateIntervalMs().value() && + (target_bitrate < + (start_bitrate_.set_start_bitrate_ * + quality_scaler_settings_.InitialBitrateFactor().value()))) { + RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: " + << start_bitrate_.set_start_bitrate_.bps() + << ", target bitrate: " << target_bitrate.bps(); + initial_framedrop_ = 0; + start_bitrate_.has_seen_first_bwe_drop_ = true; + } + } } void OveruseFrameDetectorResourceAdaptationModule:: @@ -515,10 +546,6 @@ bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const { return initial_framedrop_ < kMaxInitialFramedrop; } -void OveruseFrameDetectorResourceAdaptationModule::ResetInitialFrameDropping() { - initial_framedrop_ = 0; -} - void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( absl::optional qp_thresholds) { if (qp_thresholds.has_value()) { @@ -608,9 +635,9 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { case DegradationPreference::BALANCED: { // Check if quality should be increased based on bitrate. if (reason == kQuality && - !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(), - LastInputFrameSizeOrDefault(), - target_bitrate_bps_.value_or(0))) { + !balanced_settings_.CanAdaptUp( + GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault(), + encoder_target_bitrate_bps_.value_or(0))) { return; } // Try scale up framerate, if higher. @@ -631,7 +658,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { if (reason == kQuality && !balanced_settings_.CanAdaptUpResolution( GetVideoCodecTypeOrGeneric(), LastInputFrameSizeOrDefault(), - target_bitrate_bps_.value_or(0))) { + encoder_target_bitrate_bps_.value_or(0))) { return; } // Scale up resolution. @@ -642,7 +669,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { // limits specified by encoder capabilities. if (reason == kQuality && !CanAdaptUpResolution(LastInputFrameSizeOrDefault(), - target_bitrate_bps_.value_or(0))) { + encoder_target_bitrate_bps_.value_or(0))) { return; } diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 12471c7b36..87676074ce 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -27,6 +27,8 @@ #include "api/video_codecs/video_encoder_config.h" #include "call/adaptation/resource_adaptation_module_interface.h" #include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/experiments/quality_scaler_settings.h" +#include "system_wrappers/include/clock.h" #include "video/overuse_frame_detector.h" namespace webrtc { @@ -54,6 +56,7 @@ class OveruseFrameDetectorResourceAdaptationModule // The module can be constructed on any sequence, but must be initialized and // used on a single sequence, e.g. the encoder queue. OveruseFrameDetectorResourceAdaptationModule( + Clock* clock, bool experiment_cpu_load_estimator, std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer, @@ -73,8 +76,8 @@ class OveruseFrameDetectorResourceAdaptationModule void SetDegradationPreference( DegradationPreference degradation_preference) override; void SetEncoderSettings(EncoderSettings encoder_settings) override; - void SetEncoderTargetBitrate( - absl::optional target_bitrate_bps) override; + void SetStartBitrate(DataRate start_bitrate) override; + void SetTargetBitrate(DataRate target_bitrate) override; void ResetVideoSourceRestrictions() override; void OnFrame(const VideoFrame& frame) override; @@ -87,8 +90,6 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional encode_duration_us) override; void OnFrameDropped(EncodedImageCallback::DropReason reason) override; bool DropInitialFrames() const; - // TODO(eshr): Remove once all qp-scaling is in this class. - void ResetInitialFrameDropping(); // TODO(eshr): This can be made private if we configure on // SetDegredationPreference and SetEncoderSettings. @@ -159,6 +160,12 @@ class OveruseFrameDetectorResourceAdaptationModule enum class Mode { kAdaptUp, kAdaptDown } mode_; }; + struct StartBitrate { + bool has_seen_first_bwe_drop_ = false; + DataRate set_start_bitrate_ = DataRate::Zero(); + int64_t set_start_bitrate_time_ms_ = 0; + }; + CpuOveruseOptions GetCpuOveruseOptions() const; VideoCodecType GetVideoCodecTypeOrGeneric() const; int LastInputFrameSizeOrDefault() const; @@ -184,6 +191,7 @@ class OveruseFrameDetectorResourceAdaptationModule bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const; ResourceAdaptationModuleListener* const adaptation_listener_; + Clock* clock_; const bool experiment_cpu_load_estimator_; // The restrictions that |adaptation_listener_| is informed of. VideoSourceRestrictions video_source_restrictions_; @@ -205,9 +213,12 @@ class OveruseFrameDetectorResourceAdaptationModule bool overuse_detector_is_started_; absl::optional last_input_frame_size_; absl::optional target_frame_rate_; - absl::optional target_bitrate_bps_; + // This is the last non-zero target bitrate for the encoder. + absl::optional encoder_target_bitrate_bps_; std::unique_ptr quality_scaler_; const bool quality_scaling_experiment_enabled_; + const QualityScalerSettings quality_scaler_settings_; + StartBitrate start_bitrate_; absl::optional encoder_settings_; VideoStreamEncoderObserver* const encoder_stats_observer_; // Counts how many frames we've dropped in the initial framedrop phase. diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index cef407a42c..fe27bc0ff9 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -264,9 +264,6 @@ VideoStreamEncoder::VideoStreamEncoder( crop_width_(0), crop_height_(0), encoder_target_bitrate_bps_(absl::nullopt), - set_start_bitrate_bps_(0), - set_start_bitrate_time_ms_(0), - has_seen_first_bwe_drop_(false), max_data_payload_length_(0), encoder_paused_and_dropped_frame_(false), was_encode_called_since_last_initialization_(false), @@ -303,6 +300,7 @@ VideoStreamEncoder::VideoStreamEncoder( /*source=*/nullptr)), resource_adaptation_module_( std::make_unique( + clock_, settings_.experiment_cpu_load_estimator, std::move(overuse_detector), encoder_stats_observer, @@ -391,10 +389,8 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { encoder_target_bitrate_bps_ = start_bitrate_bps != 0 ? absl::optional(start_bitrate_bps) : absl::nullopt; - resource_adaptation_module_->SetEncoderTargetBitrate( - encoder_target_bitrate_bps_); - set_start_bitrate_bps_ = start_bitrate_bps; - set_start_bitrate_time_ms_ = clock_->TimeInMilliseconds(); + resource_adaptation_module_->SetStartBitrate( + DataRate::bps(start_bitrate_bps)); }); } @@ -1537,23 +1533,6 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, << " packet loss " << static_cast(fraction_lost) << " rtt " << round_trip_time_ms; - if (set_start_bitrate_bps_ > 0 && !has_seen_first_bwe_drop_ && - resource_adaptation_module_->quality_scaler() && - quality_scaler_settings_.InitialBitrateIntervalMs() && - quality_scaler_settings_.InitialBitrateFactor()) { - int64_t diff_ms = clock_->TimeInMilliseconds() - set_start_bitrate_time_ms_; - if (diff_ms < quality_scaler_settings_.InitialBitrateIntervalMs().value() && - (target_bitrate.bps() < - (set_start_bitrate_bps_ * - quality_scaler_settings_.InitialBitrateFactor().value()))) { - RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: " - << set_start_bitrate_bps_ - << ", target bitrate: " << target_bitrate.bps(); - resource_adaptation_module_->ResetInitialFrameDropping(); - has_seen_first_bwe_drop_ = true; - } - } - if (encoder_) { encoder_->OnPacketLossRateUpdate(static_cast(fraction_lost) / 256.f); encoder_->OnRttUpdate(round_trip_time_ms); @@ -1571,8 +1550,8 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, if (target_bitrate.bps() != 0) encoder_target_bitrate_bps_ = target_bitrate.bps(); - resource_adaptation_module_->SetEncoderTargetBitrate( - encoder_target_bitrate_bps_); + + resource_adaptation_module_->SetTargetBitrate(target_bitrate); if (video_suspension_changed) { RTC_LOG(LS_INFO) << "Video suspend state changed to: " diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 1258054be8..bba2d8d070 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -248,9 +248,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, int crop_height_ RTC_GUARDED_BY(&encoder_queue_); absl::optional encoder_target_bitrate_bps_ RTC_GUARDED_BY(&encoder_queue_); - int set_start_bitrate_bps_ RTC_GUARDED_BY(&encoder_queue_); - int64_t set_start_bitrate_time_ms_ RTC_GUARDED_BY(&encoder_queue_); - bool has_seen_first_bwe_drop_ RTC_GUARDED_BY(&encoder_queue_); size_t max_data_payload_length_ RTC_GUARDED_BY(&encoder_queue_); absl::optional last_encoder_rate_settings_ RTC_GUARDED_BY(&encoder_queue_);