diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h index 074857d866..4f4a302fb2 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -113,7 +113,13 @@ class ResourceAdaptationModuleInterface { // TODO(hbos): If we take frame rate into account perhaps it would be valid to // adapt down in frame rate as well. virtual void OnFrameDroppedDueToSize() = 0; - // 2.ii) An input frame is about to be encoded. It may have been cropped and + // 2.ii) If the frame will not be dropped due to size then signal that it may + // get encoded. However the frame is not guaranteed to be encoded right away + // or ever (for example if encoding is paused). + // TODO(eshr): Try replace OnMaybeEncodeFrame and merge behaviour into + // EncodeStarted. + virtual void OnMaybeEncodeFrame() = 0; + // 2.iii) An input frame is about to be encoded. It may have been cropped and // have different dimensions than what was observed at OnFrame(). Next // up: encoding completes or fails, see OnEncodeCompleted(). There is // currently no signal for encode failure. diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index 44038579ee..c67e47e262 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -66,6 +66,10 @@ VideoSourceRestrictions ApplyDegradationPreference( return source_restrictions; } +// The maximum number of frames to drop at beginning of stream +// to try and achieve desired bitrate. +const int kMaxInitialFramedrop = 4; + } // namespace // VideoSourceRestrictor is responsible for keeping track of current @@ -361,7 +365,8 @@ OveruseFrameDetectorResourceAdaptationModule:: target_bitrate_bps_(absl::nullopt), quality_scaler_(nullptr), encoder_settings_(absl::nullopt), - encoder_stats_observer_(encoder_stats_observer) { + encoder_stats_observer_(encoder_stats_observer), + initial_framedrop_(0) { RTC_DCHECK(adaptation_listener_); RTC_DCHECK(overuse_detector_); RTC_DCHECK(encoder_stats_observer_); @@ -459,6 +464,7 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { AdaptationObserverInterface::AdaptReason::kQuality) > res_count) { encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); } + ++initial_framedrop_; } void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted( @@ -499,13 +505,29 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped( } } +void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() { + initial_framedrop_ = kMaxInitialFramedrop; +} + +bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const { + return initial_framedrop_ < kMaxInitialFramedrop; +} + +void OveruseFrameDetectorResourceAdaptationModule::ResetInitialFrameDropping() { + initial_framedrop_ = 0; +} + void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( absl::optional qp_thresholds) { if (qp_thresholds.has_value()) { quality_scaler_ = std::make_unique(this, qp_thresholds.value()); + // Restart frame drops due to size. + initial_framedrop_ = 0; } else { quality_scaler_ = nullptr; + // Quality scaling disabled so we shouldn't drop initial frames. + initial_framedrop_ = kMaxInitialFramedrop; } } diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 21c055b3e5..379e1ad723 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -79,12 +79,16 @@ class OveruseFrameDetectorResourceAdaptationModule void OnFrame(const VideoFrame& frame) override; void OnFrameDroppedDueToSize() override; + void OnMaybeEncodeFrame() override; void OnEncodeStarted(const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) override; void OnEncodeCompleted(const EncodedImage& encoded_image, int64_t time_sent_in_us, absl::optional encode_duration_us) override; void OnFrameDropped(EncodedImageCallback::DropReason reason) override; + bool DropInitialFrames() const; + // TODO(eshr): Remove once all qp-scaling is in this class. + void ResetInitialFrameDropping(); // Use nullopt to disable quality scaling. void UpdateQualityScalerSettings( @@ -210,6 +214,8 @@ class OveruseFrameDetectorResourceAdaptationModule std::unique_ptr quality_scaler_; absl::optional encoder_settings_; VideoStreamEncoderObserver* const encoder_stats_observer_; + // Counts how many frames we've dropped in the initial framedrop phase. + int initial_framedrop_; }; } // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index cf712714f4..e1fdb8c2c6 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -52,10 +52,6 @@ const int64_t kPendingFrameTimeoutMs = 1000; constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper"; -// The maximum number of frames to drop at beginning of stream -// to try and achieve desired bitrate. -const int kMaxInitialFramedrop = 4; - // Averaging window spanning 90 frames at default 30fps, matching old media // optimization module defaults. const int64_t kFrameRateAvergingWindowSizeMs = (1000 / 30) * 90; @@ -258,7 +254,6 @@ VideoStreamEncoder::VideoStreamEncoder( TaskQueueFactory* task_queue_factory) : shutdown_event_(true /* manual_reset */, false), number_of_cores_(number_of_cores), - initial_framedrop_(0), quality_rampup_done_(false), quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), @@ -798,11 +793,9 @@ void VideoStreamEncoder::ConfigureQualityScaler( resource_adaptation_module_->UpdateQualityScalerSettings( experimental_thresholds ? *experimental_thresholds : *(scaling_settings.thresholds)); - initial_framedrop_ = 0; } } else { resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt); - initial_framedrop_ = kMaxInitialFramedrop; } QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler(); @@ -1112,7 +1105,6 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, if (DropDueToSize(video_frame.size())) { RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; resource_adaptation_module_->OnFrameDroppedDueToSize(); - ++initial_framedrop_; // Storing references to a native buffer risks blocking frame capture. if (video_frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) { @@ -1126,7 +1118,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, } return; } - initial_framedrop_ = kMaxInitialFramedrop; + resource_adaptation_module_->OnMaybeEncodeFrame(); if (!quality_rampup_done_ && TryQualityRampup(now_ms) && resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount( @@ -1608,7 +1600,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: " << set_start_bitrate_bps_ << ", target bitrate: " << target_bitrate.bps(); - initial_framedrop_ = 0; + resource_adaptation_module_->ResetInitialFrameDropping(); has_seen_first_bwe_drop_ = true; } } @@ -1648,7 +1640,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, } bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { - if (initial_framedrop_ >= kMaxInitialFramedrop || + if (!resource_adaptation_module_->DropInitialFrames() || !encoder_target_bitrate_bps_.has_value()) { return false; } diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 97a4a66a05..d0b037df1d 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -213,8 +213,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, rtc::Event shutdown_event_; const uint32_t number_of_cores_; - // Counts how many frames we've dropped in the initial framedrop phase. - int initial_framedrop_; bool quality_rampup_done_ RTC_GUARDED_BY(&encoder_queue_); QualityRampupExperiment quality_rampup_experiment_ RTC_GUARDED_BY(&encoder_queue_);