diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 22f0b56e35..379df7b1d0 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -98,6 +98,8 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { set_start_bitrate_(DataRate::Zero()), set_start_bitrate_time_ms_(0), initial_framedrop_(0), + use_bandwidth_allocation_(false), + bandwidth_allocation_(DataRate::Zero()), last_input_width_(0), last_input_height_(0) { RTC_DCHECK(quality_scaler_resource_); @@ -112,12 +114,23 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { return single_active_stream_pixels_; } + absl::optional UseBandwidthAllocationBps() const { + return (use_bandwidth_allocation_ && + bandwidth_allocation_ > DataRate::Zero()) + ? absl::optional(bandwidth_allocation_.bps()) + : absl::nullopt; + } + // Input signals. void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) { set_start_bitrate_ = start_bitrate; set_start_bitrate_time_ms_ = now_ms; } + void SetBandwidthAllocation(DataRate bandwidth_allocation) { + bandwidth_allocation_ = bandwidth_allocation; + } + void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) { if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ && quality_scaler_resource_->is_started() && @@ -158,6 +171,11 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed " "stream parameters"; initial_framedrop_ = 0; + if (single_active_stream_pixels_ && + GetSingleActiveLayerPixels(codec) > *single_active_stream_pixels_) { + // Resolution increased. + use_bandwidth_allocation_ = true; + } } } last_adaptation_counters_ = adaptation_counters; @@ -169,7 +187,10 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { void OnFrameDroppedDueToSize() { ++initial_framedrop_; } - void Disable() { initial_framedrop_ = kMaxInitialFramedrop; } + void Disable() { + initial_framedrop_ = kMaxInitialFramedrop; + use_bandwidth_allocation_ = false; + } void OnQualityScalerSettingsUpdated() { if (quality_scaler_resource_->is_started()) { @@ -177,7 +198,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { initial_framedrop_ = 0; } else { // Quality scaling disabled so we shouldn't drop initial frames. - initial_framedrop_ = kMaxInitialFramedrop; + Disable(); } } @@ -194,6 +215,8 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { // Counts how many frames we've dropped in the initial framedrop phase. int initial_framedrop_; absl::optional single_active_stream_pixels_; + bool use_bandwidth_allocation_; + DataRate bandwidth_allocation_; std::vector last_active_flags_; VideoAdaptationCounters last_adaptation_counters_; @@ -398,6 +421,8 @@ void VideoStreamEncoderResourceManager::SetEncoderRates( const VideoEncoder::RateControlParameters& encoder_rates) { RTC_DCHECK_RUN_ON(encoder_queue_); encoder_rates_ = encoder_rates; + initial_frame_dropper_->SetBandwidthAllocation( + encoder_rates.bandwidth_allocation); } void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() { @@ -449,6 +474,12 @@ VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const { return initial_frame_dropper_->single_active_stream_pixels(); } +absl::optional +VideoStreamEncoderResourceManager::UseBandwidthAllocationBps() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return initial_frame_dropper_->UseBandwidthAllocationBps(); +} + void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() { RTC_DCHECK_RUN_ON(encoder_queue_); initial_frame_dropper_->Disable(); diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index e5cbeb0692..8eb52292a9 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -130,6 +130,7 @@ class VideoStreamEncoderResourceManager // frames based on size and bitrate. bool DropInitialFrames() const; absl::optional SingleActiveStreamPixels() const; + absl::optional UseBandwidthAllocationBps() const; // VideoSourceRestrictionsListener implementation. // Updates |video_source_restrictions_|. diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 025481ae30..1a2c20e818 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -2055,19 +2055,23 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { } } + uint32_t bitrate_bps = + stream_resource_manager_.UseBandwidthAllocationBps().value_or( + encoder_target_bitrate_bps_.value()); + absl::optional encoder_bitrate_limits = encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( pixel_count); if (encoder_bitrate_limits.has_value()) { // Use bitrate limits provided by encoder. - return encoder_target_bitrate_bps_.value() < + return bitrate_bps < static_cast(encoder_bitrate_limits->min_start_bitrate_bps); } - if (encoder_target_bitrate_bps_.value() < 300000 /* qvga */) { + if (bitrate_bps < 300000 /* qvga */) { return pixel_count > 320 * 240; - } else if (encoder_target_bitrate_bps_.value() < 500000 /* vga */) { + } else if (bitrate_bps < 500000 /* vga */) { return pixel_count > 640 * 480; } return false; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 38e5111b8c..cf46413631 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -5252,6 +5252,10 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) { VideoEncoderConfig video_encoder_config; test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3, &video_encoder_config); + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject( + "VP8", /*max qp*/ 56, /*screencast*/ false, + /*screenshare enabled*/ false); for (auto& layer : video_encoder_config.simulcast_layers) { layer.num_temporal_layers = 1; layer.max_framerate = kDefaultFramerate; @@ -5565,6 +5569,59 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) { + const int kMinStartBps360p = 222000; + fake_encoder_.SetResolutionBitrateLimits( + {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000), + VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000, + 800000)}); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate + DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate + DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation + 0, 0, 0); + // Frame should not be dropped, bitrate not too low for frame. + video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180)); + WaitForEncodedFrame(1); + + // Incoming resolution increases, initial frame drop activates. + // Frame should be dropped, link allocation too low for frame. + video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360)); + ExpectDroppedFrame(); + + // Expect sink_wants to specify a scaled frame. + EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360, + 5000); + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) { + const int kMinStartBps360p = 222000; + fake_encoder_.SetResolutionBitrateLimits( + {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000), + VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000, + 800000)}); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate + DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate + DataRate::BitsPerSec(kMinStartBps360p), // link_allocation + 0, 0, 0); + // Frame should not be dropped, bitrate not too low for frame. + video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180)); + WaitForEncodedFrame(1); + + // Incoming resolution increases, initial frame drop activates. + // Frame should be dropped, link allocation not too low for frame. + video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360)); + WaitForEncodedFrame(2); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { webrtc::test::ScopedFieldTrials field_trials( "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");