From 7f354f860699375d96da5999b8664e47aa30f4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Thu, 4 Feb 2021 15:52:15 +0100 Subject: [PATCH] Use bandwidth allocation in DropDueToSize when incoming resolution increases. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use bandwidth allocation instead of encoder target bitrate in DropDueToSize when incoming resolution increases to avoid downgrades due to target bitrate being limited by the max bitrate at low resolutions. Bug: none Change-Id: Ic41b31c1a86911d4e97b61b0cbc41ce0da739bd4 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/205622 Commit-Queue: Åsa Persson Reviewed-by: Sergey Silkin Cr-Commit-Position: refs/heads/master@{#33168} --- .../video_stream_encoder_resource_manager.cc | 35 +++++++++++- .../video_stream_encoder_resource_manager.h | 1 + video/video_stream_encoder.cc | 10 +++- video/video_stream_encoder_unittest.cc | 57 +++++++++++++++++++ 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 22f0b56e35..379df7b1d0 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -98,6 +98,8 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { set_start_bitrate_(DataRate::Zero()), set_start_bitrate_time_ms_(0), initial_framedrop_(0), + use_bandwidth_allocation_(false), + bandwidth_allocation_(DataRate::Zero()), last_input_width_(0), last_input_height_(0) { RTC_DCHECK(quality_scaler_resource_); @@ -112,12 +114,23 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { return single_active_stream_pixels_; } + absl::optional UseBandwidthAllocationBps() const { + return (use_bandwidth_allocation_ && + bandwidth_allocation_ > DataRate::Zero()) + ? absl::optional(bandwidth_allocation_.bps()) + : absl::nullopt; + } + // Input signals. void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) { set_start_bitrate_ = start_bitrate; set_start_bitrate_time_ms_ = now_ms; } + void SetBandwidthAllocation(DataRate bandwidth_allocation) { + bandwidth_allocation_ = bandwidth_allocation; + } + void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) { if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ && quality_scaler_resource_->is_started() && @@ -158,6 +171,11 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed " "stream parameters"; initial_framedrop_ = 0; + if (single_active_stream_pixels_ && + GetSingleActiveLayerPixels(codec) > *single_active_stream_pixels_) { + // Resolution increased. + use_bandwidth_allocation_ = true; + } } } last_adaptation_counters_ = adaptation_counters; @@ -169,7 +187,10 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { void OnFrameDroppedDueToSize() { ++initial_framedrop_; } - void Disable() { initial_framedrop_ = kMaxInitialFramedrop; } + void Disable() { + initial_framedrop_ = kMaxInitialFramedrop; + use_bandwidth_allocation_ = false; + } void OnQualityScalerSettingsUpdated() { if (quality_scaler_resource_->is_started()) { @@ -177,7 +198,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { initial_framedrop_ = 0; } else { // Quality scaling disabled so we shouldn't drop initial frames. - initial_framedrop_ = kMaxInitialFramedrop; + Disable(); } } @@ -194,6 +215,8 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { // Counts how many frames we've dropped in the initial framedrop phase. int initial_framedrop_; absl::optional single_active_stream_pixels_; + bool use_bandwidth_allocation_; + DataRate bandwidth_allocation_; std::vector last_active_flags_; VideoAdaptationCounters last_adaptation_counters_; @@ -398,6 +421,8 @@ void VideoStreamEncoderResourceManager::SetEncoderRates( const VideoEncoder::RateControlParameters& encoder_rates) { RTC_DCHECK_RUN_ON(encoder_queue_); encoder_rates_ = encoder_rates; + initial_frame_dropper_->SetBandwidthAllocation( + encoder_rates.bandwidth_allocation); } void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() { @@ -449,6 +474,12 @@ VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const { return initial_frame_dropper_->single_active_stream_pixels(); } +absl::optional +VideoStreamEncoderResourceManager::UseBandwidthAllocationBps() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return initial_frame_dropper_->UseBandwidthAllocationBps(); +} + void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() { RTC_DCHECK_RUN_ON(encoder_queue_); initial_frame_dropper_->Disable(); diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index e5cbeb0692..8eb52292a9 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -130,6 +130,7 @@ class VideoStreamEncoderResourceManager // frames based on size and bitrate. bool DropInitialFrames() const; absl::optional SingleActiveStreamPixels() const; + absl::optional UseBandwidthAllocationBps() const; // VideoSourceRestrictionsListener implementation. // Updates |video_source_restrictions_|. diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 025481ae30..1a2c20e818 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -2055,19 +2055,23 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { } } + uint32_t bitrate_bps = + stream_resource_manager_.UseBandwidthAllocationBps().value_or( + encoder_target_bitrate_bps_.value()); + absl::optional encoder_bitrate_limits = encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( pixel_count); if (encoder_bitrate_limits.has_value()) { // Use bitrate limits provided by encoder. - return encoder_target_bitrate_bps_.value() < + return bitrate_bps < static_cast(encoder_bitrate_limits->min_start_bitrate_bps); } - if (encoder_target_bitrate_bps_.value() < 300000 /* qvga */) { + if (bitrate_bps < 300000 /* qvga */) { return pixel_count > 320 * 240; - } else if (encoder_target_bitrate_bps_.value() < 500000 /* vga */) { + } else if (bitrate_bps < 500000 /* vga */) { return pixel_count > 640 * 480; } return false; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 38e5111b8c..cf46413631 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -5252,6 +5252,10 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) { VideoEncoderConfig video_encoder_config; test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3, &video_encoder_config); + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject( + "VP8", /*max qp*/ 56, /*screencast*/ false, + /*screenshare enabled*/ false); for (auto& layer : video_encoder_config.simulcast_layers) { layer.num_temporal_layers = 1; layer.max_framerate = kDefaultFramerate; @@ -5565,6 +5569,59 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) { + const int kMinStartBps360p = 222000; + fake_encoder_.SetResolutionBitrateLimits( + {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000), + VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000, + 800000)}); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate + DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate + DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation + 0, 0, 0); + // Frame should not be dropped, bitrate not too low for frame. + video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180)); + WaitForEncodedFrame(1); + + // Incoming resolution increases, initial frame drop activates. + // Frame should be dropped, link allocation too low for frame. + video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360)); + ExpectDroppedFrame(); + + // Expect sink_wants to specify a scaled frame. + EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360, + 5000); + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) { + const int kMinStartBps360p = 222000; + fake_encoder_.SetResolutionBitrateLimits( + {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000), + VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000, + 800000)}); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate + DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate + DataRate::BitsPerSec(kMinStartBps360p), // link_allocation + 0, 0, 0); + // Frame should not be dropped, bitrate not too low for frame. + video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180)); + WaitForEncodedFrame(1); + + // Incoming resolution increases, initial frame drop activates. + // Frame should be dropped, link allocation not too low for frame. + video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360)); + WaitForEncodedFrame(2); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { webrtc::test::ScopedFieldTrials field_trials( "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");