From 41c650bea239be0730d31b5879df376b63b49628 Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Mon, 14 Oct 2019 13:12:19 +0200 Subject: [PATCH] Use bitrate limits provided by encoder. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use minimum start bitrate to drop frame and adapt resolution in the beginning of call. - Use minimum bitrate to decide whether or not resolution should be increased based on quality in MAINTAIN_FRAMERATE and BALANCED modes. In BALANCED mode bitrate limits provided by the corresponding field trial are prioritized over the limits provided by encoder. Bug: webrtc:10853 Change-Id: I8257eb64565bcafa6ae9887a1af18e90f8400cac Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/156302 Commit-Queue: Sergey Silkin Reviewed-by: Åsa Persson Cr-Commit-Position: refs/heads/master@{#29461} --- video/video_stream_encoder.cc | 64 ++++++++++++++----- video/video_stream_encoder.h | 2 + video/video_stream_encoder_unittest.cc | 85 ++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 14 deletions(-) diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 1ae4476e73..ead279425a 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -320,6 +320,16 @@ class VideoStreamEncoder::VideoSourceProxy { return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; } + int GetHigherResolutionThan(int pixel_count) const { + // On step down we request at most 3/5 the pixel count of the previous + // resolution, so in order to take "one step up" we request a resolution + // as close as possible to 5/3 of the current resolution. The actual pixel + // count selected depends on the capabilities of the source. In order to + // not take a too large step up, we cap the requested pixel count to be at + // most four time the current number of pixels. + return (pixel_count * 5) / 3; + } + bool RequestHigherResolutionThan(int pixel_count) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); @@ -340,13 +350,7 @@ class VideoStreamEncoder::VideoSourceProxy { // Remove any constraints. sink_wants_.target_pixel_count.reset(); } else { - // On step down we request at most 3/5 the pixel count of the previous - // resolution, so in order to take "one step up" we request a resolution - // as close as possible to 5/3 of the current resolution. The actual pixel - // count selected depends on the capabilities of the source. In order to - // not take a too large step up, we cap the requested pixel count to be at - // most four time the current number of pixels. - sink_wants_.target_pixel_count = (pixel_count * 5) / 3; + sink_wants_.target_pixel_count = GetHigherResolutionThan(pixel_count); } RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; @@ -1861,13 +1865,24 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, } bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { - if (initial_framedrop_ < kMaxInitialFramedrop && - encoder_start_bitrate_bps_ > 0) { - if (encoder_start_bitrate_bps_ < 300000 /* qvga */) { - return pixel_count > 320 * 240; - } else if (encoder_start_bitrate_bps_ < 500000 /* vga */) { - return pixel_count > 640 * 480; - } + if (initial_framedrop_ >= kMaxInitialFramedrop || + encoder_start_bitrate_bps_ == 0) { + return false; + } + + absl::optional encoder_bitrate_limits = + GetEncoderBitrateLimits(encoder_->GetEncoderInfo(), pixel_count); + + if (encoder_bitrate_limits.has_value()) { + // Use bitrate limits provided by encoder. + return encoder_start_bitrate_bps_ < + static_cast(encoder_bitrate_limits->min_start_bitrate_bps); + } + + if (encoder_start_bitrate_bps_ < 300000 /* qvga */) { + return pixel_count > 320 * 240; + } else if (encoder_start_bitrate_bps_ < 500000 /* vga */) { + return pixel_count > 640 * 480; } return false; } @@ -2032,6 +2047,14 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) { RTC_FALLTHROUGH(); } case DegradationPreference::MAINTAIN_FRAMERATE: { + // Check if resolution should be increased based on bitrate and + // limits specified by encoder capabilities. + if (reason == kQuality && + !CanAdaptUpResolution(last_frame_info_->pixel_count(), + encoder_start_bitrate_bps_)) { + return; + } + // Scale up resolution. int pixel_count = adaptation_request.input_pixel_count_; if (adapt_counter.ResolutionCount() == 1) { @@ -2073,6 +2096,19 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) { RTC_LOG(LS_INFO) << adapt_counter.ToString(); } +bool VideoStreamEncoder::CanAdaptUpResolution(int pixels, + uint32_t bitrate_bps) const { + absl::optional bitrate_limits = + GetEncoderBitrateLimits(encoder_info_, + source_proxy_->GetHigherResolutionThan(pixels)); + if (!bitrate_limits.has_value() || bitrate_bps == 0) { + return true; // No limit configured or bitrate provided. + } + RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels); + return bitrate_bps >= + static_cast(bitrate_limits->min_start_bitrate_bps); +} + // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. void VideoStreamEncoder::UpdateAdaptationStats(AdaptReason reason) { switch (reason) { diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index ba9f519475..309052f79a 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -223,6 +223,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(&encoder_queue_); VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( AdaptReason reason) RTC_RUN_ON(&encoder_queue_); + bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const + RTC_RUN_ON(&encoder_queue_); void RunPostEncode(EncodedImage encoded_image, int64_t time_sent_us, int temporal_index); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 31d3aa15eb..2cea12670c 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -67,6 +67,10 @@ const int kMaxInitialFramedrop = 4; const int kDefaultFramerate = 30; const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate; const int64_t kProcessIntervalMs = 1000; +const VideoEncoder::ResolutionBitrateLimits + kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000); +const VideoEncoder::ResolutionBitrateLimits + kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000); uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps, 0x00, 0x00, 0x03, 0x03, 0xF4, @@ -2668,6 +2672,87 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) { + fake_encoder_.SetResolutionBitrateLimits( + {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p}); + + video_stream_encoder_->OnBitrateUpdated( + DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), + DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), + DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, 0); + + // Enable MAINTAIN_FRAMERATE preference, no initial limitation. + AdaptingFrameForwarder source; + source.set_adaptation_enabled(true); + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + + // Insert 720p frame. + int64_t timestamp_ms = kFrameIntervalMs; + source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + WaitForEncodedFrame(1280, 720); + + // Reduce bitrate and trigger adapt down. + video_stream_encoder_->OnBitrateUpdated( + DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), + DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), + DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, 0); + video_stream_encoder_->TriggerQualityLow(); + + // Insert 720p frame. It should be downscaled and encoded. + timestamp_ms += kFrameIntervalMs; + source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + WaitForEncodedFrame(960, 540); + + // Trigger adapt up. Higher resolution should not be requested duo to lack + // of bitrate. + video_stream_encoder_->TriggerQualityHigh(); + VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720); + + // Increase bitrate. + video_stream_encoder_->OnBitrateUpdated( + DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), + DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), + DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, 0); + + // Trigger adapt up. Higher resolution should be requested. + video_stream_encoder_->TriggerQualityHigh(); + VerifyFpsMaxResolutionMax(source.sink_wants()); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) { + fake_encoder_.SetResolutionBitrateLimits( + {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p}); + + // Set bitrate equal to min bitrate of 540p. + video_stream_encoder_->OnBitrateUpdated( + DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), + DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), + DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, 0); + + // Enable MAINTAIN_FRAMERATE preference, no initial limitation. + AdaptingFrameForwarder source; + source.set_adaptation_enabled(true); + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + + // Insert 720p frame. It should be dropped and lower resolution should be + // requested. + int64_t timestamp_ms = kFrameIntervalMs; + source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + ExpectDroppedFrame(); + VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720); + + // Insert 720p frame. It should be downscaled and encoded. + timestamp_ms += kFrameIntervalMs; + source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720)); + WaitForEncodedFrame(960, 540); + + video_stream_encoder_->Stop(); +} + class BalancedDegradationTest : public VideoStreamEncoderTest { protected: void SetupTest() {