From ac4a90dfdcc77411855a32757f33bc74369bf3b6 Mon Sep 17 00:00:00 2001 From: sprang Date: Wed, 28 Dec 2016 05:58:07 -0800 Subject: [PATCH] Add frame rate throttling to vp8 screenshare_layers. Drop frames if incoming frame rate is higher than the configured max framerate. BUG=webrtc:6897 Review-Url: https://codereview.webrtc.org/2578993002 Cr-Commit-Position: refs/heads/master@{#15819} --- .../codecs/vp8/screenshare_layers.cc | 47 +++++++++++++++---- .../codecs/vp8/screenshare_layers.h | 9 +++- .../codecs/vp8/screenshare_layers_unittest.cc | 42 +++++++++++++++++ 3 files changed, 87 insertions(+), 11 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index e81ab3df37..e99deed68b 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -76,7 +76,7 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, min_qp_(-1), max_qp_(-1), max_debt_bytes_(0), - framerate_(-1), + encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale. bitrate_updated_(false) { RTC_CHECK_GT(num_temporal_layers, 0); RTC_CHECK_LE(num_temporal_layers, 2); @@ -97,8 +97,15 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { return 0; } + const int64_t now_ms = clock_->TimeInMilliseconds(); + if (target_framerate_.value_or(0) > 0 && + encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) { + // Max framerate exceeded, drop frame. + return -1; + } + if (stats_.first_frame_time_ms_ == -1) - stats_.first_frame_time_ms_ = clock_->TimeInMilliseconds(); + stats_.first_frame_time_ms_ = now_ms; int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); int flags = 0; @@ -148,7 +155,8 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { int64_t ts_diff; if (last_timestamp_ == -1) { - ts_diff = kOneSecond90Khz / (framerate_ <= 0 ? 5 : framerate_); + ts_diff = + kOneSecond90Khz / incoming_framerate_.value_or(*target_framerate_); } else { ts_diff = unwrapped_timestamp - last_timestamp_; } @@ -162,13 +170,28 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, int max_bitrate_kbps, int framerate) { - bitrate_updated_ = - bitrate_kbps != static_cast(layers_[0].target_rate_kbps_) || - max_bitrate_kbps != static_cast(layers_[1].target_rate_kbps_) || - framerate != framerate_; + RTC_DCHECK_GT(framerate, 0); + if (!target_framerate_) { + // First OnRatesUpdated() is called during construction, with the configured + // targets as parameters. + target_framerate_.emplace(framerate); + incoming_framerate_ = target_framerate_; + bitrate_updated_ = true; + } else { + bitrate_updated_ = + bitrate_kbps != static_cast(layers_[0].target_rate_kbps_) || + max_bitrate_kbps != static_cast(layers_[1].target_rate_kbps_) || + (incoming_framerate_ && + framerate != static_cast(*incoming_framerate_)); + if (framerate < 0) { + incoming_framerate_.reset(); + } else { + incoming_framerate_.emplace(framerate); + } + } + layers_[0].target_rate_kbps_ = bitrate_kbps; layers_[1].target_rate_kbps_ = max_bitrate_kbps; - framerate_ = framerate; std::vector allocation; allocation.push_back(bitrate_kbps); @@ -180,6 +203,9 @@ std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, void ScreenshareLayers::FrameEncoded(unsigned int size, uint32_t timestamp, int qp) { + if (size > 0) + encode_framerate_.Update(1, clock_->TimeInMilliseconds()); + if (number_of_temporal_layers_ == 1) return; @@ -301,8 +327,9 @@ bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100); } - if (framerate_ > 0) { - int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate_); + if (incoming_framerate_) { + int avg_frame_size = + (target_bitrate_kbps * 1000) / (8 * *incoming_framerate_); max_debt_bytes_ = 4 * avg_frame_size; } diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index d10d75ad9f..c7180607e9 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -11,6 +11,7 @@ #include +#include "webrtc/base/rate_statistics.h" #include "webrtc/base/timeutils.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/utility/frame_dropper.h" @@ -74,7 +75,13 @@ class ScreenshareLayers : public TemporalLayers { int min_qp_; int max_qp_; uint32_t max_debt_bytes_; - int framerate_; + + // Configured max framerate. + rtc::Optional target_framerate_; + // Incoming framerate from capturer. + rtc::Optional incoming_framerate_; + // Tracks what framerate we actually encode, and drops frames on overshoot. + RateStatistics encode_framerate_; bool bitrate_updated_; static const int kMaxNumTemporalLayers = 2; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index b41621c83f..cbbf1e33ad 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -577,4 +577,46 @@ TEST_F(ScreenshareLayerTest, AllowsUpdateConfigBeforeSetRates) { EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); } +TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { + ConfigureBitrates(); + + int64_t kTestSpanMs = 2000; + int64_t kFrameIntervalsMs = 1000 / kFrameRate; + + uint32_t timestamp = 1234; + int num_input_frames = 0; + int num_discarded_frames = 0; + + // Send at regular rate - no drops expected. + for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs) { + if (layers_->EncodeFlags(timestamp) == -1) { + ++num_discarded_frames; + } else { + size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; + layers_->FrameEncoded(frame_size_bytes, timestamp, kDefaultQp); + } + timestamp += kFrameIntervalsMs * 90; + clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs); + ++num_input_frames; + } + EXPECT_EQ(0, num_discarded_frames); + + // Send at twice the configured rate - drop every other frame. + num_input_frames = 0; + num_discarded_frames = 0; + for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs / 2) { + if (layers_->EncodeFlags(timestamp) == -1) { + ++num_discarded_frames; + } else { + size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; + layers_->FrameEncoded(frame_size_bytes, timestamp, kDefaultQp); + } + timestamp += kFrameIntervalsMs * 90 / 2; + clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs / 2); + ++num_input_frames; + } + + // Allow for some rounding errors in the measurements. + EXPECT_NEAR(num_discarded_frames, num_input_frames / 2, 2); +} } // namespace webrtc