From afe1f74c04d908f8bd8ad5b15c6adf6ce6250969 Mon Sep 17 00:00:00 2001 From: sprang Date: Tue, 12 Apr 2016 02:45:13 -0700 Subject: [PATCH] Make sure temporal layered screenshare frames are sent in at least 2s. If a very large frame is sent (high res slide change) when the available send bitrate is very low, the it might take many seconds before any new frames are emitted as the accrued debt will take time to pay off. Add a bailout, so that if a frame hasn't been sent for 2 seconds, cancel the debt immediately, even if the target bitrate is then exceeded. BUG=webrtc:5750 Review URL: https://codereview.webrtc.org/1869003002 Cr-Commit-Position: refs/heads/master@{#12328} --- .../codecs/vp8/screenshare_layers.cc | 17 +++++-- .../codecs/vp8/screenshare_layers.h | 2 + .../codecs/vp8/screenshare_layers_unittest.cc | 49 +++++++++++++++---- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index 2b7f7195d5..e821497440 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -47,6 +47,10 @@ const int ScreenshareLayers::kTl1SyncFlags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; +// Always emit a frame with certain interval, even if bitrate targets have +// been exceeded. +const int ScreenshareLayers::kMaxFrameIntervalMs = 2000; + ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, uint8_t initial_tl0_pic_idx, Clock* clock) @@ -57,6 +61,7 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, active_layer_(-1), last_timestamp_(-1), last_sync_timestamp_(-1), + last_emitted_tl0_timestamp_(-1), min_qp_(-1), max_qp_(-1), max_debt_bytes_(0), @@ -85,9 +90,15 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); int flags = 0; - if (active_layer_ == -1 || layers_[active_layer_].state != TemporalLayer::State::kDropped) { + if (last_emitted_tl0_timestamp_ != -1 && + (unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 > + kMaxFrameIntervalMs) { + // Too long time has passed since the last frame was emitted, cancel + // enough debt to allow a single frame. + layers_[0].debt_bytes_ = max_debt_bytes_ - 1; + } if (layers_[0].debt_bytes_ > max_debt_bytes_) { // Must drop TL0, encode TL1 instead. if (layers_[1].debt_bytes_ > max_debt_bytes_) { @@ -104,6 +115,7 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { switch (active_layer_) { case 0: flags = kTl0Flags; + last_emitted_tl0_timestamp_ = unwrapped_timestamp; break; case 1: if (TimeToSync(unwrapped_timestamp)) { @@ -122,14 +134,13 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { RTC_NOTREACHED(); } - // Make sure both frame droppers leak out bits. int64_t ts_diff; if (last_timestamp_ == -1) { ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_); } else { ts_diff = unwrapped_timestamp - last_timestamp_; } - + // Make sure both frame droppers leak out bits. layers_[0].UpdateDebt(ts_diff / 90); layers_[1].UpdateDebt(ts_diff / 90); last_timestamp_ = timestamp; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index e11f3d27ef..8392bc4f35 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -30,6 +30,7 @@ class ScreenshareLayers : public TemporalLayers { static const int kTl0Flags; static const int kTl1Flags; static const int kTl1SyncFlags; + static const int kMaxFrameIntervalMs; ScreenshareLayers(int num_temporal_layers, uint8_t initial_tl0_pic_idx, @@ -69,6 +70,7 @@ class ScreenshareLayers : public TemporalLayers { int active_layer_; int64_t last_timestamp_; int64_t last_sync_timestamp_; + int64_t last_emitted_tl0_timestamp_; rtc::TimestampWrapAroundHandler time_wrap_handler_; int min_qp_; int max_qp_; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index 7be4eb1bb2..667f6b0922 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -108,6 +108,14 @@ class ScreenshareLayerTest : public ::testing::Test { return 0; } + vpx_codec_enc_cfg_t GetConfig() { + vpx_codec_enc_cfg_t cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.rc_min_quantizer = 2; + cfg.rc_max_quantizer = kDefaultQp; + return cfg; + } + int min_qp_; int max_qp_; int frame_size_; @@ -359,13 +367,12 @@ TEST_F(ScreenshareLayerTest, TooHighBitrate) { } } - EXPECT_EQ(5, tl0_frames); - EXPECT_EQ(45, tl1_frames); + EXPECT_EQ(50, tl0_frames + tl1_frames); EXPECT_EQ(50, dropped_frames); } TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { - vpx_codec_enc_cfg_t cfg; + vpx_codec_enc_cfg_t cfg = GetConfig(); layers_->ConfigureBitrates(100, 1000, 5, &cfg); EXPECT_EQ(static_cast( @@ -374,7 +381,7 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { } TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { - vpx_codec_enc_cfg_t cfg; + vpx_codec_enc_cfg_t cfg = GetConfig(); layers_->ConfigureBitrates(100, 450, 5, &cfg); EXPECT_EQ(static_cast( @@ -383,7 +390,7 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { } TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { - vpx_codec_enc_cfg_t cfg; + vpx_codec_enc_cfg_t cfg = GetConfig(); layers_->ConfigureBitrates(100, 100, 5, &cfg); EXPECT_EQ(100U, cfg.rc_target_bitrate); @@ -392,8 +399,7 @@ TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { TEST_F(ScreenshareLayerTest, EncoderDrop) { ConfigureBitrates(); CodecSpecificInfoVP8 vp8_info; - vpx_codec_enc_cfg_t cfg; - cfg.rc_max_quantizer = kDefaultQp; + vpx_codec_enc_cfg_t cfg = GetConfig(); uint32_t timestamp = RunGracePeriod(); timestamp = SkipUntilTl(0, timestamp); @@ -441,10 +447,35 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); } +TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { + const int kLowBitrateKbps = 50; + const int kLargeFrameSizeBytes = 100000; + const uint32_t kStartTimestamp = 1234; + + vpx_codec_enc_cfg_t cfg = GetConfig(); + layers_->ConfigureBitrates(kLowBitrateKbps, kLowBitrateKbps, 5, &cfg); + + EXPECT_EQ(ScreenshareLayers::kTl0Flags, + layers_->EncodeFlags(kStartTimestamp)); + layers_->FrameEncoded(kLargeFrameSizeBytes, kStartTimestamp, kDefaultQp); + + const uint32_t kTwoSecondsLater = + kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90); + + // Sanity check, repayment time should exceed kMaxFrameIntervalMs. + ASSERT_GT(kStartTimestamp + 90 * (kLargeFrameSizeBytes * 8) / kLowBitrateKbps, + kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90)); + + EXPECT_EQ(-1, layers_->EncodeFlags(kTwoSecondsLater)); + // More than two seconds has passed since last frame, one should be emitted + // even if bitrate target is then exceeded. + EXPECT_EQ(ScreenshareLayers::kTl0Flags, + layers_->EncodeFlags(kTwoSecondsLater + 90)); +} + TEST_F(ScreenshareLayerTest, UpdatesHistograms) { ConfigureBitrates(); - vpx_codec_enc_cfg_t cfg; - cfg.rc_max_quantizer = kDefaultQp; + vpx_codec_enc_cfg_t cfg = GetConfig(); bool trigger_drop = false; bool dropped_frame = false; bool overshoot = false;