From 51f083ccb2bb41acab552fefb5118598f998c23d Mon Sep 17 00:00:00 2001 From: pbos Date: Thu, 4 May 2017 06:39:04 -0700 Subject: [PATCH] Remove layer_sync from TL frame config. This is instead derived from pattern_idx inside the frame config. This also removes active_layer_ use from ScreenshareLayers::PopulateCodecSpecific and instead ties the layer to TemporalLayers::FrameConfig. BUG=chromium:702017, webrtc:7349 R=sprang@webrtc.org Review-Url: https://codereview.webrtc.org/2860063002 Cr-Commit-Position: refs/heads/master@{#18017} --- .../codecs/vp8/default_temporal_layers.cc | 78 +++++++++++-------- .../codecs/vp8/default_temporal_layers.h | 1 + .../codecs/vp8/screenshare_layers.cc | 49 ++++++++---- .../codecs/vp8/screenshare_layers.h | 2 + .../video_coding/codecs/vp8/temporal_layers.h | 9 +-- 5 files changed, 85 insertions(+), 54 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc index b8359ade39..3573aa8f1c 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -30,22 +30,17 @@ TemporalLayers::FrameConfig::FrameConfig() {} TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last, TemporalLayers::BufferFlags golden, TemporalLayers::BufferFlags arf) - : FrameConfig(last, golden, arf, false, false) {} + : FrameConfig(last, golden, arf, false) {} TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last, TemporalLayers::BufferFlags golden, TemporalLayers::BufferFlags arf, - int extra_flags) - : FrameConfig(last, - golden, - arf, - (extra_flags & kLayerSync) != 0, - (extra_flags & kFreezeEntropy) != 0) {} + FreezeEntropy) + : FrameConfig(last, golden, arf, true) {} TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last, TemporalLayers::BufferFlags golden, TemporalLayers::BufferFlags arf, - bool layer_sync, bool freeze_entropy) : drop_frame(last == TemporalLayers::kNone && golden == TemporalLayers::kNone && @@ -53,7 +48,6 @@ TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last, last_buffer_flags(last), golden_buffer_flags(golden), arf_buffer_flags(arf), - layer_sync(layer_sync), freeze_entropy(freeze_entropy) {} namespace { @@ -90,6 +84,25 @@ std::vector GetTemporalIds(size_t num_layers) { return {0}; } +std::vector GetTemporalLayerSync(size_t num_layers) { + switch (num_layers) { + case 1: + return {false}; + case 2: + return {false, true, false, false, false, false, false, false}; + case 3: + return {false, true, true, false, false, false, false, false}; + case 4: + return {false, true, true, true, true, true, false, true, + false, true, false, true, false, true, false, true}; + default: + RTC_NOTREACHED(); + break; + } + RTC_NOTREACHED(); + return {false}; +} + std::vector GetTemporalPattern(size_t num_layers) { // For indexing in the patterns described below (which temporal layers they // belong to), see the diagram above. @@ -118,9 +131,9 @@ std::vector GetTemporalPattern(size_t num_layers) { return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, TemporalLayers::kUpdate, TemporalLayers::kReference), - TemporalLayers::FrameConfig( - TemporalLayers::kReference, TemporalLayers::kUpdate, - TemporalLayers::kReference, kLayerSync), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kUpdate, + TemporalLayers::kReference), TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, TemporalLayers::kNone, TemporalLayers::kReference), @@ -138,7 +151,7 @@ std::vector GetTemporalPattern(size_t num_layers) { TemporalLayers::kReference), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kFreezeEntropy)}; + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy)}; case 3: // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. @@ -150,25 +163,25 @@ std::vector GetTemporalPattern(size_t num_layers) { TemporalLayers::kReference), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kNone, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy), - TemporalLayers::FrameConfig( - TemporalLayers::kReference, TemporalLayers::kUpdate, - TemporalLayers::kReference, kLayerSync), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kUpdate, + TemporalLayers::kReference), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, TemporalLayers::kNone, TemporalLayers::kReference), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReference, TemporalLayers::kReferenceAndUpdate, TemporalLayers::kReference), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kFreezeEntropy)}; + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy)}; case 4: // TL0 references and updates only the 'last' buffer. // TL1 references 'last' and updates and references 'golden'. @@ -179,49 +192,49 @@ std::vector GetTemporalPattern(size_t num_layers) { TemporalLayers::kNone), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReference, TemporalLayers::kNone, - TemporalLayers::kUpdate, kLayerSync), + TemporalLayers::kUpdate), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReference, TemporalLayers::kUpdate, - TemporalLayers::kNone, kLayerSync), + TemporalLayers::kNone), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReference, TemporalLayers::kReference, TemporalLayers::kReferenceAndUpdate), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, TemporalLayers::kNone, TemporalLayers::kNone), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReference, TemporalLayers::kReference, TemporalLayers::kReferenceAndUpdate), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReference, TemporalLayers::kReferenceAndUpdate, TemporalLayers::kNone), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy), TemporalLayers::FrameConfig(TemporalLayers::kReference, TemporalLayers::kReference, TemporalLayers::kReferenceAndUpdate), TemporalLayers::FrameConfig( TemporalLayers::kReference, TemporalLayers::kReference, - TemporalLayers::kReference, kLayerSync | kFreezeEntropy)}; + TemporalLayers::kReference, TemporalLayers::kFreezeEntropy)}; default: RTC_NOTREACHED(); break; @@ -237,11 +250,13 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers, uint8_t initial_tl0_pic_idx) : num_layers_(std::max(1, number_of_temporal_layers)), temporal_ids_(GetTemporalIds(num_layers_)), + temporal_layer_sync_(GetTemporalLayerSync(num_layers_)), temporal_pattern_(GetTemporalPattern(num_layers_)), tl0_pic_idx_(initial_tl0_pic_idx), pattern_idx_(255), timestamp_(0), last_base_layer_sync_(false) { + RTC_DCHECK_EQ(temporal_pattern_.size(), temporal_layer_sync_.size()); RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); RTC_CHECK_GE(number_of_temporal_layers, 0); RTC_CHECK_LE(number_of_temporal_layers, 4); @@ -339,7 +354,8 @@ void DefaultTemporalLayers::PopulateCodecSpecific( } else { vp8_info->temporalIdx = GetTemporalLayerId(tl_config); - vp8_info->layerSync = tl_config.layer_sync; + vp8_info->layerSync = temporal_layer_sync_[tl_config.pattern_idx % + temporal_layer_sync_.size()]; } if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) { // Regardless of pattern the frame after a base layer sync will always diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h index fd2c205549..4681fe9dc8 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -53,6 +53,7 @@ class DefaultTemporalLayers : public TemporalLayers { private: const size_t num_layers_; const std::vector temporal_ids_; + const std::vector temporal_layer_sync_; const std::vector temporal_pattern_; uint8_t tl0_pic_idx_; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index 6bfd03fb50..d535c617cd 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -93,8 +93,10 @@ TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( if (number_of_temporal_layers_ <= 1) { // No flags needed for 1 layer screenshare. // TODO(pbos): Consider updating only last, and not all buffers. - return TemporalLayers::FrameConfig(kReferenceAndUpdate, kReferenceAndUpdate, - kReferenceAndUpdate); + TemporalLayers::FrameConfig tl_config( + kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate); + tl_config.pattern_idx = static_cast(kTl1); + return tl_config; } const int64_t now_ms = clock_->TimeInMilliseconds(); @@ -108,8 +110,7 @@ TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( stats_.first_frame_time_ms_ = now_ms; int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); - enum TemporalLayerState { kDrop, kTl0, kTl1, kTl1Sync }; - enum TemporalLayerState layer_state = kDrop; + TemporalLayerState layer_state = kDrop; if (active_layer_ == -1 || layers_[active_layer_].state != TemporalLayer::State::kDropped) { if (last_emitted_tl0_timestamp_ != -1 && @@ -163,27 +164,33 @@ TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( layers_[0].UpdateDebt(ts_diff / 90); layers_[1].UpdateDebt(ts_diff / 90); last_timestamp_ = timestamp; + TemporalLayers::FrameConfig tl_config; // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all // layers. switch (layer_state) { case kDrop: - return TemporalLayers::FrameConfig(kNone, kNone, kNone); + tl_config = TemporalLayers::FrameConfig(kNone, kNone, kNone); + break; case kTl0: // TL0 only references and updates 'last'. - return TemporalLayers::FrameConfig(kReferenceAndUpdate, kNone, kNone); + tl_config = + TemporalLayers::FrameConfig(kReferenceAndUpdate, kNone, kNone); + break; case kTl1: // TL1 references both 'last' and 'golden' but only updates 'golden'. - return TemporalLayers::FrameConfig(kReference, kReferenceAndUpdate, - kNone); + tl_config = + TemporalLayers::FrameConfig(kReference, kReferenceAndUpdate, kNone); + break; case kTl1Sync: // Predict from only TL0 to allow participants to switch to the high // bitrate stream. Updates 'golden' so that TL1 can continue to refer to // and update 'golden' from this point on. - return TemporalLayers::FrameConfig(kReference, kUpdate, kNone, - kLayerSync); + tl_config = TemporalLayers::FrameConfig(kReference, kUpdate, kNone); + break; } - RTC_NOTREACHED(); - return TemporalLayers::FrameConfig(kNone, kNone, kNone); + + tl_config.pattern_idx = static_cast(layer_state); + return tl_config; } std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, @@ -265,10 +272,20 @@ void ScreenshareLayers::PopulateCodecSpecific( vp8_info->layerSync = false; vp8_info->tl0PicIdx = kNoTl0PicIdx; } else { - // TODO(pbos): Add active_layer_ to TemporalLayers::FrameConfig (as - // pattern_idx) and make function const. - RTC_DCHECK_NE(-1, active_layer_); - vp8_info->temporalIdx = active_layer_; + TemporalLayerState layer_state = + static_cast(tl_config.pattern_idx); + switch (layer_state) { + case kDrop: + RTC_NOTREACHED(); + break; + case kTl0: + vp8_info->temporalIdx = 0; + break; + case kTl1: + case kTl1Sync: + vp8_info->temporalIdx = 1; + break; + } if (frame_is_keyframe) { vp8_info->temporalIdx = 0; last_sync_timestamp_ = unwrapped_timestamp; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index cfa29116e6..a13f3c9aa7 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -60,6 +60,8 @@ class ScreenshareLayers : public TemporalLayers { uint8_t Tl0PicIdx() const override; private: + enum TemporalLayerState { kDrop, kTl0, kTl1, kTl1Sync }; + bool TimeToSync(int64_t timestamp) const; uint32_t GetCodecTargetBitrateKbps() const; diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h index 471daa636e..f5d2aabcbe 100644 --- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -24,11 +24,6 @@ namespace webrtc { struct CodecSpecificInfoVP8; -// TODO(pbos): Remove along with layer_sync and freeze_entropy, they are -// derivable from picture_idx and should be exposed with -// TL::IsLayerSync(TL::FrameConfig). -enum TemporalFlags { kLayerSync = 1, kFreezeEntropy = 2 }; - class TemporalLayers { public: enum BufferFlags { @@ -37,6 +32,7 @@ class TemporalLayers { kUpdate = 2, kReferenceAndUpdate = kReference | kUpdate, }; + enum FreezeEntropy { kFreezeEntropy }; struct FrameConfig { FrameConfig(); @@ -45,7 +41,7 @@ class TemporalLayers { FrameConfig(BufferFlags last, BufferFlags golden, BufferFlags arf, - int extra_flags); + FreezeEntropy); bool drop_frame; BufferFlags last_buffer_flags; @@ -64,7 +60,6 @@ class TemporalLayers { FrameConfig(BufferFlags last, BufferFlags golden, BufferFlags arf, - bool layer_sync, bool freeze_entropy); };