diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 325e8a5941..b8359ade39 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -25,27 +25,31 @@ namespace webrtc { -TemporalReferences::TemporalReferences(TemporalBufferFlags last, - TemporalBufferFlags golden, - TemporalBufferFlags arf) - : TemporalReferences(last, golden, arf, false, false) {} +TemporalLayers::FrameConfig::FrameConfig() {} -TemporalReferences::TemporalReferences(TemporalBufferFlags last, - TemporalBufferFlags golden, - TemporalBufferFlags arf, - int extra_flags) - : TemporalReferences(last, - golden, - arf, - (extra_flags & kLayerSync) != 0, - (extra_flags & kFreezeEntropy) != 0) {} +TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last, + TemporalLayers::BufferFlags golden, + TemporalLayers::BufferFlags arf) + : FrameConfig(last, golden, arf, false, false) {} -TemporalReferences::TemporalReferences(TemporalBufferFlags last, - TemporalBufferFlags golden, - TemporalBufferFlags arf, - bool layer_sync, - bool freeze_entropy) - : drop_frame(last == kNone && golden == kNone && arf == kNone), +TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last, + TemporalLayers::BufferFlags golden, + TemporalLayers::BufferFlags arf, + int extra_flags) + : FrameConfig(last, + golden, + arf, + (extra_flags & kLayerSync) != 0, + (extra_flags & kFreezeEntropy) != 0) {} + +TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last, + TemporalLayers::BufferFlags golden, + TemporalLayers::BufferFlags arf, + bool layer_sync, + bool freeze_entropy) + : drop_frame(last == TemporalLayers::kNone && + golden == TemporalLayers::kNone && + arf == TemporalLayers::kNone), last_buffer_flags(last), golden_buffer_flags(golden), arf_buffer_flags(arf), @@ -86,8 +90,7 @@ std::vector GetTemporalIds(size_t num_layers) { return {0}; } -std::vector GetTemporalPattern( - size_t num_layers) { +std::vector GetTemporalPattern(size_t num_layers) { // For indexing in the patterns described below (which temporal layers they // belong to), see the diagram above. // Layer sync is done similarly for all patterns (except single stream) and @@ -104,74 +107,128 @@ std::vector GetTemporalPattern( switch (num_layers) { case 1: // All frames reference all buffers and the 'last' buffer is updated. - return {TemporalReferences(kReferenceAndUpdate, kReference, kReference)}; + return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kReference, + TemporalLayers::kReference)}; case 2: // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. // TL0 also references and updates the 'last' buffer. // TL1 also references 'last' and references and updates 'golden'. - return {TemporalReferences(kReferenceAndUpdate, kUpdate, kReference), - TemporalReferences(kReference, kUpdate, kReference, kLayerSync), - TemporalReferences(kReferenceAndUpdate, kNone, kReference), - TemporalReferences(kReference, kReferenceAndUpdate, kReference), - TemporalReferences(kReferenceAndUpdate, kNone, kReference), - TemporalReferences(kReference, kReferenceAndUpdate, kReference), - TemporalReferences(kReferenceAndUpdate, kNone, kReference), - TemporalReferences(kReference, kReference, kReference, - kFreezeEntropy)}; + return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kUpdate, + TemporalLayers::kReference), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kUpdate, + TemporalLayers::kReference, kLayerSync), + TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kNone, + TemporalLayers::kReference), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kReference), + TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kNone, + TemporalLayers::kReference), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kReference), + TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kNone, + TemporalLayers::kReference), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kFreezeEntropy)}; case 3: // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. // TL0 also references and updates the 'last' buffer. // TL1 also references 'last' and references and updates 'golden'. // TL2 references both 'last' and 'golden' but updates no buffer. - return {TemporalReferences(kReferenceAndUpdate, kUpdate, kReference), - TemporalReferences(kReference, kNone, kReference, - kLayerSync | kFreezeEntropy), - TemporalReferences(kReference, kUpdate, kReference, kLayerSync), - TemporalReferences(kReference, kReference, kReference, - kFreezeEntropy), - TemporalReferences(kReferenceAndUpdate, kNone, kReference), - TemporalReferences(kReference, kReference, kReference, - kFreezeEntropy), - TemporalReferences(kReference, kReferenceAndUpdate, kReference), - TemporalReferences(kReference, kReference, kReference, - kFreezeEntropy)}; + return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kUpdate, + TemporalLayers::kReference), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kNone, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kUpdate, + TemporalLayers::kReference, kLayerSync), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kNone, + TemporalLayers::kReference), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kReference), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kFreezeEntropy)}; case 4: // TL0 references and updates only the 'last' buffer. // TL1 references 'last' and updates and references 'golden'. // TL2 references 'last' and 'golden', and references and updates 'arf'. // TL3 references all buffers but update none of them. - return {TemporalReferences(kReferenceAndUpdate, kNone, kNone), - TemporalReferences(kReference, kReference, kReference, - kLayerSync | kFreezeEntropy), - TemporalReferences(kReference, kNone, kUpdate, kLayerSync), - TemporalReferences(kReference, kReference, kReference, - kLayerSync | kFreezeEntropy), - TemporalReferences(kReference, kUpdate, kNone, kLayerSync), - TemporalReferences(kReference, kReference, kReference, - kLayerSync | kFreezeEntropy), - TemporalReferences(kReference, kReference, kReferenceAndUpdate), - TemporalReferences(kReference, kReference, kReference, - kLayerSync | kFreezeEntropy), - TemporalReferences(kReferenceAndUpdate, kNone, kNone), - TemporalReferences(kReference, kReference, kReference, - kLayerSync | kFreezeEntropy), - TemporalReferences(kReference, kReference, kReferenceAndUpdate), - TemporalReferences(kReference, kReference, kReference, - kLayerSync | kFreezeEntropy), - TemporalReferences(kReference, kReferenceAndUpdate, kNone), - TemporalReferences(kReference, kReference, kReference, - kLayerSync | kFreezeEntropy), - TemporalReferences(kReference, kReference, kReferenceAndUpdate), - TemporalReferences(kReference, kReference, kReference, - kLayerSync | kFreezeEntropy)}; + return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kNone, + TemporalLayers::kNone), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kNone, + TemporalLayers::kUpdate, kLayerSync), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kUpdate, + TemporalLayers::kNone, kLayerSync), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kReference, + TemporalLayers::kReferenceAndUpdate), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kNone, + TemporalLayers::kNone), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kReference, + TemporalLayers::kReferenceAndUpdate), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kReferenceAndUpdate, + TemporalLayers::kNone), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy), + TemporalLayers::FrameConfig(TemporalLayers::kReference, + TemporalLayers::kReference, + TemporalLayers::kReferenceAndUpdate), + TemporalLayers::FrameConfig( + TemporalLayers::kReference, TemporalLayers::kReference, + TemporalLayers::kReference, kLayerSync | kFreezeEntropy)}; default: RTC_NOTREACHED(); break; } RTC_NOTREACHED(); - return {TemporalReferences(kNone, kNone, kNone)}; + return {TemporalLayers::FrameConfig( + TemporalLayers::kNone, TemporalLayers::kNone, TemporalLayers::kNone)}; } } // namespace @@ -188,10 +245,16 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers, RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); RTC_CHECK_GE(number_of_temporal_layers, 0); RTC_CHECK_LE(number_of_temporal_layers, 4); + // pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if + // temporal_ids_ are ever longer. If this is no longer correct it needs to + // wrap at max(temporal_ids_.size(), temporal_pattern_.size()). + RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size()); } -int DefaultTemporalLayers::CurrentLayerId() const { - return temporal_ids_[pattern_idx_ % temporal_ids_.size()]; +int DefaultTemporalLayers::GetTemporalLayerId( + const TemporalLayers::FrameConfig& tl_config) const { + RTC_DCHECK(!tl_config.drop_frame); + return temporal_ids_[tl_config.pattern_idx % temporal_ids_.size()]; } uint8_t DefaultTemporalLayers::Tl0PicIdx() const { @@ -248,16 +311,19 @@ bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { return true; } -// TODO(pbos): Name method so that it's obvious that it updates state. -TemporalReferences DefaultTemporalLayers::UpdateLayerConfig( +TemporalLayers::FrameConfig DefaultTemporalLayers::UpdateLayerConfig( uint32_t timestamp) { RTC_DCHECK_GT(num_layers_, 0); RTC_DCHECK_LT(0, temporal_pattern_.size()); - return temporal_pattern_[++pattern_idx_ % temporal_pattern_.size()]; + pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size(); + TemporalLayers::FrameConfig tl_config = temporal_pattern_[pattern_idx_]; + tl_config.pattern_idx = pattern_idx_; + return tl_config; } void DefaultTemporalLayers::PopulateCodecSpecific( bool frame_is_keyframe, + const TemporalLayers::FrameConfig& tl_config, CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) { RTC_DCHECK_GT(num_layers_, 0); @@ -271,11 +337,9 @@ void DefaultTemporalLayers::PopulateCodecSpecific( vp8_info->temporalIdx = 0; vp8_info->layerSync = true; } else { - vp8_info->temporalIdx = CurrentLayerId(); - TemporalReferences temporal_reference = - temporal_pattern_[pattern_idx_ % temporal_pattern_.size()]; + vp8_info->temporalIdx = GetTemporalLayerId(tl_config); - vp8_info->layerSync = temporal_reference.layer_sync; + vp8_info->layerSync = tl_config.layer_sync; } if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) { // Regardless of pattern the frame after a base layer sync will always diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h index be2d892816..fd2c205549 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -28,7 +28,7 @@ class DefaultTemporalLayers : public TemporalLayers { // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - TemporalReferences UpdateLayerConfig(uint32_t timestamp) override; + TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) override; // Update state based on new bitrate target and incoming framerate. // Returns the bitrate allocation for the active temporal layers. @@ -39,19 +39,21 @@ class DefaultTemporalLayers : public TemporalLayers { bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; void PopulateCodecSpecific(bool frame_is_keyframe, + const TemporalLayers::FrameConfig& tl_config, CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) override; void FrameEncoded(unsigned int size, int qp) override {} - int CurrentLayerId() const override; + int GetTemporalLayerId( + const TemporalLayers::FrameConfig& references) const override; uint8_t Tl0PicIdx() const override; private: const size_t num_layers_; const std::vector temporal_ids_; - const std::vector temporal_pattern_; + const std::vector temporal_pattern_; uint8_t tl0_pic_idx_; uint8_t pattern_idx_; diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index f7502447b8..edab34c16d 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -84,9 +84,9 @@ TEST(TemporalLayersTest, 2Layers) { uint32_t timestamp = 0; for (int i = 0; i < 16; ++i) { - TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)); - tl.PopulateCodecSpecific(false, &vp8_info, 0); + tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); timestamp += 3000; @@ -127,9 +127,9 @@ TEST(TemporalLayersTest, 3Layers) { unsigned int timestamp = 0; for (int i = 0; i < 16; ++i) { - TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)); - tl.PopulateCodecSpecific(false, &vp8_info, 0); + tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); timestamp += 3000; @@ -169,9 +169,9 @@ TEST(TemporalLayersTest, 4Layers) { uint32_t timestamp = 0; for (int i = 0; i < 16; ++i) { - TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)); - tl.PopulateCodecSpecific(false, &vp8_info, 0); + tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); timestamp += 3000; @@ -199,16 +199,16 @@ TEST(TemporalLayersTest, KeyFrame) { uint32_t timestamp = 0; for (int i = 0; i < 7; ++i) { - TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)); - tl.PopulateCodecSpecific(true, &vp8_info, 0); + tl.PopulateCodecSpecific(true, tl_config, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(true, vp8_info.layerSync); timestamp += 3000; } - TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[7], VP8EncoderImpl::EncodeFlags(tl_config)); - tl.PopulateCodecSpecific(false, &vp8_info, 0); + tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[7], vp8_info.temporalIdx); EXPECT_EQ(true, vp8_info.layerSync); } diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index d217bdbed7..6bfd03fb50 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -78,7 +78,8 @@ ScreenshareLayers::~ScreenshareLayers() { UpdateHistograms(); } -int ScreenshareLayers::CurrentLayerId() const { +int ScreenshareLayers::GetTemporalLayerId( + const TemporalLayers::FrameConfig& tl_config) const { // Codec does not use temporal layers for screenshare. return 0; } @@ -87,19 +88,20 @@ uint8_t ScreenshareLayers::Tl0PicIdx() const { return tl0_pic_idx_; } -TemporalReferences ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) { +TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( + uint32_t timestamp) { if (number_of_temporal_layers_ <= 1) { // No flags needed for 1 layer screenshare. // TODO(pbos): Consider updating only last, and not all buffers. - return TemporalReferences(kReferenceAndUpdate, kReferenceAndUpdate, - kReferenceAndUpdate); + return TemporalLayers::FrameConfig(kReferenceAndUpdate, kReferenceAndUpdate, + kReferenceAndUpdate); } const int64_t now_ms = clock_->TimeInMilliseconds(); if (target_framerate_.value_or(0) > 0 && encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) { // Max framerate exceeded, drop frame. - return TemporalReferences(kNone, kNone, kNone); + return TemporalLayers::FrameConfig(kNone, kNone, kNone); } if (stats_.first_frame_time_ms_ == -1) @@ -165,21 +167,23 @@ TemporalReferences ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) { // layers. switch (layer_state) { case kDrop: - return TemporalReferences(kNone, kNone, kNone); + return TemporalLayers::FrameConfig(kNone, kNone, kNone); case kTl0: // TL0 only references and updates 'last'. - return TemporalReferences(kReferenceAndUpdate, kNone, kNone); + return TemporalLayers::FrameConfig(kReferenceAndUpdate, kNone, kNone); case kTl1: // TL1 references both 'last' and 'golden' but only updates 'golden'. - return TemporalReferences(kReference, kReferenceAndUpdate, kNone); + return TemporalLayers::FrameConfig(kReference, kReferenceAndUpdate, + kNone); case kTl1Sync: // Predict from only TL0 to allow participants to switch to the high // bitrate stream. Updates 'golden' so that TL1 can continue to refer to // and update 'golden' from this point on. - return TemporalReferences(kReference, kUpdate, kNone, kLayerSync); + return TemporalLayers::FrameConfig(kReference, kUpdate, kNone, + kLayerSync); } RTC_NOTREACHED(); - return TemporalReferences(kNone, kNone, kNone); + return TemporalLayers::FrameConfig(kNone, kNone, kNone); } std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, @@ -250,15 +254,19 @@ void ScreenshareLayers::FrameEncoded(unsigned int size, int qp) { } } -void ScreenshareLayers::PopulateCodecSpecific(bool frame_is_keyframe, - CodecSpecificInfoVP8* vp8_info, - uint32_t timestamp) { +void ScreenshareLayers::PopulateCodecSpecific( + bool frame_is_keyframe, + const TemporalLayers::FrameConfig& tl_config, + CodecSpecificInfoVP8* vp8_info, + uint32_t timestamp) { int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); if (number_of_temporal_layers_ == 1) { vp8_info->temporalIdx = kNoTemporalIdx; vp8_info->layerSync = false; vp8_info->tl0PicIdx = kNoTl0PicIdx; } else { + // TODO(pbos): Add active_layer_ to TemporalLayers::FrameConfig (as + // pattern_idx) and make function const. RTC_DCHECK_NE(-1, active_layer_); vp8_info->temporalIdx = active_layer_; if (frame_is_keyframe) { diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index cfb311d924..cfa29116e6 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -35,7 +35,7 @@ class ScreenshareLayers : public TemporalLayers { // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - TemporalReferences UpdateLayerConfig(uint32_t timestamp) override; + TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) override; // Update state based on new bitrate target and incoming framerate. // Returns the bitrate allocation for the active temporal layers. @@ -48,12 +48,14 @@ class ScreenshareLayers : public TemporalLayers { bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; void PopulateCodecSpecific(bool base_layer_sync, + const TemporalLayers::FrameConfig& tl_config, CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) override; void FrameEncoded(unsigned int size, int qp) override; - int CurrentLayerId() const override; + int GetTemporalLayerId( + const TemporalLayers::FrameConfig& tl_config) const override; uint8_t Tl0PicIdx() const override; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index 754a63fb7e..f6b4beb379 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -58,13 +58,14 @@ class ScreenshareLayerTest : public ::testing::Test { bool base_sync, CodecSpecificInfoVP8* vp8_info, int* flags) { - TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = + layers_->UpdateLayerConfig(timestamp); if (tl_config.drop_frame) { *flags = -1; return; } *flags = VP8EncoderImpl::EncodeFlags(tl_config); - layers_->PopulateCodecSpecific(base_sync, vp8_info, timestamp); + layers_->PopulateCodecSpecific(base_sync, tl_config, vp8_info, timestamp); ASSERT_NE(-1, frame_size_); layers_->FrameEncoded(frame_size_, kDefaultQp); } @@ -111,10 +112,11 @@ class ScreenshareLayerTest : public ::testing::Test { int SkipUntilTl(int layer, int timestamp) { CodecSpecificInfoVP8 vp8_info; for (int i = 0; i < 5; ++i) { - TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = + layers_->UpdateLayerConfig(timestamp); VP8EncoderImpl::EncodeFlags(tl_config); timestamp += kTimestampDelta5Fps; - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); if (vp8_info.temporalIdx != layer) { layers_->FrameEncoded(frame_size_, kDefaultQp); } else { @@ -149,17 +151,20 @@ TEST_F(ScreenshareLayerTest, 1Layer) { // One layer screenshare should not use the frame dropper as all frames will // belong to the base layer. const int kSingleLayerFlags = 0; - flags = VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp)); + TemporalLayers::FrameConfig tl_config; + tl_config = layers_->UpdateLayerConfig(timestamp); + flags = VP8EncoderImpl::EncodeFlags(tl_config); EXPECT_EQ(kSingleLayerFlags, flags); - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); EXPECT_EQ(static_cast(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); layers_->FrameEncoded(frame_size_, kDefaultQp); - flags = VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp)); + tl_config = layers_->UpdateLayerConfig(timestamp); + flags = VP8EncoderImpl::EncodeFlags(tl_config); EXPECT_EQ(kSingleLayerFlags, flags); timestamp += kTimestampDelta5Fps; - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); EXPECT_EQ(static_cast(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); @@ -244,8 +249,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1; for (int i = 0; i < kNumFrames; ++i) { timestamp += kTimestampDelta5Fps; - layers_->UpdateLayerConfig(timestamp); - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + TemporalLayers::FrameConfig tl_config = + layers_->UpdateLayerConfig(timestamp); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); // Simulate TL1 being at least 8 qp steps better. if (vp8_info.temporalIdx == 0) { @@ -273,8 +279,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { kFrameRate; for (int i = 0; i < kNumFrames; ++i) { timestamp += kTimestampDelta5Fps; - layers_->UpdateLayerConfig(timestamp); - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + TemporalLayers::FrameConfig tl_config = + layers_->UpdateLayerConfig(timestamp); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); // Simulate TL1 being at least 8 qp steps better. if (vp8_info.temporalIdx == 0) { @@ -292,9 +299,10 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { bool bumped_tl0_quality = false; for (int i = 0; i < 3; ++i) { timestamp += kTimestampDelta5Fps; - TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = + layers_->UpdateLayerConfig(timestamp); int flags = VP8EncoderImpl::EncodeFlags(tl_config); - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); if (vp8_info.temporalIdx == 0) { // Bump TL0 to same quality as TL1. @@ -442,9 +450,9 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { layers_->FrameEncoded(0, kDefaultQp); timestamp += kTimestampDelta5Fps; EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); - EXPECT_EQ(kTl0Flags, - VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp))); - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + TemporalLayers::FrameConfig tl_config = layers_->UpdateLayerConfig(timestamp); + EXPECT_EQ(kTl0Flags, VP8EncoderImpl::EncodeFlags(tl_config)); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); layers_->FrameEncoded(frame_size_, kDefaultQp); timestamp = SkipUntilTl(0, timestamp); @@ -452,10 +460,10 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { EXPECT_LT(cfg.rc_max_quantizer, static_cast(kDefaultQp)); layers_->FrameEncoded(frame_size_, kDefaultQp); - layers_->UpdateLayerConfig(timestamp); + tl_config = layers_->UpdateLayerConfig(timestamp); timestamp += kTimestampDelta5Fps; EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); EXPECT_EQ(cfg.rc_max_quantizer, static_cast(kDefaultQp)); layers_->FrameEncoded(frame_size_, kDefaultQp); @@ -465,9 +473,9 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { layers_->FrameEncoded(0, kDefaultQp); timestamp += kTimestampDelta5Fps; EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); - EXPECT_EQ(kTl1Flags, - VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp))); - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + tl_config = layers_->UpdateLayerConfig(timestamp); + EXPECT_EQ(kTl1Flags, VP8EncoderImpl::EncodeFlags(tl_config)); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); layers_->FrameEncoded(frame_size_, kDefaultQp); timestamp = SkipUntilTl(1, timestamp); @@ -475,10 +483,10 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { EXPECT_LT(cfg.rc_max_quantizer, static_cast(kDefaultQp)); layers_->FrameEncoded(frame_size_, kDefaultQp); - layers_->UpdateLayerConfig(timestamp); + tl_config = layers_->UpdateLayerConfig(timestamp); timestamp += kTimestampDelta5Fps; EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); - layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp); EXPECT_EQ(cfg.rc_max_quantizer, static_cast(kDefaultQp)); layers_->FrameEncoded(frame_size_, kDefaultQp); } @@ -522,7 +530,8 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) { for (int64_t timestamp = 0; timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds; timestamp += kTimestampDelta5Fps) { - TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp); + TemporalLayers::FrameConfig tl_config = + layers_->UpdateLayerConfig(timestamp); if (tl_config.drop_frame) { dropped_frame = true; continue; diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h index 12917fd389..471daa636e 100644 --- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -24,52 +24,57 @@ namespace webrtc { struct CodecSpecificInfoVP8; -enum TemporalBufferFlags { - kNone = 0, - kReference = 1, - kUpdate = 2, - kReferenceAndUpdate = kReference | kUpdate, -}; - +// TODO(pbos): Remove along with layer_sync and freeze_entropy, they are +// derivable from picture_idx and should be exposed with +// TL::IsLayerSync(TL::FrameConfig). enum TemporalFlags { kLayerSync = 1, kFreezeEntropy = 2 }; -struct TemporalReferences { - TemporalReferences(TemporalBufferFlags last, - TemporalBufferFlags golden, - TemporalBufferFlags arf); - TemporalReferences(TemporalBufferFlags last, - TemporalBufferFlags golden, - TemporalBufferFlags arf, - int extra_flags); - - const bool drop_frame; - const TemporalBufferFlags last_buffer_flags; - const TemporalBufferFlags golden_buffer_flags; - const TemporalBufferFlags arf_buffer_flags; - - // TODO(pbos): Consider breaking these out of here and returning only a - // pattern index that needs to be returned to fill CodecSpecificInfoVP8 or - // EncodeFlags. - const bool layer_sync; - const bool freeze_entropy; - - private: - TemporalReferences(TemporalBufferFlags last, - TemporalBufferFlags golden, - TemporalBufferFlags arf, - bool layer_sync, - bool freeze_entropy); -}; - class TemporalLayers { public: + enum BufferFlags { + kNone = 0, + kReference = 1, + kUpdate = 2, + kReferenceAndUpdate = kReference | kUpdate, + }; + + struct FrameConfig { + FrameConfig(); + + FrameConfig(BufferFlags last, BufferFlags golden, BufferFlags arf); + FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + int extra_flags); + + bool drop_frame; + BufferFlags last_buffer_flags; + BufferFlags golden_buffer_flags; + BufferFlags arf_buffer_flags; + + // TODO(pbos): Consider breaking these out of here and returning only a + // pattern index that needs to be returned to fill CodecSpecificInfoVP8 or + // EncodeFlags. + bool layer_sync; + bool freeze_entropy; + + int pattern_idx; + + private: + FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + bool layer_sync, + bool freeze_entropy); + }; + // Factory for TemporalLayer strategy. Default behavior is a fixed pattern // of temporal layers. See default_temporal_layers.cc virtual ~TemporalLayers() {} // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - virtual TemporalReferences UpdateLayerConfig(uint32_t timestamp) = 0; + virtual FrameConfig UpdateLayerConfig(uint32_t timestamp) = 0; // Update state based on new bitrate target and incoming framerate. // Returns the bitrate allocation for the active temporal layers. @@ -81,17 +86,19 @@ class TemporalLayers { // Returns true iff the configuration was actually modified. virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0; - virtual void PopulateCodecSpecific(bool is_keyframe, - CodecSpecificInfoVP8* vp8_info, - uint32_t timestamp) = 0; + virtual void PopulateCodecSpecific( + bool is_keyframe, + const TemporalLayers::FrameConfig& tl_config, + CodecSpecificInfoVP8* vp8_info, + uint32_t timestamp) = 0; virtual void FrameEncoded(unsigned int size, int qp) = 0; - virtual int CurrentLayerId() const = 0; - // Returns the current tl0_pic_idx, so it can be reused in future // instantiations. virtual uint8_t Tl0PicIdx() const = 0; + virtual int GetTemporalLayerId( + const TemporalLayers::FrameConfig& tl_config) const = 0; }; class TemporalLayersListener; diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 595cea6af6..d45abcd1b6 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -133,22 +133,22 @@ VP8Decoder* VP8Decoder::Create() { } vpx_enc_frame_flags_t VP8EncoderImpl::EncodeFlags( - TemporalReferences references) { + const TemporalLayers::FrameConfig& references) { RTC_DCHECK(!references.drop_frame); vpx_enc_frame_flags_t flags = 0; - if ((references.last_buffer_flags & kReference) == 0) + if ((references.last_buffer_flags & TemporalLayers::kReference) == 0) flags |= VP8_EFLAG_NO_REF_LAST; - if ((references.last_buffer_flags & kUpdate) == 0) + if ((references.last_buffer_flags & TemporalLayers::kUpdate) == 0) flags |= VP8_EFLAG_NO_UPD_LAST; - if ((references.golden_buffer_flags & kReference) == 0) + if ((references.golden_buffer_flags & TemporalLayers::kReference) == 0) flags |= VP8_EFLAG_NO_REF_GF; - if ((references.golden_buffer_flags & kUpdate) == 0) + if ((references.golden_buffer_flags & TemporalLayers::kUpdate) == 0) flags |= VP8_EFLAG_NO_UPD_GF; - if ((references.arf_buffer_flags & kReference) == 0) + if ((references.arf_buffer_flags & TemporalLayers::kReference) == 0) flags |= VP8_EFLAG_NO_REF_ARF; - if ((references.arf_buffer_flags & kUpdate) == 0) + if ((references.arf_buffer_flags & TemporalLayers::kUpdate) == 0) flags |= VP8_EFLAG_NO_UPD_ARF; if (references.freeze_entropy) flags |= VP8_EFLAG_NO_UPD_ENTROPY; @@ -703,15 +703,15 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, raw_images_[i].d_h, libyuv::kFilterBilinear); } vpx_enc_frame_flags_t flags[kMaxSimulcastStreams]; + TemporalLayers::FrameConfig tl_configs[kMaxSimulcastStreams]; for (size_t i = 0; i < encoders_.size(); ++i) { - TemporalReferences tl_config = - temporal_layers_[i]->UpdateLayerConfig(frame.timestamp()); + tl_configs[i] = temporal_layers_[i]->UpdateLayerConfig(frame.timestamp()); - if (tl_config.drop_frame) { + if (tl_configs[i].drop_frame) { // Drop this frame. return WEBRTC_VIDEO_CODEC_OK; } - flags[i] = EncodeFlags(tl_config); + flags[i] = EncodeFlags(tl_configs[i]); } bool send_key_frame = false; for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size(); @@ -764,8 +764,9 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, } vpx_codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS, flags[stream_idx]); - vpx_codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID, - temporal_layers_[stream_idx]->CurrentLayerId()); + vpx_codec_control( + &encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID, + temporal_layers_[stream_idx]->GetTemporalLayerId(tl_configs[i])); } // TODO(holmer): Ideally the duration should be the timestamp diff of this // frame and the next frame to be encoded, which we don't have. Instead we @@ -788,11 +789,12 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, return WEBRTC_VIDEO_CODEC_ERROR; timestamp_ += duration; // Examines frame timestamps only. - return GetEncodedPartitions(frame); + return GetEncodedPartitions(tl_configs, frame); } void VP8EncoderImpl::PopulateCodecSpecific( CodecSpecificInfo* codec_specific, + const TemporalLayers::FrameConfig& tl_config, const vpx_codec_cx_pkt_t& pkt, int stream_idx, uint32_t timestamp) { @@ -805,12 +807,15 @@ void VP8EncoderImpl::PopulateCodecSpecific( vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0; temporal_layers_[stream_idx]->PopulateCodecSpecific( - (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, vp8Info, timestamp); + (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, tl_config, vp8Info, + timestamp); // Prepare next. picture_id_[stream_idx] = (picture_id_[stream_idx] + 1) & 0x7FFF; } -int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image) { +int VP8EncoderImpl::GetEncodedPartitions( + const TemporalLayers::FrameConfig tl_configs[], + const VideoFrame& input_image) { int bw_resolutions_disabled = (encoders_.size() > 1) ? NumStreamsDisabled(send_stream_) : -1; @@ -860,8 +865,8 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image) { if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) { encoded_images_[encoder_idx]._frameType = kVideoFrameKey; } - PopulateCodecSpecific(&codec_specific, *pkt, stream_idx, - input_image.timestamp()); + PopulateCodecSpecific(&codec_specific, tl_configs[stream_idx], *pkt, + stream_idx, input_image.timestamp()); break; } } diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h index 34f21e0cf5..0d7ecbaa5f 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h @@ -62,7 +62,8 @@ class VP8EncoderImpl : public VP8Encoder { const char* ImplementationName() const override; - static vpx_enc_frame_flags_t EncodeFlags(TemporalReferences references); + static vpx_enc_frame_flags_t EncodeFlags( + const TemporalLayers::FrameConfig& references); private: void SetupTemporalLayers(int num_streams, @@ -79,11 +80,13 @@ class VP8EncoderImpl : public VP8Encoder { int InitAndSetControlSettings(); void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + const TemporalLayers::FrameConfig& tl_config, const vpx_codec_cx_pkt& pkt, int stream_idx, uint32_t timestamp); - int GetEncodedPartitions(const VideoFrame& input_image); + int GetEncodedPartitions(const TemporalLayers::FrameConfig tl_configs[], + const VideoFrame& input_image); // Set the stream state for stream |stream_idx|. void SetStreamState(bool send_stream, int stream_idx); diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index 562718600b..e0aac1056b 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -29,14 +29,18 @@ constexpr uint32_t kFramerateFps = 5; class MockTemporalLayers : public TemporalLayers { public: - MOCK_METHOD1(UpdateLayerConfig, TemporalReferences(uint32_t)); + MOCK_METHOD1(UpdateLayerConfig, TemporalLayers::FrameConfig(uint32_t)); MOCK_METHOD3(OnRatesUpdated, std::vector(int, int, int)); MOCK_METHOD1(UpdateConfiguration, bool(vpx_codec_enc_cfg_t*)); - MOCK_METHOD3(PopulateCodecSpecific, - void(bool, CodecSpecificInfoVP8*, uint32_t)); + MOCK_METHOD4(PopulateCodecSpecific, + void(bool, + const TemporalLayers::FrameConfig&, + CodecSpecificInfoVP8*, + uint32_t)); MOCK_METHOD2(FrameEncoded, void(unsigned int, int)); - MOCK_CONST_METHOD0(CurrentLayerId, int()); MOCK_CONST_METHOD0(Tl0PicIdx, uint8_t()); + MOCK_CONST_METHOD1(GetTemporalLayerId, + int(const TemporalLayers::FrameConfig&)); }; } // namespace