From fa18e25461c258dca30bd3c382863411f8c3b77b Mon Sep 17 00:00:00 2001 From: pbos Date: Tue, 2 May 2017 02:51:12 -0700 Subject: [PATCH] Extract TL config to VP8 libvpx flag conversion. libvpx flags aren't applicable to hardware encoders or non-libvpx software encoders. This moves libvpx flag conversion into the VP8EncoderImpl integration. BUG=chromium:702017,webrtc:7349 R=brandt@webrtc.org Review-Url: https://codereview.webrtc.org/2849723002 Cr-Commit-Position: refs/heads/master@{#17970} --- .../codecs/vp8/default_temporal_layers.cc | 25 --------- .../vp8/default_temporal_layers_unittest.cc | 18 ++++--- .../codecs/vp8/screenshare_layers_unittest.cc | 54 ++++++++++++------- .../video_coding/codecs/vp8/temporal_layers.h | 2 - .../video_coding/codecs/vp8/vp8_impl.cc | 32 +++++++++-- .../video_coding/codecs/vp8/vp8_impl.h | 5 +- 6 files changed, 80 insertions(+), 56 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc index e2246b06ad..30374a3b33 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -252,31 +252,6 @@ TemporalReferences DefaultTemporalLayers::UpdateLayerConfig( return temporal_pattern_[++pattern_idx_ % temporal_pattern_.size()]; } -int TemporalLayers::EncodeFlags(uint32_t timestamp) { - TemporalReferences references = UpdateLayerConfig(timestamp); - if (references.drop_frame) - return -1; - - int flags = 0; - - if ((references.last_buffer_flags & kReference) == 0) - flags |= VP8_EFLAG_NO_REF_LAST; - if ((references.last_buffer_flags & kUpdate) == 0) - flags |= VP8_EFLAG_NO_UPD_LAST; - if ((references.golden_buffer_flags & kReference) == 0) - flags |= VP8_EFLAG_NO_REF_GF; - if ((references.golden_buffer_flags & kUpdate) == 0) - flags |= VP8_EFLAG_NO_UPD_GF; - if ((references.arf_buffer_flags & kReference) == 0) - flags |= VP8_EFLAG_NO_REF_ARF; - if ((references.arf_buffer_flags & kUpdate) == 0) - flags |= VP8_EFLAG_NO_UPD_ARF; - if (references.freeze_entropy) - flags |= VP8_EFLAG_NO_UPD_ENTROPY; - - return flags; -} - void DefaultTemporalLayers::PopulateCodecSpecific( bool frame_is_keyframe, CodecSpecificInfoVP8* vp8_info, diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index e22c8b4675..f7502447b8 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -8,9 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h" #include "vpx/vp8cx.h" #include "vpx/vpx_encoder.h" -#include "webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h" +#include "webrtc/modules/video_coding/codecs/vp8/vp8_impl.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/test/gtest.h" @@ -83,7 +84,8 @@ TEST(TemporalLayersTest, 2Layers) { uint32_t timestamp = 0; for (int i = 0; i < 16; ++i) { - EXPECT_EQ(expected_flags[i], tl.EncodeFlags(timestamp)); + TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)); tl.PopulateCodecSpecific(false, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); @@ -125,7 +127,8 @@ TEST(TemporalLayersTest, 3Layers) { unsigned int timestamp = 0; for (int i = 0; i < 16; ++i) { - EXPECT_EQ(expected_flags[i], tl.EncodeFlags(timestamp)); + TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)); tl.PopulateCodecSpecific(false, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); @@ -166,7 +169,8 @@ TEST(TemporalLayersTest, 4Layers) { uint32_t timestamp = 0; for (int i = 0; i < 16; ++i) { - EXPECT_EQ(expected_flags[i], tl.EncodeFlags(timestamp)); + TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)); tl.PopulateCodecSpecific(false, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync); @@ -195,13 +199,15 @@ TEST(TemporalLayersTest, KeyFrame) { uint32_t timestamp = 0; for (int i = 0; i < 7; ++i) { - EXPECT_EQ(expected_flags[i], tl.EncodeFlags(timestamp)); + TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)); tl.PopulateCodecSpecific(true, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); EXPECT_EQ(true, vp8_info.layerSync); timestamp += 3000; } - EXPECT_EQ(expected_flags[7], tl.EncodeFlags(timestamp)); + TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp); + EXPECT_EQ(expected_flags[7], VP8EncoderImpl::EncodeFlags(tl_config)); tl.PopulateCodecSpecific(false, &vp8_info, 0); EXPECT_EQ(expected_temporal_idx[7], vp8_info.temporalIdx); EXPECT_EQ(true, vp8_info.layerSync); diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index 8a227a23fa..754a63fb7e 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -14,6 +14,7 @@ #include "vpx/vp8cx.h" #include "vpx/vpx_encoder.h" #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" +#include "webrtc/modules/video_coding/codecs/vp8/vp8_impl.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/utility/mock/mock_frame_dropper.h" #include "webrtc/system_wrappers/include/clock.h" @@ -57,9 +58,12 @@ class ScreenshareLayerTest : public ::testing::Test { bool base_sync, CodecSpecificInfoVP8* vp8_info, int* flags) { - *flags = layers_->EncodeFlags(timestamp); - if (*flags == -1) + TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp); + if (tl_config.drop_frame) { + *flags = -1; return; + } + *flags = VP8EncoderImpl::EncodeFlags(tl_config); layers_->PopulateCodecSpecific(base_sync, vp8_info, timestamp); ASSERT_NE(-1, frame_size_); layers_->FrameEncoded(frame_size_, kDefaultQp); @@ -107,7 +111,8 @@ class ScreenshareLayerTest : public ::testing::Test { int SkipUntilTl(int layer, int timestamp) { CodecSpecificInfoVP8 vp8_info; for (int i = 0; i < 5; ++i) { - layers_->EncodeFlags(timestamp); + TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp); + VP8EncoderImpl::EncodeFlags(tl_config); timestamp += kTimestampDelta5Fps; layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); if (vp8_info.temporalIdx != layer) { @@ -144,14 +149,14 @@ TEST_F(ScreenshareLayerTest, 1Layer) { // One layer screenshare should not use the frame dropper as all frames will // belong to the base layer. const int kSingleLayerFlags = 0; - flags = layers_->EncodeFlags(timestamp); + flags = VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp)); EXPECT_EQ(kSingleLayerFlags, flags); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); EXPECT_EQ(static_cast(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); layers_->FrameEncoded(frame_size_, kDefaultQp); - flags = layers_->EncodeFlags(timestamp); + flags = VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp)); EXPECT_EQ(kSingleLayerFlags, flags); timestamp += kTimestampDelta5Fps; layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); @@ -239,7 +244,7 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1; for (int i = 0; i < kNumFrames; ++i) { timestamp += kTimestampDelta5Fps; - layers_->EncodeFlags(timestamp); + layers_->UpdateLayerConfig(timestamp); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); // Simulate TL1 being at least 8 qp steps better. @@ -268,7 +273,7 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { kFrameRate; for (int i = 0; i < kNumFrames; ++i) { timestamp += kTimestampDelta5Fps; - layers_->EncodeFlags(timestamp); + layers_->UpdateLayerConfig(timestamp); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); // Simulate TL1 being at least 8 qp steps better. @@ -287,7 +292,8 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { bool bumped_tl0_quality = false; for (int i = 0; i < 3; ++i) { timestamp += kTimestampDelta5Fps; - int flags = layers_->EncodeFlags(timestamp); + TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp); + int flags = VP8EncoderImpl::EncodeFlags(tl_config); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); if (vp8_info.temporalIdx == 0) { @@ -436,7 +442,8 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { layers_->FrameEncoded(0, kDefaultQp); timestamp += kTimestampDelta5Fps; EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); - EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(timestamp)); + EXPECT_EQ(kTl0Flags, + VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp))); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); layers_->FrameEncoded(frame_size_, kDefaultQp); @@ -445,7 +452,7 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { EXPECT_LT(cfg.rc_max_quantizer, static_cast(kDefaultQp)); layers_->FrameEncoded(frame_size_, kDefaultQp); - layers_->EncodeFlags(timestamp); + layers_->UpdateLayerConfig(timestamp); timestamp += kTimestampDelta5Fps; EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); @@ -458,7 +465,8 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { layers_->FrameEncoded(0, kDefaultQp); timestamp += kTimestampDelta5Fps; EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); - EXPECT_EQ(kTl1Flags, layers_->EncodeFlags(timestamp)); + EXPECT_EQ(kTl1Flags, + VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp))); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); layers_->FrameEncoded(frame_size_, kDefaultQp); @@ -467,7 +475,7 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { EXPECT_LT(cfg.rc_max_quantizer, static_cast(kDefaultQp)); layers_->FrameEncoded(frame_size_, kDefaultQp); - layers_->EncodeFlags(timestamp); + layers_->UpdateLayerConfig(timestamp); timestamp += kTimestampDelta5Fps; EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); @@ -484,7 +492,8 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { layers_->OnRatesUpdated(kLowBitrateKbps, kLowBitrateKbps, 5); layers_->UpdateConfiguration(&cfg); - EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(kStartTimestamp)); + EXPECT_EQ(kTl0Flags, VP8EncoderImpl::EncodeFlags( + layers_->UpdateLayerConfig(kStartTimestamp))); layers_->FrameEncoded(kLargeFrameSizeBytes, kDefaultQp); const uint32_t kTwoSecondsLater = @@ -494,10 +503,11 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { ASSERT_GT(kStartTimestamp + 90 * (kLargeFrameSizeBytes * 8) / kLowBitrateKbps, kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90)); - EXPECT_EQ(-1, layers_->EncodeFlags(kTwoSecondsLater)); + EXPECT_TRUE(layers_->UpdateLayerConfig(kTwoSecondsLater).drop_frame); // More than two seconds has passed since last frame, one should be emitted // even if bitrate target is then exceeded. - EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(kTwoSecondsLater + 90)); + EXPECT_EQ(kTl0Flags, VP8EncoderImpl::EncodeFlags( + layers_->UpdateLayerConfig(kTwoSecondsLater + 90))); } TEST_F(ScreenshareLayerTest, UpdatesHistograms) { @@ -512,7 +522,12 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) { for (int64_t timestamp = 0; timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds; timestamp += kTimestampDelta5Fps) { - int flags = layers_->EncodeFlags(timestamp); + TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp); + if (tl_config.drop_frame) { + dropped_frame = true; + continue; + } + int flags = VP8EncoderImpl::EncodeFlags(tl_config); if (flags != -1) layers_->UpdateConfiguration(&cfg); @@ -520,7 +535,8 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) { // Simulate one overshoot. layers_->FrameEncoded(0, 0); overshoot = true; - flags = layers_->EncodeFlags(timestamp); + flags = + VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp)); } if (flags == kTl0Flags) { @@ -594,7 +610,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { // Send at regular rate - no drops expected. for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs) { - if (layers_->EncodeFlags(timestamp) == -1) { + if (layers_->UpdateLayerConfig(timestamp).drop_frame) { ++num_discarded_frames; } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; @@ -610,7 +626,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { num_input_frames = 0; num_discarded_frames = 0; for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs / 2) { - if (layers_->EncodeFlags(timestamp) == -1) { + if (layers_->UpdateLayerConfig(timestamp).drop_frame) { ++num_discarded_frames; } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h index ba7f406b31..2a41c6da8b 100644 --- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -71,8 +71,6 @@ class TemporalLayers { // and/or update the reference buffers. virtual TemporalReferences UpdateLayerConfig(uint32_t timestamp) = 0; - int EncodeFlags(uint32_t timestamp); - // Update state based on new bitrate target and incoming framerate. // Returns the bitrate allocation for the active temporal layers. virtual std::vector OnRatesUpdated(int bitrate_kbps, diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 4e30986e82..3935c96b27 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -131,6 +131,30 @@ VP8Decoder* VP8Decoder::Create() { return new VP8DecoderImpl(); } +vpx_enc_frame_flags_t VP8EncoderImpl::EncodeFlags( + TemporalReferences references) { + RTC_DCHECK(!references.drop_frame); + + vpx_enc_frame_flags_t flags = 0; + + if ((references.last_buffer_flags & kReference) == 0) + flags |= VP8_EFLAG_NO_REF_LAST; + if ((references.last_buffer_flags & kUpdate) == 0) + flags |= VP8_EFLAG_NO_UPD_LAST; + if ((references.golden_buffer_flags & kReference) == 0) + flags |= VP8_EFLAG_NO_REF_GF; + if ((references.golden_buffer_flags & kUpdate) == 0) + flags |= VP8_EFLAG_NO_UPD_GF; + if ((references.arf_buffer_flags & kReference) == 0) + flags |= VP8_EFLAG_NO_REF_ARF; + if ((references.arf_buffer_flags & kUpdate) == 0) + flags |= VP8_EFLAG_NO_UPD_ARF; + if (references.freeze_entropy) + flags |= VP8_EFLAG_NO_UPD_ENTROPY; + + return flags; +} + VP8EncoderImpl::VP8EncoderImpl() : use_gf_boost_(webrtc::field_trial::IsEnabled(kVp8GfBoostFieldTrial)), encoded_complete_callback_(nullptr), @@ -682,12 +706,14 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, } vpx_enc_frame_flags_t flags[kMaxSimulcastStreams]; for (size_t i = 0; i < encoders_.size(); ++i) { - int ret = temporal_layers_[i]->EncodeFlags(frame.timestamp()); - if (ret < 0) { + TemporalReferences tl_config = + temporal_layers_[i]->UpdateLayerConfig(frame.timestamp()); + + if (tl_config.drop_frame) { // Drop this frame. return WEBRTC_VIDEO_CODEC_OK; } - flags[i] = ret; + flags[i] = EncodeFlags(tl_config); } bool send_key_frame = false; for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size(); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h index c333436026..f07b0f9399 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h @@ -25,8 +25,9 @@ #include "webrtc/api/video/video_frame.h" #include "webrtc/common_video/include/i420_buffer_pool.h" -#include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/utility/quality_scaler.h" #include "webrtc/video_frame.h" @@ -61,6 +62,8 @@ class VP8EncoderImpl : public VP8Encoder { const char* ImplementationName() const override; + static vpx_enc_frame_flags_t EncodeFlags(TemporalReferences references); + private: void SetupTemporalLayers(int num_streams, int num_temporal_layers,