From 90d6efbd4eb4b3a0a6fc26da0bc2f04843865b3e Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Wed, 25 Sep 2019 09:06:50 +0000 Subject: [PATCH] Revert "VP9 encoder: handle disabled layers correctly" This reverts commit 88fe84b7fbcb8dffe07b98d21d8a11572259c0d0. Reason for revert: Downstream project isn't updated to the latest libvpx roll yet, thus some tests are broken. Original change's description: > VP9 encoder: handle disabled layers correctly > > Now vp9 screenshare would enable new layers as soon as requested and will > force all spatial layers present on the next frame, even if they should be > dropped because of frame-rate limiting. > > This might cause frame-rate liming to be exceeded if layer is toggling on > and off very often, but this situation is bad itself. E.g. in realtime video > it will cause too many key-frames. > > Now SvcRateAllocator and VP9EncoderImpl are aware that there may be some skipped > layers before the first enabled. Key-frames and ss_info triggering logic is also > updated. > > Bug: webrtc:10977 > Change-Id: Ie2555210c0368a1d3c51ddf6670d0052e6d679de > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/153483 > Commit-Queue: Ilya Nikolaevskiy > Reviewed-by: Sergey Silkin > Cr-Commit-Position: refs/heads/master@{#29296} TBR=ilnik@webrtc.org,ssilkin@webrtc.org Change-Id: If33886a5f8a0c3b33168dcadfe45c11a6f4387c1 No-Presubmit: true No-Tree-Checks: true No-Try: true Bug: webrtc:10977 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/154354 Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#29299} --- .../codecs/vp9/svc_rate_allocator.cc | 103 ++++-------- .../codecs/vp9/svc_rate_allocator.h | 2 - .../codecs/vp9/svc_rate_allocator_unittest.cc | 42 +---- .../codecs/vp9/test/vp9_impl_unittest.cc | 133 ++------------- modules/video_coding/codecs/vp9/vp9_impl.cc | 152 ++++++++---------- modules/video_coding/codecs/vp9/vp9_impl.h | 3 +- 6 files changed, 117 insertions(+), 318 deletions(-) diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc index a02e69aa47..8513b432c3 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc @@ -25,47 +25,34 @@ namespace { const float kSpatialLayeringRateScalingFactor = 0.55f; const float kTemporalLayeringRateScalingFactor = 0.55f; -// Returns numberOfSpatialLayers if no layers are active. -size_t GetFirstActiveLayer(const VideoCodec& codec) { - RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9); - RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u); - size_t layer = 0; - for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) { - if (codec.spatialLayers[layer].active) { - break; - } - } - return layer; -} - static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) { RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9); RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u); - const size_t first_active_layer = GetFirstActiveLayer(codec); - size_t last_active_layer = first_active_layer; - for (; last_active_layer < codec.VP9().numberOfSpatialLayers; - ++last_active_layer) { - if (!codec.spatialLayers[last_active_layer].active) { + size_t num_spatial_layers = 0; + for (; num_spatial_layers < codec.VP9().numberOfSpatialLayers; + ++num_spatial_layers) { + if (!codec.spatialLayers[num_spatial_layers].active) { + // TODO(bugs.webrtc.org/9350): Deactivation of middle layer is not + // implemented. For now deactivation of a VP9 layer deactivates all + // layers above the deactivated one. break; } } - return last_active_layer - first_active_layer; + + return num_spatial_layers; } std::vector AdjustAndVerify( const VideoCodec& codec, - size_t first_active_layer, const std::vector& spatial_layer_rates) { std::vector adjusted_spatial_layer_rates; // Keep track of rate that couldn't be applied to the previous layer due to // max bitrate constraint, try to pass it forward to the next one. DataRate excess_rate = DataRate::Zero(); for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) { - DataRate min_rate = DataRate::kbps( - codec.spatialLayers[first_active_layer + sl_idx].minBitrate); - DataRate max_rate = DataRate::kbps( - codec.spatialLayers[first_active_layer + sl_idx].maxBitrate); + DataRate min_rate = DataRate::kbps(codec.spatialLayers[sl_idx].minBitrate); + DataRate max_rate = DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate); DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate; if (layer_rate < min_rate) { @@ -122,7 +109,6 @@ static std::vector SplitBitrate(size_t num_layers, // Returns the minimum bitrate needed for |num_active_layers| spatial layers to // become active using the configuration specified by |codec|. DataRate FindLayerTogglingThreshold(const VideoCodec& codec, - size_t first_active_layer, size_t num_active_layers) { if (num_active_layers == 1) { return DataRate::kbps(codec.spatialLayers[0].minBitrate); @@ -133,10 +119,8 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec, DataRate upper_bound = DataRate::Zero(); if (num_active_layers > 1) { for (size_t i = 0; i < num_active_layers - 1; ++i) { - lower_bound += DataRate::kbps( - codec.spatialLayers[first_active_layer + i].minBitrate); - upper_bound += DataRate::kbps( - codec.spatialLayers[first_active_layer + i].maxBitrate); + lower_bound += DataRate::kbps(codec.spatialLayers[i].minBitrate); + upper_bound += DataRate::kbps(codec.spatialLayers[i].maxBitrate); } } upper_bound += @@ -147,7 +131,7 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec, // layers respectively. while (upper_bound - lower_bound > DataRate::bps(1)) { DataRate try_rate = (lower_bound + upper_bound) / 2; - if (AdjustAndVerify(codec, first_active_layer, + if (AdjustAndVerify(codec, SplitBitrate(num_active_layers, try_rate, kSpatialLayeringRateScalingFactor)) .size() == num_active_layers) { @@ -160,12 +144,10 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec, } else { DataRate toggling_rate = DataRate::Zero(); for (size_t i = 0; i < num_active_layers - 1; ++i) { - toggling_rate += DataRate::kbps( - codec.spatialLayers[first_active_layer + i].targetBitrate); + toggling_rate += DataRate::kbps(codec.spatialLayers[i].targetBitrate); } - toggling_rate += DataRate::kbps( - codec.spatialLayers[first_active_layer + num_active_layers - 1] - .minBitrate); + toggling_rate += + DataRate::kbps(codec.spatialLayers[num_active_layers - 1].minBitrate); return toggling_rate; } } @@ -210,9 +192,7 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( return bitrate_allocation; } - const size_t first_active_layer = GetFirstActiveLayer(codec_); size_t num_spatial_layers = GetNumActiveSpatialLayers(codec_); - if (num_spatial_layers == 0) { return VideoBitrateAllocation(); // All layers are deactivated. } @@ -245,17 +225,14 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( last_active_layer_count_ = num_spatial_layers; if (codec_.mode == VideoCodecMode::kRealtimeVideo) { - return GetAllocationNormalVideo(total_bitrate, first_active_layer, - num_spatial_layers); + return GetAllocationNormalVideo(total_bitrate, num_spatial_layers); } else { - return GetAllocationScreenSharing(total_bitrate, first_active_layer, - num_spatial_layers); + return GetAllocationScreenSharing(total_bitrate, num_spatial_layers); } } VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( DataRate total_bitrate, - size_t first_active_layer, size_t num_spatial_layers) const { std::vector spatial_layer_rates; if (num_spatial_layers == 0) { @@ -264,10 +241,9 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( num_spatial_layers = 1; spatial_layer_rates.push_back(total_bitrate); } else { - spatial_layer_rates = - AdjustAndVerify(codec_, first_active_layer, - SplitBitrate(num_spatial_layers, total_bitrate, - kSpatialLayeringRateScalingFactor)); + spatial_layer_rates = AdjustAndVerify( + codec_, SplitBitrate(num_spatial_layers, total_bitrate, + kSpatialLayeringRateScalingFactor)); RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers); } @@ -283,13 +259,10 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // layers since they are used for prediction of higher layers and their // references are far apart. if (num_temporal_layers == 1) { - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, - temporal_layer_rates[0].bps()); + bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[0].bps()); } else if (num_temporal_layers == 2) { - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, - temporal_layer_rates[1].bps()); - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, - temporal_layer_rates[0].bps()); + bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[1].bps()); + bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps()); } else { RTC_CHECK_EQ(num_temporal_layers, 3); // In case of three temporal layers the high layer has two frames and the @@ -297,12 +270,9 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // layer frames). Thus high layer requires more bits (comparing pure // bitrate of layer, excluding bitrate of base layers) to keep quality on // par with lower layers. - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, - temporal_layer_rates[2].bps()); - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, - temporal_layer_rates[0].bps()); - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2, - temporal_layer_rates[1].bps()); + bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[2].bps()); + bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps()); + bitrate_allocation.SetBitrate(sl_idx, 2, temporal_layer_rates[1].bps()); } } @@ -314,11 +284,9 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // bit-rate allocated. VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( DataRate total_bitrate, - size_t first_active_layer, size_t num_spatial_layers) const { if (num_spatial_layers == 0 || - total_bitrate < - DataRate::kbps(codec_.spatialLayers[first_active_layer].minBitrate)) { + total_bitrate < DataRate::kbps(codec_.spatialLayers[0].minBitrate)) { return VideoBitrateAllocation(); } VideoBitrateAllocation bitrate_allocation; @@ -326,8 +294,7 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( DataRate allocated_rate = DataRate::Zero(); DataRate top_layer_rate = DataRate::Zero(); size_t sl_idx; - for (sl_idx = first_active_layer; - sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) { + for (sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { const DataRate min_rate = DataRate::kbps(codec_.spatialLayers[sl_idx].minBitrate); const DataRate target_rate = @@ -373,13 +340,11 @@ size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const { } DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) { - const size_t first_active_layer = GetFirstActiveLayer(codec); const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec); DataRate max_bitrate = DataRate::Zero(); for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - max_bitrate += DataRate::kbps( - codec.spatialLayers[first_active_layer + sl_idx].maxBitrate); + max_bitrate += DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate); } if (codec.maxBitrate != 0) { @@ -401,12 +366,10 @@ DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) { absl::InlinedVector SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) { absl::InlinedVector start_bitrates; - const size_t first_active_layer = GetFirstActiveLayer(codec); - const size_t num_layers = GetNumActiveSpatialLayers(codec); + size_t num_layers = GetNumActiveSpatialLayers(codec); DataRate last_rate = DataRate::Zero(); for (size_t i = 1; i <= num_layers; ++i) { - DataRate layer_toggling_rate = - FindLayerTogglingThreshold(codec, first_active_layer, i); + DataRate layer_toggling_rate = FindLayerTogglingThreshold(codec, i); start_bitrates.push_back(layer_toggling_rate); RTC_DCHECK_LE(last_rate, layer_toggling_rate); last_rate = layer_toggling_rate; diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.h b/modules/video_coding/codecs/vp9/svc_rate_allocator.h index a4e0c28cc0..1b14dd6c69 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.h +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.h @@ -38,12 +38,10 @@ class SvcRateAllocator : public VideoBitrateAllocator { private: VideoBitrateAllocation GetAllocationNormalVideo( DataRate total_bitrate, - size_t first_active_layer, size_t num_spatial_layers) const; VideoBitrateAllocation GetAllocationScreenSharing( DataRate total_bitrate, - size_t first_active_layer, size_t num_spatial_layers) const; // Returns the number of layers that are active and have enough bitrate to diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc index 06240a32d8..f4d0924ffa 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc @@ -173,13 +173,12 @@ TEST(SvcRateAllocatorTest, MinBitrateToGetQualityLayer) { EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].minBitrate); } -TEST(SvcRateAllocatorTest, DeactivateHigherLayers) { +TEST(SvcRateAllocatorTest, DeativateLayers) { for (int deactivated_idx = 2; deactivated_idx >= 0; --deactivated_idx) { VideoCodec codec = Configure(1280, 720, 3, 1, false); EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U); - for (int i = deactivated_idx; i < 3; ++i) - codec.spatialLayers[i].active = false; + codec.spatialLayers[deactivated_idx].active = false; SvcRateAllocator allocator = SvcRateAllocator(codec); @@ -198,39 +197,11 @@ TEST(SvcRateAllocatorTest, DeactivateHigherLayers) { } } -TEST(SvcRateAllocatorTest, DeactivateLowerLayers) { - for (int deactivated_idx = 0; deactivated_idx < 3; ++deactivated_idx) { - VideoCodec codec = Configure(1280, 720, 3, 1, false); - EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U); - - for (int i = deactivated_idx; i >= 0; --i) - codec.spatialLayers[i].active = false; - - SvcRateAllocator allocator = SvcRateAllocator(codec); - - VideoBitrateAllocation allocation = allocator.Allocate( - VideoBitrateAllocationParameters(10 * 1000 * 1000, 30)); - - // Ensure layers spatial_idx <= deactivated_idx are deactivated. - for (int spatial_idx = 0; spatial_idx <= deactivated_idx; ++spatial_idx) { - EXPECT_EQ(allocation.GetSpatialLayerSum(spatial_idx), 0UL); - } - - // Ensure layers spatial_idx > deactivated_idx are activated. - for (int spatial_idx = deactivated_idx + 1; spatial_idx < 3; - ++spatial_idx) { - EXPECT_GT(allocation.GetSpatialLayerSum(spatial_idx), 0UL); - } - } -} - TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) { VideoCodec codec = Configure(1280, 720, 3, 1, false); EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U); // Deactivation of base layer deactivates all layers. codec.spatialLayers[0].active = false; - codec.spatialLayers[1].active = false; - codec.spatialLayers[2].active = false; DataRate padding_rate = SvcRateAllocator::GetPaddingBitrate(codec); EXPECT_EQ(padding_rate, DataRate::Zero()); } @@ -309,15 +280,6 @@ TEST_P(SvcRateAllocatorTestParametrizedContentType, PaddingBitrate) { EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL); EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL); EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL); - - // Deactivate all layers. - codec.spatialLayers[0].active = false; - codec.spatialLayers[1].active = false; - codec.spatialLayers[2].active = false; - - padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec); - // No padding expected. - EXPECT_EQ(DataRate::Zero(), padding_bitrate); } TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) { diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index e54ac348ba..648bf648bd 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -352,120 +352,6 @@ TEST_F(TestVp9Impl, EnableDisableSpatialLayers) { } } -TEST_F(TestVp9Impl, DisableEnableBaseLayerTriggersKeyFrame) { - // Configure encoder to produce N spatial layers. Encode frames for all - // layers. Then disable all but the last layer. Then reenable all back again. - const size_t num_spatial_layers = 3; - const size_t num_frames_to_encode = 5; - - ConfigureSvc(num_spatial_layers); - codec_settings_.VP9()->frameDroppingOn = false; - - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->InitEncode(&codec_settings_, kSettings)); - - VideoBitrateAllocation bitrate_allocation; - for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - // Allocate high bit rate to avoid frame dropping due to rate control. - bitrate_allocation.SetBitrate( - sl_idx, 0, - codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 * 2); - } - encoder_->SetRates(VideoEncoder::RateControlParameters( - bitrate_allocation, codec_settings_.maxFramerate)); - - for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) { - SetWaitForEncodedFramesThreshold(num_spatial_layers); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->Encode(*NextInputFrame(), nullptr)); - std::vector encoded_frame; - std::vector codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); - EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available, - frame_num == 0); - } - - // Disable all but top layer. - for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) { - bitrate_allocation.SetBitrate(sl_idx, 0, 0); - } - encoder_->SetRates(VideoEncoder::RateControlParameters( - bitrate_allocation, codec_settings_.maxFramerate)); - - for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) { - SetWaitForEncodedFramesThreshold(1); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->Encode(*NextInputFrame(), nullptr)); - std::vector encoded_frame; - std::vector codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); - // SS available immediatly after switching off. - EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available, - frame_num == 0); - // No key-frames generated for disabling layers. - EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta); - } - - // Force key-frame. - std::vector frame_types = {VideoFrameType::kVideoFrameKey}; - SetWaitForEncodedFramesThreshold(1); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->Encode(*NextInputFrame(), &frame_types)); - std::vector encoded_frame; - std::vector codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); - // Key-frame should be produced. - EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameKey); - - // Enable the second layer back. - // Allocate high bit rate to avoid frame dropping due to rate control. - bitrate_allocation.SetBitrate( - 1, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000 * 2); - encoder_->SetRates(VideoEncoder::RateControlParameters( - bitrate_allocation, codec_settings_.maxFramerate)); - - for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) { - SetWaitForEncodedFramesThreshold(2); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->Encode(*NextInputFrame(), nullptr)); - std::vector encoded_frame; - std::vector codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); - // SS available immediatly after switching on. - EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available, - frame_num == 0); - // Keyframe should be generated when enabling lower layers. - const VideoFrameType expected_type = frame_num == 0 - ? VideoFrameType::kVideoFrameKey - : VideoFrameType::kVideoFrameDelta; - EXPECT_EQ(encoded_frame[0]._frameType, expected_type); - } - - // Enable the first layer back. - // Allocate high bit rate to avoid frame dropping due to rate control. - bitrate_allocation.SetBitrate( - 0, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 * 2); - encoder_->SetRates(VideoEncoder::RateControlParameters( - bitrate_allocation, codec_settings_.maxFramerate)); - - for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) { - SetWaitForEncodedFramesThreshold(num_spatial_layers); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->Encode(*NextInputFrame(), nullptr)); - std::vector encoded_frame; - std::vector codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); - // SS available immediatly after switching on. - EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available, - frame_num == 0); - // Keyframe should be generated when enabling lower layers. - const VideoFrameType expected_type = frame_num == 0 - ? VideoFrameType::kVideoFrameKey - : VideoFrameType::kVideoFrameDelta; - EXPECT_EQ(encoded_frame[0]._frameType, expected_type); - } -} - TEST_F(TestVp9Impl, EndOfPicture) { const size_t num_spatial_layers = 2; ConfigureSvc(num_spatial_layers); @@ -846,11 +732,14 @@ TEST_F(TestVp9Impl, EnablingDisablingUpperLayerAccrossGof) { false); } -TEST_F(TestVp9Impl, EnablingNewLayerInScreenshareForcesAllLayersWithSS) { +TEST_F(TestVp9Impl, EnablingNewLayerIsDelayedInScreenshareAndAddsSsInfo) { const size_t num_spatial_layers = 3; // Chosen by hand, the 2nd frame is dropped with configured per-layer max // framerate. const size_t num_frames_to_encode_before_drop = 1; + // Chosen by hand, exactly 5 frames are dropped for input fps=30 and max + // framerate = 5. + const size_t num_dropped_frames = 5; codec_settings_.maxFramerate = 30; ConfigureSvc(num_spatial_layers); @@ -895,8 +784,18 @@ TEST_F(TestVp9Impl, EnablingNewLayerInScreenshareForcesAllLayersWithSS) { encoder_->SetRates(VideoEncoder::RateControlParameters( bitrate_allocation, codec_settings_.maxFramerate)); - // All layers are encoded, even though frame dropping should happen. - SetWaitForEncodedFramesThreshold(num_spatial_layers); + for (size_t frame_num = 0; frame_num < num_dropped_frames; ++frame_num) { + SetWaitForEncodedFramesThreshold(1); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(*NextInputFrame(), nullptr)); + // First layer is dropped due to frame rate cap. The last layer should not + // be enabled yet. + std::vector encoded_frames; + std::vector codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info)); + } + + SetWaitForEncodedFramesThreshold(2); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*NextInputFrame(), nullptr)); // Now all 3 layers should be encoded. diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index b379e798c9..42ab4f77c6 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -15,7 +15,6 @@ #include #include -#include #include #include "absl/memory/memory.h" @@ -138,19 +137,15 @@ ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t, return ColorSpace(primaries, transfer, matrix, range); } -std::pair GetActiveLayers( - const VideoBitrateAllocation& allocation) { +bool MoreLayersEnabled(const VideoBitrateAllocation& first, + const VideoBitrateAllocation& second) { for (size_t sl_idx = 0; sl_idx < kMaxSpatialLayers; ++sl_idx) { - if (allocation.GetSpatialLayerSum(sl_idx) > 0) { - size_t last_layer = sl_idx + 1; - while (last_layer < kMaxSpatialLayers && - allocation.GetSpatialLayerSum(last_layer) > 0) { - ++last_layer; - } - return std::make_pair(sl_idx, last_layer); + if (first.GetSpatialLayerSum(sl_idx) > 0 && + second.GetSpatialLayerSum(sl_idx) == 0) { + return true; } } - return {0, 0}; + return false; } uint32_t Interpolate(uint32_t low, @@ -229,7 +224,6 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) num_temporal_layers_(0), num_spatial_layers_(0), num_active_spatial_layers_(0), - first_active_layer_(0), layer_deactivation_requires_key_frame_( field_trial::IsEnabled("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation")), is_svc_(false), @@ -243,7 +237,6 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) full_superframe_drop_(true), first_frame_in_picture_(true), ss_info_needed_(false), - force_all_active_layers_(false), is_flexible_mode_(false), variable_framerate_experiment_(ParseVariableFramerateConfig( "WebRTC-VP9VariableFramerateScreenshare")), @@ -296,31 +289,13 @@ bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const { bool VP9EncoderImpl::SetSvcRates( const VideoBitrateAllocation& bitrate_allocation) { - std::pair current_layers = - GetActiveLayers(current_bitrate_allocation_); - std::pair new_layers = GetActiveLayers(bitrate_allocation); - - const bool layer_activation_requires_key_frame = - inter_layer_pred_ == InterLayerPredMode::kOff || - inter_layer_pred_ == InterLayerPredMode::kOnKeyPic; - const bool lower_layers_enabled = new_layers.first < current_layers.first; - const bool higher_layers_enabled = new_layers.second > current_layers.second; - const bool disabled_layers = new_layers.first > current_layers.first || - new_layers.second < current_layers.second; - - if (lower_layers_enabled || - (higher_layers_enabled && layer_activation_requires_key_frame) || - (disabled_layers && layer_deactivation_requires_key_frame_)) { - force_key_frame_ = true; - } - - if (current_layers != new_layers) { - ss_info_needed_ = true; - } - config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps(); if (ExplicitlyConfiguredSpatialLayers()) { + const bool layer_activation_requires_key_frame = + inter_layer_pred_ == InterLayerPredMode::kOff || + inter_layer_pred_ == InterLayerPredMode::kOnKeyPic; + for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) { const bool was_layer_active = (config_->ss_target_bitrate[sl_idx] > 0); config_->ss_target_bitrate[sl_idx] = @@ -331,6 +306,15 @@ bool VP9EncoderImpl::SetSvcRates( bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000; } + const bool is_active_layer = (config_->ss_target_bitrate[sl_idx] > 0); + if (!was_layer_active && is_active_layer && + layer_activation_requires_key_frame) { + force_key_frame_ = true; + } else if (was_layer_active && !is_active_layer && + layer_deactivation_requires_key_frame_) { + force_key_frame_ = true; + } + if (!was_layer_active) { // Reset frame rate controller if layer is resumed after pause. framerate_controller_[sl_idx].Reset(); @@ -383,34 +367,13 @@ bool VP9EncoderImpl::SetSvcRates( } num_active_spatial_layers_ = 0; - first_active_layer_ = 0; - bool seen_active_layer = false; - bool expect_no_more_active_layers = false; for (int i = 0; i < num_spatial_layers_; ++i) { if (config_->ss_target_bitrate[i] > 0) { - RTC_DCHECK(!expect_no_more_active_layers) << "Only middle layer is " - "deactivated."; - if (!seen_active_layer) { - first_active_layer_ = i; - } - num_active_spatial_layers_ = i + 1; - seen_active_layer = true; - } else { - expect_no_more_active_layers = seen_active_layer; + ++num_active_spatial_layers_; } } RTC_DCHECK_GT(num_active_spatial_layers_, 0); - if (higher_layers_enabled && !force_key_frame_) { - // Prohibit drop of all layers for the next frame, so newly enabled - // layer would have a valid spatial reference. - for (size_t i = 0; i < num_spatial_layers_; ++i) { - svc_drop_frame_.framedrop_thresh[i] = 0; - } - force_all_active_layers_ = true; - } - - current_bitrate_allocation_ = bitrate_allocation; return true; } @@ -430,16 +393,7 @@ void VP9EncoderImpl::SetRates(const RateControlParameters& parameters) { } codec_.maxFramerate = static_cast(parameters.framerate_fps + 0.5); - - if (dynamic_rate_settings_) { - // Tweak rate control settings based on available network headroom. - UpdateRateSettings( - config_, GetRateSettings(parameters.bandwidth_allocation.bps() / - parameters.bitrate.get_sum_bps())); - } - - bool res = SetSvcRates(parameters.bitrate); - RTC_DCHECK(res) << "Failed to set new bitrate allocation"; + requested_rate_settings_ = parameters; } // TODO(eladalon): s/inst/codec_settings/g. @@ -876,10 +830,6 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, num_steady_state_frames_ >= variable_framerate_experiment_.frames_before_steady_state; - // Need to check all frame limiters, even if lower layers are disabled, - // because variable frame-rate limiter should be checked after the first - // layer. It's easier to overwrite active layers after, then check all - // cases. for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) { const float layer_fps = framerate_controller_[layer_id.spatial_layer_id].GetTargetRate(); @@ -906,11 +856,6 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, } } - if (force_all_active_layers_) { - layer_id.spatial_layer_id = first_active_layer_; - force_all_active_layers_ = false; - } - RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_); if (layer_id.spatial_layer_id >= num_active_spatial_layers_) { // Drop entire picture. @@ -922,12 +867,50 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, layer_id.temporal_layer_id_per_spatial[sl_idx] = layer_id.temporal_layer_id; } - if (layer_id.spatial_layer_id < first_active_layer_) { - layer_id.spatial_layer_id = first_active_layer_; - } - vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id); + if (requested_rate_settings_) { + if (dynamic_rate_settings_) { + // Tweak rate control settings based on available network headroom. + UpdateRateSettings( + config_, + GetRateSettings( + requested_rate_settings_->bandwidth_allocation.bps() / + requested_rate_settings_->bitrate.get_sum_bps())); + } + + bool more_layers_requested = MoreLayersEnabled( + requested_rate_settings_->bitrate, current_bitrate_allocation_); + bool less_layers_requested = MoreLayersEnabled( + current_bitrate_allocation_, requested_rate_settings_->bitrate); + // In SVC can enable new layers only if all lower layers are encoded and at + // the base temporal layer. + // This will delay rate allocation change until the next frame on the base + // spatial layer. + // In KSVC or simulcast modes KF will be generated for a new layer, so can + // update allocation any time. + bool can_upswitch = + inter_layer_pred_ != InterLayerPredMode::kOn || + (layer_id.spatial_layer_id == 0 && layer_id.temporal_layer_id == 0); + if (!more_layers_requested || can_upswitch) { + current_bitrate_allocation_ = requested_rate_settings_->bitrate; + requested_rate_settings_ = absl::nullopt; + if (!SetSvcRates(current_bitrate_allocation_)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (less_layers_requested || more_layers_requested) { + ss_info_needed_ = true; + } + if (more_layers_requested && !force_key_frame_) { + // Prohibit drop of all layers for the next frame, so newly enabled + // layer would have a valid spatial reference. + for (size_t i = 0; i < num_spatial_layers_; ++i) { + svc_drop_frame_.framedrop_thresh[i] = 0; + } + } + } + } + if (num_spatial_layers_ > 1) { // Update frame dropping settings as they may change on per-frame basis. vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, @@ -1134,15 +1117,10 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, // of key picture (inter-layer prediction is enabled). const bool is_key_frame = is_key_pic && !vp9_info->inter_layer_predicted; if (is_key_frame || (ss_info_needed_ && layer_id.temporal_layer_id == 0 && - layer_id.spatial_layer_id == first_active_layer_)) { + layer_id.spatial_layer_id == 0)) { vp9_info->ss_data_available = true; vp9_info->spatial_layer_resolution_present = true; - // Signal disabled layers. - for (size_t i = 0; i < first_active_layer_; ++i) { - vp9_info->width[i] = 0; - vp9_info->height[i] = 0; - } - for (size_t i = first_active_layer_; i < num_active_spatial_layers_; ++i) { + for (size_t i = 0; i < num_active_spatial_layers_; ++i) { vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] / svc_params_.scaling_factor_den[i]; vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] / diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index a5f2f35336..19c77b62ba 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -119,7 +119,6 @@ class VP9EncoderImpl : public VP9Encoder { uint8_t num_temporal_layers_; uint8_t num_spatial_layers_; // Number of configured SLs uint8_t num_active_spatial_layers_; // Number of actively encoded SLs - uint8_t first_active_layer_; bool layer_deactivation_requires_key_frame_; bool is_svc_; InterLayerPredMode inter_layer_pred_; @@ -131,8 +130,8 @@ class VP9EncoderImpl : public VP9Encoder { vpx_svc_frame_drop_t svc_drop_frame_; bool first_frame_in_picture_; VideoBitrateAllocation current_bitrate_allocation_; + absl::optional requested_rate_settings_; bool ss_info_needed_; - bool force_all_active_layers_; std::vector framerate_controller_;