diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc index 8513b432c3..a02e69aa47 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc @@ -25,34 +25,47 @@ namespace { const float kSpatialLayeringRateScalingFactor = 0.55f; const float kTemporalLayeringRateScalingFactor = 0.55f; +// Returns numberOfSpatialLayers if no layers are active. +size_t GetFirstActiveLayer(const VideoCodec& codec) { + RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9); + RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u); + size_t layer = 0; + for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) { + if (codec.spatialLayers[layer].active) { + break; + } + } + return layer; +} + static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) { RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9); RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u); - size_t num_spatial_layers = 0; - for (; num_spatial_layers < codec.VP9().numberOfSpatialLayers; - ++num_spatial_layers) { - if (!codec.spatialLayers[num_spatial_layers].active) { - // TODO(bugs.webrtc.org/9350): Deactivation of middle layer is not - // implemented. For now deactivation of a VP9 layer deactivates all - // layers above the deactivated one. + const size_t first_active_layer = GetFirstActiveLayer(codec); + size_t last_active_layer = first_active_layer; + for (; last_active_layer < codec.VP9().numberOfSpatialLayers; + ++last_active_layer) { + if (!codec.spatialLayers[last_active_layer].active) { break; } } - - return num_spatial_layers; + return last_active_layer - first_active_layer; } std::vector AdjustAndVerify( const VideoCodec& codec, + size_t first_active_layer, const std::vector& spatial_layer_rates) { std::vector adjusted_spatial_layer_rates; // Keep track of rate that couldn't be applied to the previous layer due to // max bitrate constraint, try to pass it forward to the next one. DataRate excess_rate = DataRate::Zero(); for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) { - DataRate min_rate = DataRate::kbps(codec.spatialLayers[sl_idx].minBitrate); - DataRate max_rate = DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate); + DataRate min_rate = DataRate::kbps( + codec.spatialLayers[first_active_layer + sl_idx].minBitrate); + DataRate max_rate = DataRate::kbps( + codec.spatialLayers[first_active_layer + sl_idx].maxBitrate); DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate; if (layer_rate < min_rate) { @@ -109,6 +122,7 @@ static std::vector SplitBitrate(size_t num_layers, // Returns the minimum bitrate needed for |num_active_layers| spatial layers to // become active using the configuration specified by |codec|. DataRate FindLayerTogglingThreshold(const VideoCodec& codec, + size_t first_active_layer, size_t num_active_layers) { if (num_active_layers == 1) { return DataRate::kbps(codec.spatialLayers[0].minBitrate); @@ -119,8 +133,10 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec, DataRate upper_bound = DataRate::Zero(); if (num_active_layers > 1) { for (size_t i = 0; i < num_active_layers - 1; ++i) { - lower_bound += DataRate::kbps(codec.spatialLayers[i].minBitrate); - upper_bound += DataRate::kbps(codec.spatialLayers[i].maxBitrate); + lower_bound += DataRate::kbps( + codec.spatialLayers[first_active_layer + i].minBitrate); + upper_bound += DataRate::kbps( + codec.spatialLayers[first_active_layer + i].maxBitrate); } } upper_bound += @@ -131,7 +147,7 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec, // layers respectively. while (upper_bound - lower_bound > DataRate::bps(1)) { DataRate try_rate = (lower_bound + upper_bound) / 2; - if (AdjustAndVerify(codec, + if (AdjustAndVerify(codec, first_active_layer, SplitBitrate(num_active_layers, try_rate, kSpatialLayeringRateScalingFactor)) .size() == num_active_layers) { @@ -144,10 +160,12 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec, } else { DataRate toggling_rate = DataRate::Zero(); for (size_t i = 0; i < num_active_layers - 1; ++i) { - toggling_rate += DataRate::kbps(codec.spatialLayers[i].targetBitrate); + toggling_rate += DataRate::kbps( + codec.spatialLayers[first_active_layer + i].targetBitrate); } - toggling_rate += - DataRate::kbps(codec.spatialLayers[num_active_layers - 1].minBitrate); + toggling_rate += DataRate::kbps( + codec.spatialLayers[first_active_layer + num_active_layers - 1] + .minBitrate); return toggling_rate; } } @@ -192,7 +210,9 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( return bitrate_allocation; } + const size_t first_active_layer = GetFirstActiveLayer(codec_); size_t num_spatial_layers = GetNumActiveSpatialLayers(codec_); + if (num_spatial_layers == 0) { return VideoBitrateAllocation(); // All layers are deactivated. } @@ -225,14 +245,17 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( last_active_layer_count_ = num_spatial_layers; if (codec_.mode == VideoCodecMode::kRealtimeVideo) { - return GetAllocationNormalVideo(total_bitrate, num_spatial_layers); + return GetAllocationNormalVideo(total_bitrate, first_active_layer, + num_spatial_layers); } else { - return GetAllocationScreenSharing(total_bitrate, num_spatial_layers); + return GetAllocationScreenSharing(total_bitrate, first_active_layer, + num_spatial_layers); } } VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( DataRate total_bitrate, + size_t first_active_layer, size_t num_spatial_layers) const { std::vector spatial_layer_rates; if (num_spatial_layers == 0) { @@ -241,9 +264,10 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( num_spatial_layers = 1; spatial_layer_rates.push_back(total_bitrate); } else { - spatial_layer_rates = AdjustAndVerify( - codec_, SplitBitrate(num_spatial_layers, total_bitrate, - kSpatialLayeringRateScalingFactor)); + spatial_layer_rates = + AdjustAndVerify(codec_, first_active_layer, + SplitBitrate(num_spatial_layers, total_bitrate, + kSpatialLayeringRateScalingFactor)); RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers); } @@ -259,10 +283,13 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // layers since they are used for prediction of higher layers and their // references are far apart. if (num_temporal_layers == 1) { - bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[0].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, + temporal_layer_rates[0].bps()); } else if (num_temporal_layers == 2) { - bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[1].bps()); - bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, + temporal_layer_rates[1].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, + temporal_layer_rates[0].bps()); } else { RTC_CHECK_EQ(num_temporal_layers, 3); // In case of three temporal layers the high layer has two frames and the @@ -270,9 +297,12 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // layer frames). Thus high layer requires more bits (comparing pure // bitrate of layer, excluding bitrate of base layers) to keep quality on // par with lower layers. - bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[2].bps()); - bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps()); - bitrate_allocation.SetBitrate(sl_idx, 2, temporal_layer_rates[1].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, + temporal_layer_rates[2].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, + temporal_layer_rates[0].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2, + temporal_layer_rates[1].bps()); } } @@ -284,9 +314,11 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // bit-rate allocated. VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( DataRate total_bitrate, + size_t first_active_layer, size_t num_spatial_layers) const { if (num_spatial_layers == 0 || - total_bitrate < DataRate::kbps(codec_.spatialLayers[0].minBitrate)) { + total_bitrate < + DataRate::kbps(codec_.spatialLayers[first_active_layer].minBitrate)) { return VideoBitrateAllocation(); } VideoBitrateAllocation bitrate_allocation; @@ -294,7 +326,8 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( DataRate allocated_rate = DataRate::Zero(); DataRate top_layer_rate = DataRate::Zero(); size_t sl_idx; - for (sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { + for (sl_idx = first_active_layer; + sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) { const DataRate min_rate = DataRate::kbps(codec_.spatialLayers[sl_idx].minBitrate); const DataRate target_rate = @@ -340,11 +373,13 @@ size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const { } DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) { + const size_t first_active_layer = GetFirstActiveLayer(codec); const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec); DataRate max_bitrate = DataRate::Zero(); for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - max_bitrate += DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate); + max_bitrate += DataRate::kbps( + codec.spatialLayers[first_active_layer + sl_idx].maxBitrate); } if (codec.maxBitrate != 0) { @@ -366,10 +401,12 @@ DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) { absl::InlinedVector SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) { absl::InlinedVector start_bitrates; - size_t num_layers = GetNumActiveSpatialLayers(codec); + const size_t first_active_layer = GetFirstActiveLayer(codec); + const size_t num_layers = GetNumActiveSpatialLayers(codec); DataRate last_rate = DataRate::Zero(); for (size_t i = 1; i <= num_layers; ++i) { - DataRate layer_toggling_rate = FindLayerTogglingThreshold(codec, i); + DataRate layer_toggling_rate = + FindLayerTogglingThreshold(codec, first_active_layer, i); start_bitrates.push_back(layer_toggling_rate); RTC_DCHECK_LE(last_rate, layer_toggling_rate); last_rate = layer_toggling_rate; diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.h b/modules/video_coding/codecs/vp9/svc_rate_allocator.h index 1b14dd6c69..a4e0c28cc0 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.h +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.h @@ -38,10 +38,12 @@ class SvcRateAllocator : public VideoBitrateAllocator { private: VideoBitrateAllocation GetAllocationNormalVideo( DataRate total_bitrate, + size_t first_active_layer, size_t num_spatial_layers) const; VideoBitrateAllocation GetAllocationScreenSharing( DataRate total_bitrate, + size_t first_active_layer, size_t num_spatial_layers) const; // Returns the number of layers that are active and have enough bitrate to diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc index f4d0924ffa..06240a32d8 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc @@ -173,12 +173,13 @@ TEST(SvcRateAllocatorTest, MinBitrateToGetQualityLayer) { EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].minBitrate); } -TEST(SvcRateAllocatorTest, DeativateLayers) { +TEST(SvcRateAllocatorTest, DeactivateHigherLayers) { for (int deactivated_idx = 2; deactivated_idx >= 0; --deactivated_idx) { VideoCodec codec = Configure(1280, 720, 3, 1, false); EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U); - codec.spatialLayers[deactivated_idx].active = false; + for (int i = deactivated_idx; i < 3; ++i) + codec.spatialLayers[i].active = false; SvcRateAllocator allocator = SvcRateAllocator(codec); @@ -197,11 +198,39 @@ TEST(SvcRateAllocatorTest, DeativateLayers) { } } +TEST(SvcRateAllocatorTest, DeactivateLowerLayers) { + for (int deactivated_idx = 0; deactivated_idx < 3; ++deactivated_idx) { + VideoCodec codec = Configure(1280, 720, 3, 1, false); + EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U); + + for (int i = deactivated_idx; i >= 0; --i) + codec.spatialLayers[i].active = false; + + SvcRateAllocator allocator = SvcRateAllocator(codec); + + VideoBitrateAllocation allocation = allocator.Allocate( + VideoBitrateAllocationParameters(10 * 1000 * 1000, 30)); + + // Ensure layers spatial_idx <= deactivated_idx are deactivated. + for (int spatial_idx = 0; spatial_idx <= deactivated_idx; ++spatial_idx) { + EXPECT_EQ(allocation.GetSpatialLayerSum(spatial_idx), 0UL); + } + + // Ensure layers spatial_idx > deactivated_idx are activated. + for (int spatial_idx = deactivated_idx + 1; spatial_idx < 3; + ++spatial_idx) { + EXPECT_GT(allocation.GetSpatialLayerSum(spatial_idx), 0UL); + } + } +} + TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) { VideoCodec codec = Configure(1280, 720, 3, 1, false); EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U); // Deactivation of base layer deactivates all layers. codec.spatialLayers[0].active = false; + codec.spatialLayers[1].active = false; + codec.spatialLayers[2].active = false; DataRate padding_rate = SvcRateAllocator::GetPaddingBitrate(codec); EXPECT_EQ(padding_rate, DataRate::Zero()); } @@ -280,6 +309,15 @@ TEST_P(SvcRateAllocatorTestParametrizedContentType, PaddingBitrate) { EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL); EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL); EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL); + + // Deactivate all layers. + codec.spatialLayers[0].active = false; + codec.spatialLayers[1].active = false; + codec.spatialLayers[2].active = false; + + padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec); + // No padding expected. + EXPECT_EQ(DataRate::Zero(), padding_bitrate); } TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) { diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index 648bf648bd..e54ac348ba 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -352,6 +352,120 @@ TEST_F(TestVp9Impl, EnableDisableSpatialLayers) { } } +TEST_F(TestVp9Impl, DisableEnableBaseLayerTriggersKeyFrame) { + // Configure encoder to produce N spatial layers. Encode frames for all + // layers. Then disable all but the last layer. Then reenable all back again. + const size_t num_spatial_layers = 3; + const size_t num_frames_to_encode = 5; + + ConfigureSvc(num_spatial_layers); + codec_settings_.VP9()->frameDroppingOn = false; + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + VideoBitrateAllocation bitrate_allocation; + for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { + // Allocate high bit rate to avoid frame dropping due to rate control. + bitrate_allocation.SetBitrate( + sl_idx, 0, + codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 * 2); + } + encoder_->SetRates(VideoEncoder::RateControlParameters( + bitrate_allocation, codec_settings_.maxFramerate)); + + for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) { + SetWaitForEncodedFramesThreshold(num_spatial_layers); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(*NextInputFrame(), nullptr)); + std::vector encoded_frame; + std::vector codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); + EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available, + frame_num == 0); + } + + // Disable all but top layer. + for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) { + bitrate_allocation.SetBitrate(sl_idx, 0, 0); + } + encoder_->SetRates(VideoEncoder::RateControlParameters( + bitrate_allocation, codec_settings_.maxFramerate)); + + for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) { + SetWaitForEncodedFramesThreshold(1); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(*NextInputFrame(), nullptr)); + std::vector encoded_frame; + std::vector codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); + // SS available immediatly after switching off. + EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available, + frame_num == 0); + // No key-frames generated for disabling layers. + EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta); + } + + // Force key-frame. + std::vector frame_types = {VideoFrameType::kVideoFrameKey}; + SetWaitForEncodedFramesThreshold(1); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(*NextInputFrame(), &frame_types)); + std::vector encoded_frame; + std::vector codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); + // Key-frame should be produced. + EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameKey); + + // Enable the second layer back. + // Allocate high bit rate to avoid frame dropping due to rate control. + bitrate_allocation.SetBitrate( + 1, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000 * 2); + encoder_->SetRates(VideoEncoder::RateControlParameters( + bitrate_allocation, codec_settings_.maxFramerate)); + + for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) { + SetWaitForEncodedFramesThreshold(2); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(*NextInputFrame(), nullptr)); + std::vector encoded_frame; + std::vector codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); + // SS available immediatly after switching on. + EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available, + frame_num == 0); + // Keyframe should be generated when enabling lower layers. + const VideoFrameType expected_type = frame_num == 0 + ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta; + EXPECT_EQ(encoded_frame[0]._frameType, expected_type); + } + + // Enable the first layer back. + // Allocate high bit rate to avoid frame dropping due to rate control. + bitrate_allocation.SetBitrate( + 0, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 * 2); + encoder_->SetRates(VideoEncoder::RateControlParameters( + bitrate_allocation, codec_settings_.maxFramerate)); + + for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) { + SetWaitForEncodedFramesThreshold(num_spatial_layers); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(*NextInputFrame(), nullptr)); + std::vector encoded_frame; + std::vector codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info)); + // SS available immediatly after switching on. + EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available, + frame_num == 0); + // Keyframe should be generated when enabling lower layers. + const VideoFrameType expected_type = frame_num == 0 + ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta; + EXPECT_EQ(encoded_frame[0]._frameType, expected_type); + } +} + TEST_F(TestVp9Impl, EndOfPicture) { const size_t num_spatial_layers = 2; ConfigureSvc(num_spatial_layers); @@ -732,14 +846,11 @@ TEST_F(TestVp9Impl, EnablingDisablingUpperLayerAccrossGof) { false); } -TEST_F(TestVp9Impl, EnablingNewLayerIsDelayedInScreenshareAndAddsSsInfo) { +TEST_F(TestVp9Impl, EnablingNewLayerInScreenshareForcesAllLayersWithSS) { const size_t num_spatial_layers = 3; // Chosen by hand, the 2nd frame is dropped with configured per-layer max // framerate. const size_t num_frames_to_encode_before_drop = 1; - // Chosen by hand, exactly 5 frames are dropped for input fps=30 and max - // framerate = 5. - const size_t num_dropped_frames = 5; codec_settings_.maxFramerate = 30; ConfigureSvc(num_spatial_layers); @@ -784,18 +895,8 @@ TEST_F(TestVp9Impl, EnablingNewLayerIsDelayedInScreenshareAndAddsSsInfo) { encoder_->SetRates(VideoEncoder::RateControlParameters( bitrate_allocation, codec_settings_.maxFramerate)); - for (size_t frame_num = 0; frame_num < num_dropped_frames; ++frame_num) { - SetWaitForEncodedFramesThreshold(1); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->Encode(*NextInputFrame(), nullptr)); - // First layer is dropped due to frame rate cap. The last layer should not - // be enabled yet. - std::vector encoded_frames; - std::vector codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info)); - } - - SetWaitForEncodedFramesThreshold(2); + // All layers are encoded, even though frame dropping should happen. + SetWaitForEncodedFramesThreshold(num_spatial_layers); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*NextInputFrame(), nullptr)); // Now all 3 layers should be encoded. diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index 42ab4f77c6..b379e798c9 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -15,6 +15,7 @@ #include #include +#include #include #include "absl/memory/memory.h" @@ -137,15 +138,19 @@ ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t, return ColorSpace(primaries, transfer, matrix, range); } -bool MoreLayersEnabled(const VideoBitrateAllocation& first, - const VideoBitrateAllocation& second) { +std::pair GetActiveLayers( + const VideoBitrateAllocation& allocation) { for (size_t sl_idx = 0; sl_idx < kMaxSpatialLayers; ++sl_idx) { - if (first.GetSpatialLayerSum(sl_idx) > 0 && - second.GetSpatialLayerSum(sl_idx) == 0) { - return true; + if (allocation.GetSpatialLayerSum(sl_idx) > 0) { + size_t last_layer = sl_idx + 1; + while (last_layer < kMaxSpatialLayers && + allocation.GetSpatialLayerSum(last_layer) > 0) { + ++last_layer; + } + return std::make_pair(sl_idx, last_layer); } } - return false; + return {0, 0}; } uint32_t Interpolate(uint32_t low, @@ -224,6 +229,7 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) num_temporal_layers_(0), num_spatial_layers_(0), num_active_spatial_layers_(0), + first_active_layer_(0), layer_deactivation_requires_key_frame_( field_trial::IsEnabled("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation")), is_svc_(false), @@ -237,6 +243,7 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) full_superframe_drop_(true), first_frame_in_picture_(true), ss_info_needed_(false), + force_all_active_layers_(false), is_flexible_mode_(false), variable_framerate_experiment_(ParseVariableFramerateConfig( "WebRTC-VP9VariableFramerateScreenshare")), @@ -289,13 +296,31 @@ bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const { bool VP9EncoderImpl::SetSvcRates( const VideoBitrateAllocation& bitrate_allocation) { + std::pair current_layers = + GetActiveLayers(current_bitrate_allocation_); + std::pair new_layers = GetActiveLayers(bitrate_allocation); + + const bool layer_activation_requires_key_frame = + inter_layer_pred_ == InterLayerPredMode::kOff || + inter_layer_pred_ == InterLayerPredMode::kOnKeyPic; + const bool lower_layers_enabled = new_layers.first < current_layers.first; + const bool higher_layers_enabled = new_layers.second > current_layers.second; + const bool disabled_layers = new_layers.first > current_layers.first || + new_layers.second < current_layers.second; + + if (lower_layers_enabled || + (higher_layers_enabled && layer_activation_requires_key_frame) || + (disabled_layers && layer_deactivation_requires_key_frame_)) { + force_key_frame_ = true; + } + + if (current_layers != new_layers) { + ss_info_needed_ = true; + } + config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps(); if (ExplicitlyConfiguredSpatialLayers()) { - const bool layer_activation_requires_key_frame = - inter_layer_pred_ == InterLayerPredMode::kOff || - inter_layer_pred_ == InterLayerPredMode::kOnKeyPic; - for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) { const bool was_layer_active = (config_->ss_target_bitrate[sl_idx] > 0); config_->ss_target_bitrate[sl_idx] = @@ -306,15 +331,6 @@ bool VP9EncoderImpl::SetSvcRates( bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000; } - const bool is_active_layer = (config_->ss_target_bitrate[sl_idx] > 0); - if (!was_layer_active && is_active_layer && - layer_activation_requires_key_frame) { - force_key_frame_ = true; - } else if (was_layer_active && !is_active_layer && - layer_deactivation_requires_key_frame_) { - force_key_frame_ = true; - } - if (!was_layer_active) { // Reset frame rate controller if layer is resumed after pause. framerate_controller_[sl_idx].Reset(); @@ -367,13 +383,34 @@ bool VP9EncoderImpl::SetSvcRates( } num_active_spatial_layers_ = 0; + first_active_layer_ = 0; + bool seen_active_layer = false; + bool expect_no_more_active_layers = false; for (int i = 0; i < num_spatial_layers_; ++i) { if (config_->ss_target_bitrate[i] > 0) { - ++num_active_spatial_layers_; + RTC_DCHECK(!expect_no_more_active_layers) << "Only middle layer is " + "deactivated."; + if (!seen_active_layer) { + first_active_layer_ = i; + } + num_active_spatial_layers_ = i + 1; + seen_active_layer = true; + } else { + expect_no_more_active_layers = seen_active_layer; } } RTC_DCHECK_GT(num_active_spatial_layers_, 0); + if (higher_layers_enabled && !force_key_frame_) { + // Prohibit drop of all layers for the next frame, so newly enabled + // layer would have a valid spatial reference. + for (size_t i = 0; i < num_spatial_layers_; ++i) { + svc_drop_frame_.framedrop_thresh[i] = 0; + } + force_all_active_layers_ = true; + } + + current_bitrate_allocation_ = bitrate_allocation; return true; } @@ -393,7 +430,16 @@ void VP9EncoderImpl::SetRates(const RateControlParameters& parameters) { } codec_.maxFramerate = static_cast(parameters.framerate_fps + 0.5); - requested_rate_settings_ = parameters; + + if (dynamic_rate_settings_) { + // Tweak rate control settings based on available network headroom. + UpdateRateSettings( + config_, GetRateSettings(parameters.bandwidth_allocation.bps() / + parameters.bitrate.get_sum_bps())); + } + + bool res = SetSvcRates(parameters.bitrate); + RTC_DCHECK(res) << "Failed to set new bitrate allocation"; } // TODO(eladalon): s/inst/codec_settings/g. @@ -830,6 +876,10 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, num_steady_state_frames_ >= variable_framerate_experiment_.frames_before_steady_state; + // Need to check all frame limiters, even if lower layers are disabled, + // because variable frame-rate limiter should be checked after the first + // layer. It's easier to overwrite active layers after, then check all + // cases. for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) { const float layer_fps = framerate_controller_[layer_id.spatial_layer_id].GetTargetRate(); @@ -856,6 +906,11 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, } } + if (force_all_active_layers_) { + layer_id.spatial_layer_id = first_active_layer_; + force_all_active_layers_ = false; + } + RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_); if (layer_id.spatial_layer_id >= num_active_spatial_layers_) { // Drop entire picture. @@ -867,50 +922,12 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, layer_id.temporal_layer_id_per_spatial[sl_idx] = layer_id.temporal_layer_id; } - vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id); - - if (requested_rate_settings_) { - if (dynamic_rate_settings_) { - // Tweak rate control settings based on available network headroom. - UpdateRateSettings( - config_, - GetRateSettings( - requested_rate_settings_->bandwidth_allocation.bps() / - requested_rate_settings_->bitrate.get_sum_bps())); - } - - bool more_layers_requested = MoreLayersEnabled( - requested_rate_settings_->bitrate, current_bitrate_allocation_); - bool less_layers_requested = MoreLayersEnabled( - current_bitrate_allocation_, requested_rate_settings_->bitrate); - // In SVC can enable new layers only if all lower layers are encoded and at - // the base temporal layer. - // This will delay rate allocation change until the next frame on the base - // spatial layer. - // In KSVC or simulcast modes KF will be generated for a new layer, so can - // update allocation any time. - bool can_upswitch = - inter_layer_pred_ != InterLayerPredMode::kOn || - (layer_id.spatial_layer_id == 0 && layer_id.temporal_layer_id == 0); - if (!more_layers_requested || can_upswitch) { - current_bitrate_allocation_ = requested_rate_settings_->bitrate; - requested_rate_settings_ = absl::nullopt; - if (!SetSvcRates(current_bitrate_allocation_)) { - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - if (less_layers_requested || more_layers_requested) { - ss_info_needed_ = true; - } - if (more_layers_requested && !force_key_frame_) { - // Prohibit drop of all layers for the next frame, so newly enabled - // layer would have a valid spatial reference. - for (size_t i = 0; i < num_spatial_layers_; ++i) { - svc_drop_frame_.framedrop_thresh[i] = 0; - } - } - } + if (layer_id.spatial_layer_id < first_active_layer_) { + layer_id.spatial_layer_id = first_active_layer_; } + vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id); + if (num_spatial_layers_ > 1) { // Update frame dropping settings as they may change on per-frame basis. vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, @@ -1117,10 +1134,15 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, // of key picture (inter-layer prediction is enabled). const bool is_key_frame = is_key_pic && !vp9_info->inter_layer_predicted; if (is_key_frame || (ss_info_needed_ && layer_id.temporal_layer_id == 0 && - layer_id.spatial_layer_id == 0)) { + layer_id.spatial_layer_id == first_active_layer_)) { vp9_info->ss_data_available = true; vp9_info->spatial_layer_resolution_present = true; - for (size_t i = 0; i < num_active_spatial_layers_; ++i) { + // Signal disabled layers. + for (size_t i = 0; i < first_active_layer_; ++i) { + vp9_info->width[i] = 0; + vp9_info->height[i] = 0; + } + for (size_t i = first_active_layer_; i < num_active_spatial_layers_; ++i) { vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] / svc_params_.scaling_factor_den[i]; vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] / diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index 19c77b62ba..a5f2f35336 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -119,6 +119,7 @@ class VP9EncoderImpl : public VP9Encoder { uint8_t num_temporal_layers_; uint8_t num_spatial_layers_; // Number of configured SLs uint8_t num_active_spatial_layers_; // Number of actively encoded SLs + uint8_t first_active_layer_; bool layer_deactivation_requires_key_frame_; bool is_svc_; InterLayerPredMode inter_layer_pred_; @@ -130,8 +131,8 @@ class VP9EncoderImpl : public VP9Encoder { vpx_svc_frame_drop_t svc_drop_frame_; bool first_frame_in_picture_; VideoBitrateAllocation current_bitrate_allocation_; - absl::optional requested_rate_settings_; bool ss_info_needed_; + bool force_all_active_layers_; std::vector framerate_controller_;