diff --git a/modules/video_coding/svc/BUILD.gn b/modules/video_coding/svc/BUILD.gn index 784d56f0a8..0246b10590 100644 --- a/modules/video_coding/svc/BUILD.gn +++ b/modules/video_coding/svc/BUILD.gn @@ -134,6 +134,8 @@ if (rtc_include_tests) { deps = [ ":svc_rate_allocator", "..:webrtc_vp9_helpers", + "../../../api/video:video_frame", + "../../../modules/video_coding/codecs/av1:av1_svc_config", "../../../rtc_base:checks", "../../../test:field_trial", "../../../test:test_support", diff --git a/modules/video_coding/svc/svc_rate_allocator.cc b/modules/video_coding/svc/svc_rate_allocator.cc index 0f29fcafaa..e637408a38 100644 --- a/modules/video_coding/svc/svc_rate_allocator.cc +++ b/modules/video_coding/svc/svc_rate_allocator.cc @@ -117,6 +117,46 @@ static std::vector SplitBitrate(size_t num_layers, return bitrates; } +VideoBitrateAllocation DistributeAllocationToTemporalLayers( + std::vector spatial_layer_birates, + size_t first_active_layer, + size_t num_temporal_layers) { + // Distribute rate across temporal layers. Allocate more bits to lower + // layers since they are used for prediction of higher layers and their + // references are far apart. + VideoBitrateAllocation bitrate_allocation; + for (size_t sl_idx = 0; sl_idx < spatial_layer_birates.size(); ++sl_idx) { + std::vector temporal_layer_bitrates = + SplitBitrate(num_temporal_layers, spatial_layer_birates[sl_idx], + kTemporalLayeringRateScalingFactor); + + if (num_temporal_layers == 1) { + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, + temporal_layer_bitrates[0].bps()); + } else if (num_temporal_layers == 2) { + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, + temporal_layer_bitrates[1].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, + temporal_layer_bitrates[0].bps()); + } else { + RTC_CHECK_EQ(num_temporal_layers, 3); + // In case of three temporal layers the high layer has two frames and the + // middle layer has one frame within GOP (in between two consecutive low + // layer frames). Thus high layer requires more bits (comparing pure + // bitrate of layer, excluding bitrate of base layers) to keep quality on + // par with lower layers. + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, + temporal_layer_bitrates[2].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, + temporal_layer_bitrates[0].bps()); + bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2, + temporal_layer_bitrates[1].bps()); + } + } + + return bitrate_allocation; +} + // Returns the minimum bitrate needed for `num_active_layers` spatial layers to // become active using the configuration specified by `codec`. DataRate FindLayerTogglingThreshold(const VideoCodec& codec, @@ -269,19 +309,24 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( } last_active_layer_count_ = num_spatial_layers; - VideoBitrateAllocation allocation; + std::vector spatial_layer_bitrates; if (codec_.mode == VideoCodecMode::kRealtimeVideo) { - allocation = GetAllocationNormalVideo(total_bitrate, active_layers.first, - num_spatial_layers); + spatial_layer_bitrates = DistributeAllocationToSpatialLayersNormalVideo( + total_bitrate, active_layers.first, num_spatial_layers); } else { - allocation = GetAllocationScreenSharing(total_bitrate, active_layers.first, - num_spatial_layers); + spatial_layer_bitrates = DistributeAllocationToSpatialLayersScreenSharing( + total_bitrate, active_layers.first, num_spatial_layers); } + + VideoBitrateAllocation allocation = DistributeAllocationToTemporalLayers( + spatial_layer_bitrates, active_layers.first, num_layers_.temporal); + allocation.set_bw_limited(num_spatial_layers < active_layers.num); return allocation; } -VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( +std::vector +SvcRateAllocator::DistributeAllocationToSpatialLayersNormalVideo( DataRate total_bitrate, size_t first_active_layer, size_t num_spatial_layers) const { @@ -291,67 +336,33 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // bitrate anyway. num_spatial_layers = 1; spatial_layer_rates.push_back(total_bitrate); - } else { - spatial_layer_rates = - AdjustAndVerify(codec_, first_active_layer, - SplitBitrate(num_spatial_layers, total_bitrate, - kSpatialLayeringRateScalingFactor)); - RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers); + return spatial_layer_rates; } - VideoBitrateAllocation bitrate_allocation; - - for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - std::vector temporal_layer_rates = - SplitBitrate(num_layers_.temporal, spatial_layer_rates[sl_idx], - kTemporalLayeringRateScalingFactor); - - // Distribute rate across temporal layers. Allocate more bits to lower - // layers since they are used for prediction of higher layers and their - // references are far apart. - if (num_layers_.temporal == 1) { - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, - temporal_layer_rates[0].bps()); - } else if (num_layers_.temporal == 2) { - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, - temporal_layer_rates[1].bps()); - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, - temporal_layer_rates[0].bps()); - } else { - RTC_CHECK_EQ(num_layers_.temporal, 3); - // In case of three temporal layers the high layer has two frames and the - // middle layer has one frame within GOP (in between two consecutive low - // layer frames). Thus high layer requires more bits (comparing pure - // bitrate of layer, excluding bitrate of base layers) to keep quality on - // par with lower layers. - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, - temporal_layer_rates[2].bps()); - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, - temporal_layer_rates[0].bps()); - bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2, - temporal_layer_rates[1].bps()); - } - } - - return bitrate_allocation; + spatial_layer_rates = + AdjustAndVerify(codec_, first_active_layer, + SplitBitrate(num_spatial_layers, total_bitrate, + kSpatialLayeringRateScalingFactor)); + RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers); + return spatial_layer_rates; } // Bit-rate is allocated in such a way, that the highest enabled layer will have // between min and max bitrate, and all others will have exactly target // bit-rate allocated. -VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( +std::vector +SvcRateAllocator::DistributeAllocationToSpatialLayersScreenSharing( DataRate total_bitrate, size_t first_active_layer, size_t num_spatial_layers) const { - VideoBitrateAllocation bitrate_allocation; - + std::vector spatial_layer_rates; if (num_spatial_layers == 0 || total_bitrate < DataRate::KilobitsPerSec( codec_.spatialLayers[first_active_layer].minBitrate)) { // Always enable at least one layer. - bitrate_allocation.SetBitrate(first_active_layer, 0, total_bitrate.bps()); - return bitrate_allocation; + spatial_layer_rates.push_back(total_bitrate); + return spatial_layer_rates; } DataRate allocated_rate = DataRate::Zero(); @@ -370,7 +381,7 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( } top_layer_rate = std::min(target_rate, total_bitrate - allocated_rate); - bitrate_allocation.SetBitrate(sl_idx, 0, top_layer_rate.bps()); + spatial_layer_rates.push_back(top_layer_rate); allocated_rate += top_layer_rate; } @@ -379,10 +390,10 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( top_layer_rate = std::min( top_layer_rate + (total_bitrate - allocated_rate), DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx - 1].maxBitrate)); - bitrate_allocation.SetBitrate(sl_idx - 1, 0, top_layer_rate.bps()); + spatial_layer_rates.back() = top_layer_rate; } - return bitrate_allocation; + return spatial_layer_rates; } size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const { diff --git a/modules/video_coding/svc/svc_rate_allocator.h b/modules/video_coding/svc/svc_rate_allocator.h index bd75fca284..f64d92bd38 100644 --- a/modules/video_coding/svc/svc_rate_allocator.h +++ b/modules/video_coding/svc/svc_rate_allocator.h @@ -14,6 +14,8 @@ #include #include +#include + #include "absl/container/inlined_vector.h" #include "api/video/video_bitrate_allocation.h" #include "api/video/video_bitrate_allocator.h" @@ -42,12 +44,12 @@ class SvcRateAllocator : public VideoBitrateAllocator { }; static NumLayers GetNumLayers(const VideoCodec& codec); - VideoBitrateAllocation GetAllocationNormalVideo( + std::vector DistributeAllocationToSpatialLayersNormalVideo( DataRate total_bitrate, size_t first_active_layer, size_t num_spatial_layers) const; - VideoBitrateAllocation GetAllocationScreenSharing( + std::vector DistributeAllocationToSpatialLayersScreenSharing( DataRate total_bitrate, size_t first_active_layer, size_t num_spatial_layers) const; diff --git a/modules/video_coding/svc/svc_rate_allocator_unittest.cc b/modules/video_coding/svc/svc_rate_allocator_unittest.cc index 44d1eae667..9aa6c5dbe9 100644 --- a/modules/video_coding/svc/svc_rate_allocator_unittest.cc +++ b/modules/video_coding/svc/svc_rate_allocator_unittest.cc @@ -13,6 +13,8 @@ #include #include +#include "api/video/video_codec_type.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" #include "modules/video_coding/codecs/vp9/svc_config.h" #include "rtc_base/checks.h" #include "test/field_trial.h" @@ -21,7 +23,11 @@ namespace webrtc { namespace test { namespace { -static VideoCodec Configure(size_t width, +using ::testing::Bool; +using ::testing::TestWithParam; + +static VideoCodec Configure(VideoCodecType codecType, + size_t width, size_t height, size_t num_spatial_layers, size_t num_temporal_layers, @@ -29,31 +35,47 @@ static VideoCodec Configure(size_t width, VideoCodec codec; codec.width = width; codec.height = height; - codec.codecType = kVideoCodecVP9; + codec.codecType = codecType; codec.mode = is_screen_sharing ? VideoCodecMode::kScreensharing : VideoCodecMode::kRealtimeVideo; - std::vector spatial_layers = - GetSvcConfig(width, height, 30, /*first_active_layer=*/0, - num_spatial_layers, num_temporal_layers, is_screen_sharing); - RTC_CHECK_LE(spatial_layers.size(), kMaxSpatialLayers); + std::vector spatial_layers; + if (codecType == kVideoCodecVP9) { + spatial_layers = GetSvcConfig(width, height, 30, /*first_active_layer=*/0, + num_spatial_layers, num_temporal_layers, + is_screen_sharing); + RTC_CHECK_LE(spatial_layers.size(), kMaxSpatialLayers); - codec.VP9()->numberOfSpatialLayers = - std::min(num_spatial_layers, spatial_layers.size()); - codec.VP9()->numberOfTemporalLayers = std::min( - num_temporal_layers, spatial_layers.back().numberOfTemporalLayers); + codec.VP9()->numberOfSpatialLayers = + std::min(num_spatial_layers, spatial_layers.size()); + codec.VP9()->numberOfTemporalLayers = std::min( + num_temporal_layers, spatial_layers.back().numberOfTemporalLayers); - for (size_t sl_idx = 0; sl_idx < spatial_layers.size(); ++sl_idx) { - codec.spatialLayers[sl_idx] = spatial_layers[sl_idx]; + for (size_t sl_idx = 0; sl_idx < spatial_layers.size(); ++sl_idx) { + codec.spatialLayers[sl_idx] = spatial_layers[sl_idx]; + } + + return codec; } + RTC_DCHECK_EQ(codecType, kVideoCodecAV1); + + if (num_spatial_layers == 1) { + // SetAv1SvcConfig expects bitrate limits for be set when single spatial + // layer is requested. + codec.minBitrate = 30; + codec.maxBitrate = 5000; + } + + SetAv1SvcConfig(codec, num_temporal_layers, num_spatial_layers); + return codec; } } // namespace TEST(SvcRateAllocatorTest, SingleLayerFor320x180Input) { - VideoCodec codec = Configure(320, 180, 3, 3, false); + VideoCodec codec = Configure(kVideoCodecVP9, 320, 180, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); VideoBitrateAllocation allocation = @@ -64,7 +86,7 @@ TEST(SvcRateAllocatorTest, SingleLayerFor320x180Input) { } TEST(SvcRateAllocatorTest, TwoLayersFor640x360Input) { - VideoCodec codec = Configure(640, 360, 3, 3, false); + VideoCodec codec = Configure(kVideoCodecVP9, 640, 360, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); VideoBitrateAllocation allocation = @@ -76,7 +98,7 @@ TEST(SvcRateAllocatorTest, TwoLayersFor640x360Input) { } TEST(SvcRateAllocatorTest, ThreeLayersFor1280x720Input) { - VideoCodec codec = Configure(1280, 720, 3, 3, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); VideoBitrateAllocation allocation = @@ -89,7 +111,7 @@ TEST(SvcRateAllocatorTest, ThreeLayersFor1280x720Input) { TEST(SvcRateAllocatorTest, BaseLayerNonZeroBitrateEvenIfTotalIfLessThanMinimum) { - VideoCodec codec = Configure(1280, 720, 3, 3, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -103,7 +125,7 @@ TEST(SvcRateAllocatorTest, } TEST(SvcRateAllocatorTest, Disable640x360Layer) { - VideoCodec codec = Configure(1280, 720, 3, 3, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -120,7 +142,7 @@ TEST(SvcRateAllocatorTest, Disable640x360Layer) { } TEST(SvcRateAllocatorTest, Disable1280x720Layer) { - VideoCodec codec = Configure(1280, 720, 3, 3, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -138,7 +160,7 @@ TEST(SvcRateAllocatorTest, Disable1280x720Layer) { } TEST(SvcRateAllocatorTest, BitrateIsCapped) { - VideoCodec codec = Configure(1280, 720, 3, 3, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -155,7 +177,7 @@ TEST(SvcRateAllocatorTest, BitrateIsCapped) { } TEST(SvcRateAllocatorTest, MinBitrateToGetQualityLayer) { - VideoCodec codec = Configure(1280, 720, 3, 1, true); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 1, true); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -175,7 +197,7 @@ TEST(SvcRateAllocatorTest, MinBitrateToGetQualityLayer) { TEST(SvcRateAllocatorTest, DeactivateHigherLayers) { for (int deactivated_idx = 2; deactivated_idx >= 0; --deactivated_idx) { - VideoCodec codec = Configure(1280, 720, 3, 1, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 1, false); EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U); for (int i = deactivated_idx; i < 3; ++i) @@ -200,7 +222,7 @@ TEST(SvcRateAllocatorTest, DeactivateHigherLayers) { TEST(SvcRateAllocatorTest, DeactivateLowerLayers) { for (int deactivated_idx = 0; deactivated_idx < 3; ++deactivated_idx) { - VideoCodec codec = Configure(1280, 720, 3, 1, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 1, false); EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U); for (int i = deactivated_idx; i >= 0; --i) @@ -225,7 +247,7 @@ TEST(SvcRateAllocatorTest, DeactivateLowerLayers) { } TEST(SvcRateAllocatorTest, SignalsBwLimited) { - VideoCodec codec = Configure(1280, 720, 3, 1, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 1, false); SvcRateAllocator allocator = SvcRateAllocator(codec); // Rough estimate calculated by hand. @@ -243,7 +265,7 @@ TEST(SvcRateAllocatorTest, SignalsBwLimited) { } TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) { - VideoCodec codec = Configure(1280, 720, 3, 1, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 1, false); EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U); // Deactivation of base layer deactivates all layers. codec.spatialLayers[0].active = false; @@ -262,7 +284,7 @@ TEST(SvcRateAllocatorTest, FindLayerTogglingThreshold) { const DataRate kTwoLayerMinRate = DataRate::BitsPerSec(299150); const DataRate kThreeLayerMinRate = DataRate::BitsPerSec(891052); - VideoCodec codec = Configure(1280, 720, 3, 1, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 1, false); absl::InlinedVector layer_start_bitrates = SvcRateAllocator::GetLayerStartBitrates(codec); ASSERT_EQ(layer_start_bitrates.size(), 3u); @@ -362,7 +384,7 @@ TEST(SvcRateAllocatorTest, UsesScalabilityModeToGetNumberOfLayers) { } TEST(SvcRateAllocatorTest, CapsAllocationToMaxBitrate) { - VideoCodec codec = Configure(1280, 720, 3, 3, false); + VideoCodec codec = Configure(kVideoCodecVP9, 1280, 720, 3, 3, false); codec.maxBitrate = 70; // Cap the overall max bitrate to 70kbps. SvcRateAllocator allocator = SvcRateAllocator(codec); @@ -376,9 +398,7 @@ TEST(SvcRateAllocatorTest, CapsAllocationToMaxBitrate) { EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0u); } -class SvcRateAllocatorTestParametrizedContentType - : public ::testing::Test, - public ::testing::WithParamInterface { +class SvcRateAllocatorTestParametrizedContentType : public TestWithParam { public: SvcRateAllocatorTestParametrizedContentType() : is_screen_sharing_(GetParam()) {} @@ -387,7 +407,8 @@ class SvcRateAllocatorTestParametrizedContentType }; TEST_P(SvcRateAllocatorTestParametrizedContentType, MaxBitrate) { - VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_); + VideoCodec codec = + Configure(kVideoCodecVP9, 1280, 720, 3, 1, is_screen_sharing_); EXPECT_EQ(SvcRateAllocator::GetMaxBitrate(codec), DataRate::KilobitsPerSec(codec.spatialLayers[0].maxBitrate + codec.spatialLayers[1].maxBitrate + @@ -400,7 +421,8 @@ TEST_P(SvcRateAllocatorTestParametrizedContentType, MaxBitrate) { } TEST_P(SvcRateAllocatorTestParametrizedContentType, PaddingBitrate) { - VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_); + VideoCodec codec = + Configure(kVideoCodecVP9, 1280, 720, 3, 1, is_screen_sharing_); SvcRateAllocator allocator = SvcRateAllocator(codec); DataRate padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec); @@ -449,7 +471,8 @@ TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) { "WebRTC-StableTargetRate/enabled:true,video_hysteresis_factor:1.0," "screenshare_hysteresis_factor:1.0/"); - const VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_); + const VideoCodec codec = + Configure(kVideoCodecVP9, 1280, 720, 3, 1, is_screen_sharing_); const auto start_rates = SvcRateAllocator::GetLayerStartBitrates(codec); const DataRate min_rate_two_layers = start_rates[1]; const DataRate min_rate_three_layers = start_rates[2]; @@ -502,7 +525,8 @@ TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) { TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrateWithHysteresis) { - const VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_); + const VideoCodec codec = + Configure(kVideoCodecVP9, 1280, 720, 3, 1, is_screen_sharing_); const auto start_rates = SvcRateAllocator::GetLayerStartBitrates(codec); const DataRate min_rate_single_layer = start_rates[0]; const DataRate min_rate_two_layers = start_rates[1]; @@ -591,9 +615,39 @@ TEST_P(SvcRateAllocatorTestParametrizedContentType, EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); } +TEST_P(SvcRateAllocatorTestParametrizedContentType, TwoTemporalLayersAv1) { + VideoCodec codec = + Configure(kVideoCodecAV1, 1280, 720, 1, 2, is_screen_sharing_); + SvcRateAllocator allocator = SvcRateAllocator(codec); + VideoBitrateAllocation allocation = + allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate_bps=*/1024'000, /*framerate=*/30)); + + EXPECT_EQ(allocation.GetBitrate(/*spatial_index=*/0, /*temporal_index=*/0), + 660645u); + EXPECT_EQ(allocation.GetBitrate(/*spatial_index=*/0, /*temporal_index=*/1), + 363355u); +} + +TEST_P(SvcRateAllocatorTestParametrizedContentType, ThreeTemporalLayersAv1) { + VideoCodec codec = + Configure(kVideoCodecAV1, 1280, 720, 1, 3, is_screen_sharing_); + SvcRateAllocator allocator = SvcRateAllocator(codec); + VideoBitrateAllocation allocation = + allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate_bps=*/1024'000, /*framerate=*/30)); + + EXPECT_EQ(allocation.GetBitrate(/*spatial_index=*/0, /*temporal_index=*/0), + 552766u); + EXPECT_EQ(allocation.GetBitrate(/*spatial_index=*/0, /*temporal_index=*/1), + 167212u); + EXPECT_EQ(allocation.GetBitrate(/*spatial_index=*/0, /*temporal_index=*/2), + 304022u); +} + INSTANTIATE_TEST_SUITE_P(_, SvcRateAllocatorTestParametrizedContentType, - ::testing::Bool()); + Bool()); } // namespace test } // namespace webrtc