diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index f81387f890..8a61224ccb 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -512,7 +512,6 @@ rtc_library("webrtc_vp9_helpers") { sources = [ "codecs/vp9/svc_config.cc", "codecs/vp9/svc_config.h", - "codecs/vp9/svc_rate_allocator.cc", "codecs/vp9/svc_rate_allocator.h", ] @@ -527,6 +526,7 @@ rtc_library("webrtc_vp9_helpers") { "../../rtc_base:checks", "../../rtc_base:logging", "../../rtc_base/experiments:stable_target_rate_experiment", + "svc:svc_rate_allocator", ] absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } @@ -900,7 +900,6 @@ if (rtc_include_tests) { "codecs/vp8/libvpx_vp8_simulcast_test.cc", "codecs/vp8/screenshare_layers_unittest.cc", "codecs/vp9/svc_config_unittest.cc", - "codecs/vp9/svc_rate_allocator_unittest.cc", "decoding_state_unittest.cc", "fec_controller_unittest.cc", "frame_buffer2_unittest.cc", @@ -1009,6 +1008,7 @@ if (rtc_include_tests) { "codecs/av1:video_coding_codecs_av1_tests", "deprecated:nack_module", "svc:scalability_structure_tests", + "svc:svc_rate_allocator_tests", ] absl_deps = [ "//third_party/abseil-cpp/absl/memory", diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.h b/modules/video_coding/codecs/vp9/svc_rate_allocator.h index a4e0c28cc0..fa53a155ab 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.h +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.h @@ -11,52 +11,7 @@ #ifndef MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_ #define MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_ -#include -#include - -#include "absl/container/inlined_vector.h" -#include "api/video/video_bitrate_allocation.h" -#include "api/video/video_bitrate_allocator.h" -#include "api/video/video_codec_constants.h" -#include "api/video_codecs/video_codec.h" -#include "rtc_base/experiments/stable_target_rate_experiment.h" - -namespace webrtc { - -class SvcRateAllocator : public VideoBitrateAllocator { - public: - explicit SvcRateAllocator(const VideoCodec& codec); - - VideoBitrateAllocation Allocate( - VideoBitrateAllocationParameters parameters) override; - - static DataRate GetMaxBitrate(const VideoCodec& codec); - static DataRate GetPaddingBitrate(const VideoCodec& codec); - static absl::InlinedVector GetLayerStartBitrates( - const VideoCodec& codec); - - private: - VideoBitrateAllocation GetAllocationNormalVideo( - DataRate total_bitrate, - size_t first_active_layer, - size_t num_spatial_layers) const; - - VideoBitrateAllocation GetAllocationScreenSharing( - DataRate total_bitrate, - size_t first_active_layer, - size_t num_spatial_layers) const; - - // Returns the number of layers that are active and have enough bitrate to - // actually be enabled. - size_t FindNumEnabledLayers(DataRate target_rate) const; - - const VideoCodec codec_; - const StableTargetRateExperiment experiment_settings_; - const absl::InlinedVector - cumulative_layer_start_bitrates_; - size_t last_active_layer_count_; -}; - -} // namespace webrtc +// TODO(danilchap): Update dependent includes and remove this forwarding header. +#include "modules/video_coding/svc/svc_rate_allocator.h" #endif // MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_ diff --git a/modules/video_coding/svc/BUILD.gn b/modules/video_coding/svc/BUILD.gn index 20f57cc60f..3e93b897b4 100644 --- a/modules/video_coding/svc/BUILD.gn +++ b/modules/video_coding/svc/BUILD.gn @@ -68,8 +68,25 @@ rtc_source_set("scalability_structures") { ] } +rtc_source_set("svc_rate_allocator") { + sources = [ + "svc_rate_allocator.cc", + "svc_rate_allocator.h", + ] + deps = [ + ":scalability_structures", + "../../../api/video:video_bitrate_allocation", + "../../../api/video:video_bitrate_allocator", + "../../../api/video:video_codec_constants", + "../../../api/video_codecs:video_codecs_api", + "../../../rtc_base:checks", + "../../../rtc_base/experiments:stable_target_rate_experiment", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] +} + if (rtc_include_tests) { - rtc_library("scalability_structure_tests") { + rtc_source_set("scalability_structure_tests") { testonly = true sources = [ "scalability_structure_key_svc_unittest.cc", @@ -93,4 +110,16 @@ if (rtc_include_tests) { ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } + + rtc_source_set("svc_rate_allocator_tests") { + testonly = true + sources = [ "svc_rate_allocator_unittest.cc" ] + deps = [ + ":svc_rate_allocator", + "..:webrtc_vp9_helpers", + "../../../rtc_base:checks", + "../../../test:field_trial", + "../../../test:test_support", + ] + } } diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/svc/svc_rate_allocator.cc similarity index 82% rename from modules/video_coding/codecs/vp9/svc_rate_allocator.cc rename to modules/video_coding/svc/svc_rate_allocator.cc index 25bca63c0e..a51bdb05dd 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc +++ b/modules/video_coding/svc/svc_rate_allocator.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" +#include "modules/video_coding/svc/svc_rate_allocator.h" #include #include @@ -17,40 +17,38 @@ #include #include "absl/container/inlined_vector.h" +#include "modules/video_coding/svc/create_scalability_structure.h" #include "rtc_base/checks.h" namespace webrtc { namespace { -const float kSpatialLayeringRateScalingFactor = 0.55f; -const float kTemporalLayeringRateScalingFactor = 0.55f; +constexpr float kSpatialLayeringRateScalingFactor = 0.55f; +constexpr float kTemporalLayeringRateScalingFactor = 0.55f; -// Returns numberOfSpatialLayers if no layers are active. -size_t GetFirstActiveLayer(const VideoCodec& codec) { - RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9); - RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u); - size_t layer = 0; - for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) { - if (codec.spatialLayers[layer].active) { +struct ActiveSpatialLayers { + size_t first = 0; + size_t num = 0; +}; + +ActiveSpatialLayers GetActiveSpatialLayers(const VideoCodec& codec, + size_t num_spatial_layers) { + ActiveSpatialLayers active; + for (active.first = 0; active.first < num_spatial_layers; ++active.first) { + if (codec.spatialLayers[active.first].active) { break; } } - return layer; -} -static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) { - RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9); - RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u); - - const size_t first_active_layer = GetFirstActiveLayer(codec); - size_t last_active_layer = first_active_layer; - for (; last_active_layer < codec.VP9().numberOfSpatialLayers; - ++last_active_layer) { + size_t last_active_layer = active.first; + for (; last_active_layer < num_spatial_layers; ++last_active_layer) { if (!codec.spatialLayers[last_active_layer].active) { break; } } - return last_active_layer - first_active_layer; + active.num = last_active_layer - active.first; + + return active; } std::vector AdjustAndVerify( @@ -173,16 +171,39 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec, } // namespace +SvcRateAllocator::NumLayers SvcRateAllocator::GetNumLayers( + const VideoCodec& codec) { + NumLayers layers; + if (!codec.ScalabilityMode().empty()) { + if (auto structure = CreateScalabilityStructure(codec.ScalabilityMode())) { + ScalableVideoController::StreamLayersConfig config = + structure->StreamConfig(); + layers.spatial = config.num_spatial_layers; + layers.temporal = config.num_temporal_layers; + return layers; + } + } + if (codec.codecType == kVideoCodecVP9) { + layers.spatial = codec.VP9().numberOfSpatialLayers; + layers.temporal = codec.VP9().numberOfTemporalLayers; + return layers; + } + layers.spatial = 1; + layers.temporal = 1; + return layers; +} + SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec) : codec_(codec), + num_layers_(GetNumLayers(codec)), experiment_settings_(StableTargetRateExperiment::ParseFromFieldTrials()), cumulative_layer_start_bitrates_(GetLayerStartBitrates(codec)), last_active_layer_count_(0) { - RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9); - RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u); - RTC_DCHECK_GT(codec.VP9().numberOfTemporalLayers, 0u); - for (size_t layer_idx = 0; layer_idx < codec.VP9().numberOfSpatialLayers; - ++layer_idx) { + RTC_DCHECK_GT(num_layers_.spatial, 0); + RTC_DCHECK_LE(num_layers_.spatial, kMaxSpatialLayers); + RTC_DCHECK_GT(num_layers_.temporal, 0); + RTC_DCHECK_LE(num_layers_.temporal, 3); + for (size_t layer_idx = 0; layer_idx < num_layers_.spatial; ++layer_idx) { // Verify min <= target <= max. if (codec.spatialLayers[layer_idx].active) { RTC_DCHECK_GT(codec.spatialLayers[layer_idx].maxBitrate, 0); @@ -205,16 +226,16 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( } if (codec_.spatialLayers[0].targetBitrate == 0) { - // Delegate rate distribution to VP9 encoder wrapper if bitrate thresholds + // Delegate rate distribution to encoder wrapper if bitrate thresholds // are not set. VideoBitrateAllocation bitrate_allocation; bitrate_allocation.SetBitrate(0, 0, total_bitrate.bps()); return bitrate_allocation; } - const size_t first_active_layer = GetFirstActiveLayer(codec_); - const size_t num_active_layers = GetNumActiveSpatialLayers(codec_); - size_t num_spatial_layers = num_active_layers; + const ActiveSpatialLayers active_layers = + GetActiveSpatialLayers(codec_, num_layers_.spatial); + size_t num_spatial_layers = active_layers.num; if (num_spatial_layers == 0) { return VideoBitrateAllocation(); // All layers are deactivated. @@ -249,13 +270,13 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( VideoBitrateAllocation allocation; if (codec_.mode == VideoCodecMode::kRealtimeVideo) { - allocation = GetAllocationNormalVideo(total_bitrate, first_active_layer, + allocation = GetAllocationNormalVideo(total_bitrate, active_layers.first, num_spatial_layers); } else { - allocation = GetAllocationScreenSharing(total_bitrate, first_active_layer, + allocation = GetAllocationScreenSharing(total_bitrate, active_layers.first, num_spatial_layers); } - allocation.set_bw_limited(num_spatial_layers < num_active_layers); + allocation.set_bw_limited(num_spatial_layers < active_layers.num); return allocation; } @@ -279,25 +300,24 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( VideoBitrateAllocation bitrate_allocation; - const size_t num_temporal_layers = codec_.VP9().numberOfTemporalLayers; for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { std::vector temporal_layer_rates = - SplitBitrate(num_temporal_layers, spatial_layer_rates[sl_idx], + SplitBitrate(num_layers_.temporal, spatial_layer_rates[sl_idx], kTemporalLayeringRateScalingFactor); // Distribute rate across temporal layers. Allocate more bits to lower // layers since they are used for prediction of higher layers and their // references are far apart. - if (num_temporal_layers == 1) { + if (num_layers_.temporal == 1) { bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, temporal_layer_rates[0].bps()); - } else if (num_temporal_layers == 2) { + } else if (num_layers_.temporal == 2) { bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0, temporal_layer_rates[1].bps()); bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1, temporal_layer_rates[0].bps()); } else { - RTC_CHECK_EQ(num_temporal_layers, 3); + RTC_CHECK_EQ(num_layers_.temporal, 3); // In case of three temporal layers the high layer has two frames and the // middle layer has one frame within GOP (in between two consecutive low // layer frames). Thus high layer requires more bits (comparing pure @@ -383,13 +403,14 @@ size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const { } DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) { - const size_t first_active_layer = GetFirstActiveLayer(codec); - const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec); + const NumLayers num_layers = GetNumLayers(codec); + const ActiveSpatialLayers active_layers = + GetActiveSpatialLayers(codec, num_layers.spatial); DataRate max_bitrate = DataRate::Zero(); - for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { + for (size_t sl_idx = 0; sl_idx < active_layers.num; ++sl_idx) { max_bitrate += DataRate::KilobitsPerSec( - codec.spatialLayers[first_active_layer + sl_idx].maxBitrate); + codec.spatialLayers[active_layers.first + sl_idx].maxBitrate); } if (codec.maxBitrate != 0) { @@ -412,12 +433,13 @@ DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) { absl::InlinedVector SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) { absl::InlinedVector start_bitrates; - const size_t first_active_layer = GetFirstActiveLayer(codec); - const size_t num_layers = GetNumActiveSpatialLayers(codec); + const NumLayers num_layers = GetNumLayers(codec); + const ActiveSpatialLayers active_layers = + GetActiveSpatialLayers(codec, num_layers.spatial); DataRate last_rate = DataRate::Zero(); - for (size_t i = 1; i <= num_layers; ++i) { + for (size_t i = 1; i <= active_layers.num; ++i) { DataRate layer_toggling_rate = - FindLayerTogglingThreshold(codec, first_active_layer, i); + FindLayerTogglingThreshold(codec, active_layers.first, i); start_bitrates.push_back(layer_toggling_rate); RTC_DCHECK_LE(last_rate, layer_toggling_rate); last_rate = layer_toggling_rate; diff --git a/modules/video_coding/svc/svc_rate_allocator.h b/modules/video_coding/svc/svc_rate_allocator.h new file mode 100644 index 0000000000..bd75fca284 --- /dev/null +++ b/modules/video_coding/svc/svc_rate_allocator.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_SVC_SVC_RATE_ALLOCATOR_H_ +#define MODULES_VIDEO_CODING_SVC_SVC_RATE_ALLOCATOR_H_ + +#include +#include + +#include "absl/container/inlined_vector.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_bitrate_allocator.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/video_codec.h" +#include "rtc_base/experiments/stable_target_rate_experiment.h" + +namespace webrtc { + +class SvcRateAllocator : public VideoBitrateAllocator { + public: + explicit SvcRateAllocator(const VideoCodec& codec); + + VideoBitrateAllocation Allocate( + VideoBitrateAllocationParameters parameters) override; + + static DataRate GetMaxBitrate(const VideoCodec& codec); + static DataRate GetPaddingBitrate(const VideoCodec& codec); + static absl::InlinedVector GetLayerStartBitrates( + const VideoCodec& codec); + + private: + struct NumLayers { + size_t spatial = 1; + size_t temporal = 1; + }; + + static NumLayers GetNumLayers(const VideoCodec& codec); + VideoBitrateAllocation GetAllocationNormalVideo( + DataRate total_bitrate, + size_t first_active_layer, + size_t num_spatial_layers) const; + + VideoBitrateAllocation GetAllocationScreenSharing( + DataRate total_bitrate, + size_t first_active_layer, + size_t num_spatial_layers) const; + + // Returns the number of layers that are active and have enough bitrate to + // actually be enabled. + size_t FindNumEnabledLayers(DataRate target_rate) const; + + const VideoCodec codec_; + const NumLayers num_layers_; + const StableTargetRateExperiment experiment_settings_; + const absl::InlinedVector + cumulative_layer_start_bitrates_; + size_t last_active_layer_count_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_SVC_SVC_RATE_ALLOCATOR_H_ diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/svc/svc_rate_allocator_unittest.cc similarity index 85% rename from modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc rename to modules/video_coding/svc/svc_rate_allocator_unittest.cc index daa0c52e09..fd22acd85d 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc +++ b/modules/video_coding/svc/svc_rate_allocator_unittest.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" +#include "modules/video_coding/svc/svc_rate_allocator.h" #include #include @@ -270,6 +270,97 @@ TEST(SvcRateAllocatorTest, FindLayerTogglingThreshold) { EXPECT_EQ(layer_start_bitrates[2], kThreeLayerMinRate); } +TEST(SvcRateAllocatorTest, SupportsAv1) { + VideoCodec codec; + codec.width = 640; + codec.height = 360; + codec.codecType = kVideoCodecAV1; + codec.SetScalabilityMode("L3T3"); + codec.spatialLayers[0].active = true; + codec.spatialLayers[0].minBitrate = 30; + codec.spatialLayers[0].targetBitrate = 51; + codec.spatialLayers[0].maxBitrate = 73; + codec.spatialLayers[1].active = true; + codec.spatialLayers[1].minBitrate = 49; + codec.spatialLayers[1].targetBitrate = 64; + codec.spatialLayers[1].maxBitrate = 97; + codec.spatialLayers[2].active = true; + codec.spatialLayers[2].minBitrate = 193; + codec.spatialLayers[2].targetBitrate = 305; + codec.spatialLayers[2].maxBitrate = 418; + + SvcRateAllocator allocator(codec); + + VideoBitrateAllocation allocation = + allocator.Allocate(VideoBitrateAllocationParameters(1'000'000, 30)); + + EXPECT_GT(allocation.GetSpatialLayerSum(0), 0u); + EXPECT_GT(allocation.GetSpatialLayerSum(1), 0u); + EXPECT_GT(allocation.GetSpatialLayerSum(2), 0u); +} + +TEST(SvcRateAllocatorTest, SupportsAv1WithSkippedLayer) { + VideoCodec codec; + codec.width = 640; + codec.height = 360; + codec.codecType = kVideoCodecAV1; + codec.SetScalabilityMode("L3T3"); + codec.spatialLayers[0].active = false; + codec.spatialLayers[0].minBitrate = 30; + codec.spatialLayers[0].targetBitrate = 51; + codec.spatialLayers[0].maxBitrate = 73; + codec.spatialLayers[1].active = true; + codec.spatialLayers[1].minBitrate = 49; + codec.spatialLayers[1].targetBitrate = 64; + codec.spatialLayers[1].maxBitrate = 97; + codec.spatialLayers[2].active = true; + codec.spatialLayers[2].minBitrate = 193; + codec.spatialLayers[2].targetBitrate = 305; + codec.spatialLayers[2].maxBitrate = 418; + + SvcRateAllocator allocator(codec); + + VideoBitrateAllocation allocation = + allocator.Allocate(VideoBitrateAllocationParameters(1'000'000, 30)); + + EXPECT_EQ(allocation.GetSpatialLayerSum(0), 0u); + EXPECT_GT(allocation.GetSpatialLayerSum(1), 0u); + EXPECT_GT(allocation.GetSpatialLayerSum(2), 0u); +} + +TEST(SvcRateAllocatorTest, UsesScalabilityModeToGetNumberOfLayers) { + VideoCodec codec; + codec.width = 640; + codec.height = 360; + codec.codecType = kVideoCodecAV1; + codec.SetScalabilityMode("L2T2"); + codec.spatialLayers[0].active = true; + codec.spatialLayers[0].minBitrate = 30; + codec.spatialLayers[0].targetBitrate = 51; + codec.spatialLayers[0].maxBitrate = 73; + codec.spatialLayers[1].active = true; + codec.spatialLayers[1].minBitrate = 49; + codec.spatialLayers[1].targetBitrate = 64; + codec.spatialLayers[1].maxBitrate = 97; + codec.spatialLayers[2].active = true; + codec.spatialLayers[2].minBitrate = 193; + codec.spatialLayers[2].targetBitrate = 305; + codec.spatialLayers[2].maxBitrate = 418; + + SvcRateAllocator allocator(codec); + VideoBitrateAllocation allocation = + allocator.Allocate(VideoBitrateAllocationParameters(1'000'000, 30)); + + // Expect bitrates for 2 temporal layers. + EXPECT_TRUE(allocation.HasBitrate(1, /*temporal_index=*/0)); + EXPECT_TRUE(allocation.HasBitrate(1, /*temporal_index=*/1)); + EXPECT_FALSE(allocation.HasBitrate(1, /*temporal_index=*/2)); + + // expect codec.spatialLayers[2].active is ignored because scability mode uses + // just 2 spatial layers. + EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0u); +} + class SvcRateAllocatorTestParametrizedContentType : public ::testing::Test, public ::testing::WithParamInterface {