diff --git a/modules/video_coding/codecs/test/test_config.cc b/modules/video_coding/codecs/test/test_config.cc index de6beb9263..0d02ba72f5 100644 --- a/modules/video_coding/codecs/test/test_config.cc +++ b/modules/video_coding/codecs/test/test_config.cc @@ -56,7 +56,7 @@ void ConfigureSvc(VideoCodec* codec_settings) { const std::vector layers = GetSvcConfig(codec_settings->width, codec_settings->height, codec_settings->VP9()->numberOfSpatialLayers, - codec_settings->VP9()->numberOfTemporalLayers); + codec_settings->VP9()->numberOfTemporalLayers, false); for (size_t i = 0; i < layers.size(); ++i) { codec_settings->spatialLayers[i] = layers[i]; diff --git a/modules/video_coding/codecs/vp9/svc_config.cc b/modules/video_coding/codecs/vp9/svc_config.cc index ac6b4e5883..ed60a0d9fd 100644 --- a/modules/video_coding/codecs/vp9/svc_config.cc +++ b/modules/video_coding/codecs/vp9/svc_config.cc @@ -19,18 +19,38 @@ namespace webrtc { namespace { -const int kMinVp9SvcBitrateKbps = 30; // Lowest VP9 video rate in kbps. +const size_t kMinVp9SvcBitrateKbps = 30; + +const size_t kMaxNumLayersForScreenSharing = 2; +const size_t kMaxScreenSharingLayerBitrateKbps[] = {200, 500}; } // namespace -std::vector GetSvcConfig(size_t input_width, - size_t input_height, - size_t num_spatial_layers, - size_t num_temporal_layers) { - RTC_DCHECK_GT(input_width, 0); - RTC_DCHECK_GT(input_height, 0); - RTC_DCHECK_GT(num_spatial_layers, 0); - RTC_DCHECK_GT(num_temporal_layers, 0); +std::vector ConfigureSvcScreenSharing(size_t input_width, + size_t input_height, + size_t num_spatial_layers) { + num_spatial_layers = + std::min(num_spatial_layers, kMaxNumLayersForScreenSharing); + std::vector spatial_layers; + for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { + SpatialLayer spatial_layer = {0}; + spatial_layer.width = input_width; + spatial_layer.height = input_height; + spatial_layer.numberOfTemporalLayers = 1; + spatial_layer.minBitrate = static_cast(kMinVp9SvcBitrateKbps); + spatial_layer.maxBitrate = + static_cast(kMaxScreenSharingLayerBitrateKbps[sl_idx]); + spatial_layer.targetBitrate = spatial_layer.maxBitrate; + spatial_layers.push_back(spatial_layer); + } + + return spatial_layers; +} + +std::vector ConfigureSvcNormalVideo(size_t input_width, + size_t input_height, + size_t num_spatial_layers, + size_t num_temporal_layers) { std::vector spatial_layers; // Limit number of layers for given resolution. @@ -57,7 +77,7 @@ std::vector GetSvcConfig(size_t input_width, // TODO(ssilkin): Add to the comment PSNR/SSIM we get at encoding certain // video to min/max bitrate specified by those formulas. const size_t num_pixels = spatial_layer.width * spatial_layer.height; - const int min_bitrate = + const size_t min_bitrate = static_cast((600. * std::sqrt(num_pixels) - 95000.) / 1000.); spatial_layer.minBitrate = std::max(min_bitrate, kMinVp9SvcBitrateKbps); spatial_layer.maxBitrate = @@ -71,4 +91,23 @@ std::vector GetSvcConfig(size_t input_width, return spatial_layers; } +std::vector GetSvcConfig(size_t input_width, + size_t input_height, + size_t num_spatial_layers, + size_t num_temporal_layers, + bool is_screen_sharing) { + RTC_DCHECK_GT(input_width, 0); + RTC_DCHECK_GT(input_height, 0); + RTC_DCHECK_GT(num_spatial_layers, 0); + RTC_DCHECK_GT(num_temporal_layers, 0); + + if (is_screen_sharing) { + return ConfigureSvcScreenSharing(input_width, input_height, + num_spatial_layers); + } else { + return ConfigureSvcNormalVideo(input_width, input_height, + num_spatial_layers, num_temporal_layers); + } +} + } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/svc_config.h b/modules/video_coding/codecs/vp9/svc_config.h index 78d2bf21f9..c8561a4c9d 100644 --- a/modules/video_coding/codecs/vp9/svc_config.h +++ b/modules/video_coding/codecs/vp9/svc_config.h @@ -19,7 +19,8 @@ namespace webrtc { std::vector GetSvcConfig(size_t input_width, size_t input_height, size_t num_spatial_layers, - size_t num_temporal_layers); + size_t num_temporal_layers, + bool is_screen_sharing); } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/svc_config_unittest.cc b/modules/video_coding/codecs/vp9/svc_config_unittest.cc index ab47a6fdee..ebefddab13 100644 --- a/modules/video_coding/codecs/vp9/svc_config_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_config_unittest.cc @@ -24,7 +24,7 @@ TEST(SvcConfig, NumSpatialLayers) { std::vector spatial_layers = GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1), kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), - max_num_spatial_layers, 1); + max_num_spatial_layers, 1, false); EXPECT_EQ(spatial_layers.size(), num_spatial_layers); } @@ -34,7 +34,7 @@ TEST(SvcConfig, BitrateThresholds) { std::vector spatial_layers = GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1), kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), - num_spatial_layers, 1); + num_spatial_layers, 1, false); EXPECT_EQ(spatial_layers.size(), num_spatial_layers); @@ -44,4 +44,20 @@ TEST(SvcConfig, BitrateThresholds) { EXPECT_LE(layer.targetBitrate, layer.maxBitrate); } } + +TEST(SvcConfig, ScreenSharing) { + std::vector spatial_layers = + GetSvcConfig(1920, 1080, 3, 3, true); + + EXPECT_EQ(spatial_layers.size(), 2UL); + + for (const SpatialLayer& layer : spatial_layers) { + EXPECT_EQ(layer.width, 1920); + EXPECT_EQ(layer.height, 1080); + EXPECT_EQ(layer.numberOfTemporalLayers, 1); + EXPECT_LE(layer.minBitrate, layer.maxBitrate); + EXPECT_LE(layer.minBitrate, layer.targetBitrate); + EXPECT_LE(layer.targetBitrate, layer.maxBitrate); + } +} } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc index 46caef42d9..61505c0883 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc @@ -30,72 +30,49 @@ SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec) : codec_(codec) { VideoBitrateAllocation SvcRateAllocator::GetAllocation( uint32_t total_bitrate_bps, uint32_t framerate_fps) { - VideoBitrateAllocation bitrate_allocation; + if (codec_.maxBitrate != 0) { + total_bitrate_bps = std::min(total_bitrate_bps, codec_.maxBitrate * 1000); + } + if (codec_.spatialLayers[0].targetBitrate == 0) { + // Delegate rate distribution to VP9 encoder wrapper if bitrate thresholds + // are not initialized. + VideoBitrateAllocation bitrate_allocation; + bitrate_allocation.SetBitrate(0, 0, total_bitrate_bps); + return bitrate_allocation; + } else if (codec_.mode == kRealtimeVideo) { + return GetAllocationNormalVideo(total_bitrate_bps); + } else { + return GetAllocationScreenSharing(total_bitrate_bps); + } +} + +VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( + uint32_t total_bitrate_bps) const { size_t num_spatial_layers = codec_.VP9().numberOfSpatialLayers; RTC_CHECK(num_spatial_layers > 0); size_t num_temporal_layers = codec_.VP9().numberOfTemporalLayers; RTC_CHECK(num_temporal_layers > 0); - if (codec_.maxBitrate != 0) { - total_bitrate_bps = std::min(total_bitrate_bps, codec_.maxBitrate * 1000); - } - - if (codec_.mode == kScreensharing) { - // At screen sharing bitrate allocation is handled by VP9 encoder wrapper. - bitrate_allocation.SetBitrate(0, 0, total_bitrate_bps); - return bitrate_allocation; - } - std::vector spatial_layer_bitrate_bps; - if (codec_.spatialLayers[0].maxBitrate == 0) { - // Layers' parameters are not initialized. Do simple split. + // Distribute total bitrate across spatial layers. If there is not enough + // bitrate to provide all layers with at least minimum required bitrate + // then number of layers is reduced by one and distribution is repeated + // until that condition is met or if number of layers is reduced to one. + for (;; --num_spatial_layers) { spatial_layer_bitrate_bps = SplitBitrate(num_spatial_layers, total_bitrate_bps, kSpatialLayeringRateScalingFactor); - } else { - // Distribute total bitrate across spatial layers. If there is not enough - // bitrate to provide all layers with at least minimum required bitrate - // then number of layers is reduced by one and distribution is repeated - // until that condition is met or if number of layers is reduced to one. - for (;; --num_spatial_layers) { - spatial_layer_bitrate_bps = - SplitBitrate(num_spatial_layers, total_bitrate_bps, - kSpatialLayeringRateScalingFactor); - bool enough_bitrate = true; - size_t excess_rate = 0; - for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - RTC_DCHECK_GT(codec_.spatialLayers[sl_idx].maxBitrate, 0); - RTC_DCHECK_GE(codec_.spatialLayers[sl_idx].maxBitrate, - codec_.spatialLayers[sl_idx].minBitrate); - - const size_t min_bitrate_bps = - codec_.spatialLayers[sl_idx].minBitrate * 1000; - const size_t max_bitrate_bps = - codec_.spatialLayers[sl_idx].maxBitrate * 1000; - - spatial_layer_bitrate_bps[sl_idx] += excess_rate; - if (spatial_layer_bitrate_bps[sl_idx] < max_bitrate_bps) { - excess_rate = 0; - } else { - excess_rate = spatial_layer_bitrate_bps[sl_idx] - max_bitrate_bps; - spatial_layer_bitrate_bps[sl_idx] = max_bitrate_bps; - } - - if (spatial_layer_bitrate_bps[sl_idx] < min_bitrate_bps) { - enough_bitrate = false; - break; - } - } - - if (enough_bitrate || num_spatial_layers == 1) { - break; - } + const bool enough_bitrate = AdjustAndVerify(&spatial_layer_bitrate_bps); + if (enough_bitrate || num_spatial_layers == 1) { + break; } } + VideoBitrateAllocation bitrate_allocation; + for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { std::vector temporal_layer_bitrate_bps = SplitBitrate(num_temporal_layers, spatial_layer_bitrate_bps[sl_idx], @@ -125,13 +102,72 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocation( return bitrate_allocation; } +bool SvcRateAllocator::AdjustAndVerify( + std::vector* spatial_layer_bitrate_bps) const { + bool enough_bitrate = true; + size_t excess_rate = 0; + for (size_t sl_idx = 0; + sl_idx < spatial_layer_bitrate_bps->size() && enough_bitrate; ++sl_idx) { + RTC_DCHECK_GT(codec_.spatialLayers[sl_idx].maxBitrate, 0); + RTC_DCHECK_GE(codec_.spatialLayers[sl_idx].maxBitrate, + codec_.spatialLayers[sl_idx].minBitrate); + + const size_t min_bitrate_bps = + codec_.spatialLayers[sl_idx].minBitrate * 1000; + const size_t max_bitrate_bps = + codec_.spatialLayers[sl_idx].maxBitrate * 1000; + + spatial_layer_bitrate_bps->at(sl_idx) += excess_rate; + if (spatial_layer_bitrate_bps->at(sl_idx) < max_bitrate_bps) { + excess_rate = 0; + } else { + excess_rate = spatial_layer_bitrate_bps->at(sl_idx) - max_bitrate_bps; + spatial_layer_bitrate_bps->at(sl_idx) = max_bitrate_bps; + } + + enough_bitrate = (spatial_layer_bitrate_bps->at(sl_idx) >= min_bitrate_bps); + } + + return enough_bitrate; +} + +VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( + uint32_t total_bitrate_bps) const { + const size_t num_spatial_layers = codec_.VP9().numberOfSpatialLayers; + RTC_CHECK(num_spatial_layers > 0); + RTC_CHECK_EQ(codec_.VP9().numberOfTemporalLayers, 1U); + + VideoBitrateAllocation bitrate_allocation; + + // Add next layer after bitrate of previous layer has reached its maximum. + size_t left_bitrate_bps = total_bitrate_bps; + for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { + const size_t min_bitrate_bps = + codec_.spatialLayers[sl_idx].minBitrate * 1000; + const size_t max_bitrate_bps = + codec_.spatialLayers[sl_idx].maxBitrate * 1000; + + const size_t bitrate_bps = std::min(left_bitrate_bps, max_bitrate_bps); + if (bitrate_bps >= min_bitrate_bps) { + bitrate_allocation.SetBitrate(sl_idx, 0, bitrate_bps); + } else { + break; + } + + left_bitrate_bps -= bitrate_bps; + } + + return bitrate_allocation; +} + uint32_t SvcRateAllocator::GetPreferredBitrateBps(uint32_t framerate) { return GetAllocation(codec_.maxBitrate * 1000, framerate).get_sum_bps(); } -std::vector SvcRateAllocator::SplitBitrate(size_t num_layers, - size_t total_bitrate, - float rate_scaling_factor) { +std::vector SvcRateAllocator::SplitBitrate( + size_t num_layers, + size_t total_bitrate, + float rate_scaling_factor) const { std::vector bitrates; double denominator = 0.0; diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.h b/modules/video_coding/codecs/vp9/svc_rate_allocator.h index cbf2096d90..268f18d37a 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.h +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.h @@ -28,9 +28,14 @@ class SvcRateAllocator : public VideoBitrateAllocator { uint32_t GetPreferredBitrateBps(uint32_t framerate_fps) override; private: + VideoBitrateAllocation GetAllocationNormalVideo( + uint32_t total_bitrate_bps) const; + VideoBitrateAllocation GetAllocationScreenSharing( + uint32_t total_bitrate_bps) const; std::vector SplitBitrate(size_t num_layers, size_t total_bitrate, - float rate_scaling_factor); + float rate_scaling_factor) const; + bool AdjustAndVerify(std::vector* spatial_layer_bitrate_bps) const; const VideoCodec codec_; }; diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc index 536950e18e..7058baf760 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc @@ -19,14 +19,17 @@ namespace { static VideoCodec Configure(size_t width, size_t height, size_t num_spatial_layers, - size_t num_temporal_layers) { + size_t num_temporal_layers, + bool is_screen_sharing) { VideoCodec codec; codec.width = width; codec.height = height; codec.codecType = kVideoCodecVP9; + codec.mode = is_screen_sharing ? kScreensharing : kRealtimeVideo; std::vector spatial_layers = - GetSvcConfig(width, height, num_spatial_layers, num_temporal_layers); + GetSvcConfig(width, height, num_spatial_layers, num_temporal_layers, + is_screen_sharing); RTC_CHECK_LE(spatial_layers.size(), kMaxSpatialLayers); codec.VP9()->numberOfSpatialLayers = @@ -43,7 +46,7 @@ static VideoCodec Configure(size_t width, } // namespace TEST(SvcRateAllocatorTest, SingleLayerFor320x180Input) { - VideoCodec codec = Configure(320, 180, 3, 3); + VideoCodec codec = Configure(320, 180, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); VideoBitrateAllocation allocation = allocator.GetAllocation(1000 * 1000, 30); @@ -53,7 +56,7 @@ TEST(SvcRateAllocatorTest, SingleLayerFor320x180Input) { } TEST(SvcRateAllocatorTest, TwoLayersFor640x360Input) { - VideoCodec codec = Configure(640, 360, 3, 3); + VideoCodec codec = Configure(640, 360, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); VideoBitrateAllocation allocation = allocator.GetAllocation(1000 * 1000, 30); @@ -64,7 +67,7 @@ TEST(SvcRateAllocatorTest, TwoLayersFor640x360Input) { } TEST(SvcRateAllocatorTest, ThreeLayersFor1280x720Input) { - VideoCodec codec = Configure(1280, 720, 3, 3); + VideoCodec codec = Configure(1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); VideoBitrateAllocation allocation = allocator.GetAllocation(1000 * 1000, 30); @@ -76,7 +79,7 @@ TEST(SvcRateAllocatorTest, ThreeLayersFor1280x720Input) { TEST(SvcRateAllocatorTest, BaseLayerNonZeroBitrateEvenIfTotalIfLessThanMinimum) { - VideoCodec codec = Configure(1280, 720, 3, 3); + VideoCodec codec = Configure(1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -90,7 +93,7 @@ TEST(SvcRateAllocatorTest, } TEST(SvcRateAllocatorTest, Disable640x360Layer) { - VideoCodec codec = Configure(1280, 720, 3, 3); + VideoCodec codec = Configure(1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -106,7 +109,7 @@ TEST(SvcRateAllocatorTest, Disable640x360Layer) { } TEST(SvcRateAllocatorTest, Disable1280x720Layer) { - VideoCodec codec = Configure(1280, 720, 3, 3); + VideoCodec codec = Configure(1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -123,7 +126,7 @@ TEST(SvcRateAllocatorTest, Disable1280x720Layer) { } TEST(SvcRateAllocatorTest, BitrateIsCapped) { - VideoCodec codec = Configure(1280, 720, 3, 3); + VideoCodec codec = Configure(1280, 720, 3, 3, false); SvcRateAllocator allocator = SvcRateAllocator(codec); const SpatialLayer* layers = codec.spatialLayers; @@ -139,4 +142,23 @@ TEST(SvcRateAllocatorTest, BitrateIsCapped) { EXPECT_EQ(allocation.GetSpatialLayerSum(2) / 1000, layers[2].maxBitrate); } +TEST(SvcRateAllocatorTest, MinBitrateToGetQualityLayer) { + VideoCodec codec = Configure(1280, 720, 3, 1, true); + SvcRateAllocator allocator = SvcRateAllocator(codec); + + const SpatialLayer* layers = codec.spatialLayers; + + EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 2U); + + VideoBitrateAllocation allocation = + allocator.GetAllocation(layers[0].minBitrate * 1000, 30); + EXPECT_EQ(allocation.GetSpatialLayerSum(0) / 1000, layers[0].minBitrate); + EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL); + + allocation = allocator.GetAllocation( + (layers[0].maxBitrate + layers[1].minBitrate) * 1000, 30); + EXPECT_EQ(allocation.GetSpatialLayerSum(0) / 1000, layers[0].maxBitrate); + EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].minBitrate); +} + } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index 6fa383b3f1..9b1a016883 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -53,8 +53,9 @@ class TestVp9Impl : public VideoCodecUnitTest { codec_settings_.VP9()->numberOfTemporalLayers = 1; codec_settings_.VP9()->frameDroppingOn = false; - std::vector layers = GetSvcConfig( - codec_settings_.width, codec_settings_.height, num_spatial_layers, 1); + std::vector layers = + GetSvcConfig(codec_settings_.width, codec_settings_.height, + num_spatial_layers, 1, false); for (size_t i = 0; i < layers.size(); ++i) { codec_settings_.spatialLayers[i] = layers[i]; } diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc index 192f80fc3e..adc7f32ba9 100644 --- a/modules/video_coding/video_codec_initializer.cc +++ b/modules/video_coding/video_codec_initializer.cc @@ -214,7 +214,7 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( spatial_layers = GetSvcConfig(video_codec.width, video_codec.height, video_codec.VP9()->numberOfSpatialLayers, - video_codec.VP9()->numberOfTemporalLayers); + video_codec.VP9()->numberOfTemporalLayers, false); const bool no_spatial_layering = (spatial_layers.size() == 1); if (no_spatial_layering) {