From 3b15e46a4c42cda999d4a10ae57b2d751671301f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Wed, 3 Jul 2024 08:52:13 +0000 Subject: [PATCH] Get min bitrate from spatial layers for AV1 (instead of bitrate limits). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bitrate limits should have been applied to the spatial layers in ApplySpatialLayerBitrateLimits (and usage is restricted to a single active stream/layer). Bug: b/299588022 Change-Id: Iaae4ece28b8a95eea7d4bacba292847ba5b4000b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/355841 Reviewed-by: Erik Språng Commit-Queue: Åsa Persson Cr-Commit-Position: refs/heads/main@{#42588} --- .../video_coding/video_codec_initializer.cc | 3 +- video/encoder_bitrate_adjuster.cc | 28 +++--- video/encoder_bitrate_adjuster.h | 3 - video/encoder_bitrate_adjuster_unittest.cc | 98 +++++++++++-------- 4 files changed, 72 insertions(+), 60 deletions(-) diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc index 709c57c8fa..2c6a255355 100644 --- a/modules/video_coding/video_codec_initializer.cc +++ b/modules/video_coding/video_codec_initializer.cc @@ -345,7 +345,8 @@ VideoCodec VideoCodecInitializer::SetupCodec( rtc::saturated_cast(experimental_min_bitrate->kbps()); video_codec.minBitrate = experimental_min_bitrate_kbps; video_codec.simulcastStream[0].minBitrate = experimental_min_bitrate_kbps; - if (video_codec.codecType == kVideoCodecVP9) { + if (video_codec.codecType == kVideoCodecVP9 || + video_codec.codecType == kVideoCodecAV1) { video_codec.spatialLayers[0].minBitrate = experimental_min_bitrate_kbps; } } diff --git a/video/encoder_bitrate_adjuster.cc b/video/encoder_bitrate_adjuster.cc index 0932607ac2..0dbcb075bd 100644 --- a/video/encoder_bitrate_adjuster.cc +++ b/video/encoder_bitrate_adjuster.cc @@ -15,6 +15,7 @@ #include #include "api/field_trials_view.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "rtc_base/experiments/rate_control_settings.h" #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" @@ -51,21 +52,30 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster( .BitrateAdjusterCanUseNetworkHeadroom()), frames_since_layout_change_(0), min_bitrates_bps_{}, - frame_size_pixels_{}, codec_(codec_settings.codecType), codec_mode_(codec_settings.mode) { // TODO(https://crbug.com/webrtc/14891): If we want to support simulcast of // SVC streams, EncoderBitrateAdjuster needs to be updated to care about both // `simulcastStream` and `spatialLayers` at the same time. - if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9 && - codec_settings.numberOfSimulcastStreams <= 1) { + if (codec_settings.codecType == VideoCodecType::kVideoCodecAV1 && + codec_settings.numberOfSimulcastStreams <= 1 && + codec_settings.GetScalabilityMode().has_value()) { + for (int si = 0; si < ScalabilityModeToNumSpatialLayers( + *(codec_settings.GetScalabilityMode())); + ++si) { + if (codec_settings.spatialLayers[si].active) { + min_bitrates_bps_[si] = + std::max(codec_settings.minBitrate * 1000, + codec_settings.spatialLayers[si].minBitrate * 1000); + } + } + } else if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9 && + codec_settings.numberOfSimulcastStreams <= 1) { for (size_t si = 0; si < codec_settings.VP9().numberOfSpatialLayers; ++si) { if (codec_settings.spatialLayers[si].active) { min_bitrates_bps_[si] = std::max(codec_settings.minBitrate * 1000, codec_settings.spatialLayers[si].minBitrate * 1000); - frame_size_pixels_[si] = codec_settings.spatialLayers[si].width * - codec_settings.spatialLayers[si].height; } } } else { @@ -74,8 +84,6 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster( min_bitrates_bps_[si] = std::max(codec_settings.minBitrate * 1000, codec_settings.simulcastStream[si].minBitrate * 1000); - frame_size_pixels_[si] = codec_settings.spatialLayers[si].width * - codec_settings.spatialLayers[si].height; } } } @@ -322,12 +330,6 @@ void EncoderBitrateAdjuster::OnEncoderInfo( // Copy allocation into current state and re-allocate. for (size_t si = 0; si < kMaxSpatialLayers; ++si) { current_fps_allocation_[si] = encoder_info.fps_allocation[si]; - if (frame_size_pixels_[si] > 0) { - if (auto bwlimit = encoder_info.GetEncoderBitrateLimitsForResolution( - frame_size_pixels_[si])) { - min_bitrates_bps_[si] = bwlimit->min_bitrate_bps; - } - } } // Trigger re-allocation so that overshoot detectors have correct targets. diff --git a/video/encoder_bitrate_adjuster.h b/video/encoder_bitrate_adjuster.h index 0ddd20b9b3..7150bee668 100644 --- a/video/encoder_bitrate_adjuster.h +++ b/video/encoder_bitrate_adjuster.h @@ -76,9 +76,6 @@ class EncoderBitrateAdjuster { // Minimum bitrates allowed, per spatial layer. uint32_t min_bitrates_bps_[kMaxSpatialLayers]; - // Size in pixels of each spatial layer. - uint32_t frame_size_pixels_[kMaxSpatialLayers]; - // Codec type used for encoding. VideoCodecType codec_; diff --git a/video/encoder_bitrate_adjuster_unittest.cc b/video/encoder_bitrate_adjuster_unittest.cc index e0f6de3120..ad672f617b 100644 --- a/video/encoder_bitrate_adjuster_unittest.cc +++ b/video/encoder_bitrate_adjuster_unittest.cc @@ -41,38 +41,10 @@ class EncoderBitrateAdjusterTest : public ::testing::Test { sequence_idx_{} {} protected: - void SetUpAdjuster(size_t num_spatial_layers, - size_t num_temporal_layers, - bool vp9_svc) { - // Initialize some default VideoCodec instance with the given number of - // layers. - if (vp9_svc) { - codec_.codecType = VideoCodecType::kVideoCodecVP9; - codec_.numberOfSimulcastStreams = 1; - codec_.VP9()->numberOfSpatialLayers = num_spatial_layers; - codec_.VP9()->numberOfTemporalLayers = num_temporal_layers; - for (size_t si = 0; si < num_spatial_layers; ++si) { - codec_.spatialLayers[si].minBitrate = 100 * (1 << si); - codec_.spatialLayers[si].targetBitrate = 200 * (1 << si); - codec_.spatialLayers[si].maxBitrate = 300 * (1 << si); - codec_.spatialLayers[si].active = true; - codec_.spatialLayers[si].numberOfTemporalLayers = num_temporal_layers; - } - } else { - codec_.codecType = VideoCodecType::kVideoCodecVP8; - codec_.numberOfSimulcastStreams = num_spatial_layers; - codec_.VP8()->numberOfTemporalLayers = num_temporal_layers; - for (size_t si = 0; si < num_spatial_layers; ++si) { - codec_.simulcastStream[si].minBitrate = 100 * (1 << si); - codec_.simulcastStream[si].targetBitrate = 200 * (1 << si); - codec_.simulcastStream[si].maxBitrate = 300 * (1 << si); - codec_.simulcastStream[si].active = true; - codec_.simulcastStream[si].numberOfTemporalLayers = num_temporal_layers; - codec_.spatialLayers[si].width = 320 * (1<numberOfSpatialLayers = num_spatial_layers; + codec.VP9()->numberOfTemporalLayers = num_temporal_layers; + for (size_t si = 0; si < num_spatial_layers; ++si) { + codec.spatialLayers[si].minBitrate = 100 * (1 << si); + codec.spatialLayers[si].targetBitrate = 200 * (1 << si); + codec.spatialLayers[si].maxBitrate = 300 * (1 << si); + codec.spatialLayers[si].active = true; + codec.spatialLayers[si].numberOfTemporalLayers = num_temporal_layers; + } + } else { + codec.codecType = VideoCodecType::kVideoCodecVP8; + codec.numberOfSimulcastStreams = num_spatial_layers; + codec.VP8()->numberOfTemporalLayers = num_temporal_layers; + for (size_t si = 0; si < num_spatial_layers; ++si) { + codec.simulcastStream[si].minBitrate = 100 * (1 << si); + codec.simulcastStream[si].targetBitrate = 200 * (1 << si); + codec.simulcastStream[si].maxBitrate = 300 * (1 << si); + codec.simulcastStream[si].active = true; + codec.simulcastStream[si].numberOfTemporalLayers = num_temporal_layers; + } + } + SetUpAdjusterWithCodec(num_spatial_layers, num_temporal_layers, codec); + } + void InsertFrames(std::vector> media_utilization_factors, int64_t duration_ms) { InsertFrames(media_utilization_factors, media_utilization_factors, @@ -509,23 +514,30 @@ TEST_F(EncoderBitrateAdjusterTest, DontExceedMediaRateEvenWithHeadroom) { } } -TEST_F(EncoderBitrateAdjusterTest, HonorMinBitrateSettingFromEncoderInfo) { +TEST_F(EncoderBitrateAdjusterTest, HonorsMinBitrateWithAv1) { // Single layer, well behaved encoder. - const int high_bitrate = 20000; - const int a_lower_min_bitrate = 12000; - current_input_allocation_.SetBitrate(0, 0, high_bitrate); + const DataRate kHighBitrate = DataRate::KilobitsPerSec(20); + const DataRate kALowerMinBitrate = DataRate::KilobitsPerSec(15); + + current_input_allocation_.SetBitrate(0, 0, kHighBitrate.bps()); + VideoBitrateAllocation expected_input_allocation; - expected_input_allocation.SetBitrate(0, 0, a_lower_min_bitrate); + expected_input_allocation.SetBitrate(0, 0, kALowerMinBitrate.bps()); target_framerate_fps_ = 30; - SetUpAdjuster(1, 1, false); + VideoCodec codec; + codec.codecType = VideoCodecType::kVideoCodecAV1; + codec.numberOfSimulcastStreams = 1; + codec.SetScalabilityMode(ScalabilityMode::kL1T1); + codec.spatialLayers[0].minBitrate = kALowerMinBitrate.kbps(); + codec.spatialLayers[0].targetBitrate = 500; + codec.spatialLayers[0].maxBitrate = 1000; + codec.spatialLayers[0].active = true; + codec.spatialLayers[0].numberOfTemporalLayers = 1; - auto new_resolution_limit = VideoEncoder::ResolutionBitrateLimits( - codec_.spatialLayers[0].width * codec_.spatialLayers[0].height, 15000, - a_lower_min_bitrate, 2000000); - encoder_info_.resolution_bitrate_limits.push_back(new_resolution_limit); - adjuster_->OnEncoderInfo(encoder_info_); + SetUpAdjusterWithCodec(/*num_spatial_layers=*/1, /*num_temporal_layers=*/1, + codec); InsertFrames({{2.0}}, kWindowSizeMs);