diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index 58fadc421d..b69196f021 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -44,6 +44,7 @@ rtc_library("resource_adaptation") { "../../api/video:video_stream_encoder", "../../api/video_codecs:video_codecs_api", "../../modules/video_coding:video_coding_utility", + "../../modules/video_coding/svc:scalability_mode_util", "../../rtc_base:checks", "../../rtc_base:logging", "../../rtc_base:macromagic", diff --git a/call/adaptation/video_stream_adapter.cc b/call/adaptation/video_stream_adapter.cc index f30a4d7abb..5a970fb2ef 100644 --- a/call/adaptation/video_stream_adapter.cc +++ b/call/adaptation/video_stream_adapter.cc @@ -22,6 +22,7 @@ #include "api/video_codecs/video_encoder.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" @@ -720,7 +721,17 @@ absl::optional VideoStreamAdapter::GetSingleActiveLayerPixels( const VideoCodec& codec) { int num_active = 0; absl::optional pixels; - if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + if (codec.codecType == VideoCodecType::kVideoCodecAV1 && + codec.GetScalabilityMode().has_value()) { + for (int i = 0; + i < ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode())); + ++i) { + if (codec.spatialLayers[i].active) { + ++num_active; + pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height; + } + } + } else if (codec.codecType == VideoCodecType::kVideoCodecVP9) { for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { if (codec.spatialLayers[i].active) { ++num_active; diff --git a/rtc_base/experiments/encoder_info_settings.cc b/rtc_base/experiments/encoder_info_settings.cc index 062ba0248c..2e90a21513 100644 --- a/rtc_base/experiments/encoder_info_settings.cc +++ b/rtc_base/experiments/encoder_info_settings.cc @@ -38,7 +38,8 @@ constexpr float kDefaultMinBitratebps = 30000; std::vector EncoderInfoSettings::GetDefaultSinglecastBitrateLimits( VideoCodecType codec_type) { - // Specific limits for VP9. Other codecs use VP8 limits. + // Specific limits for VP9. Determining specific limits for AV1 via + // field trial experiment is a work in progress. Other codecs use VP8 limits. if (codec_type == kVideoCodecVP9) { return {{320 * 180, 0, 30000, 150000}, {480 * 270, 120000, 30000, 300000}, @@ -46,6 +47,10 @@ EncoderInfoSettings::GetDefaultSinglecastBitrateLimits( {960 * 540, 350000, 30000, 1000000}, {1280 * 720, 480000, 30000, 1500000}}; } + // Don't override existing AV1 limits with default values. + if (codec_type == kVideoCodecAV1) { + return {}; + } return {{320 * 180, 0, 30000, 300000}, {480 * 270, 200000, 30000, 500000}, diff --git a/video/BUILD.gn b/video/BUILD.gn index 9acb0a51af..3161435cb3 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -403,6 +403,7 @@ rtc_library("video_stream_encoder_impl") { "../modules/video_coding:video_codec_interface", "../modules/video_coding:video_coding_utility", "../modules/video_coding:webrtc_vp9_helpers", + "../modules/video_coding/svc:scalability_mode_util", "../modules/video_coding/svc:scalability_structures", "../modules/video_coding/svc:svc_rate_allocator", "../rtc_base:checks", diff --git a/video/end_to_end_tests/resolution_bitrate_limits_tests.cc b/video/end_to_end_tests/resolution_bitrate_limits_tests.cc index 8455832885..d5503391cd 100644 --- a/video/end_to_end_tests/resolution_bitrate_limits_tests.cc +++ b/video/end_to_end_tests/resolution_bitrate_limits_tests.cc @@ -41,7 +41,18 @@ struct BitrateLimits { }; BitrateLimits GetLayerBitrateLimits(int pixels, const VideoCodec& codec) { - if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + if (codec.codecType == VideoCodecType::kVideoCodecAV1) { + EXPECT_TRUE(codec.GetScalabilityMode().has_value()); + for (int i = 0; + i < ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode())); + ++i) { + if (codec.spatialLayers[i].width * codec.spatialLayers[i].height == + pixels) { + return {DataRate::KilobitsPerSec(codec.spatialLayers[i].minBitrate), + DataRate::KilobitsPerSec(codec.spatialLayers[i].maxBitrate)}; + } + } + } else if (codec.codecType == VideoCodecType::kVideoCodecVP9) { for (size_t i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { if (codec.spatialLayers[i].width * codec.spatialLayers[i].height == pixels) { @@ -62,6 +73,10 @@ BitrateLimits GetLayerBitrateLimits(int pixels, const VideoCodec& codec) { return BitrateLimits(); } +bool SupportsSpatialLayers(const std::string& payload_name) { + return payload_name == "VP9" || payload_name == "AV1"; +} + } // namespace class ResolutionBitrateLimitsWithScalabilityModeTest : public test::CallTest {}; @@ -122,7 +137,7 @@ class InitEncodeTest : public test::EndToEndTest, const rtc::VideoSinkWants& wants) override {} size_t GetNumVideoStreams() const override { - return (payload_name_ == "VP9") ? 1 : configs_.size(); + return SupportsSpatialLayers(payload_name_) ? 1 : configs_.size(); } void ModifyVideoConfigs( @@ -142,7 +157,7 @@ class InitEncodeTest : public test::EndToEndTest, encoder_config->max_bitrate_bps = -1; if (configs_.size() == 1 && configs_[0].bitrate.max) encoder_config->max_bitrate_bps = configs_[0].bitrate.max->bps(); - if (payload_name_ == "VP9") { + if (SupportsSpatialLayers(payload_name_)) { // Simulcast layers indicates which spatial layers are active. encoder_config->simulcast_layers.resize(configs_.size()); } @@ -156,7 +171,7 @@ class InitEncodeTest : public test::EndToEndTest, if (configs_[i].bitrate.max) stream.max_bitrate_bps = configs_[i].bitrate.max->bps(); stream.scale_resolution_down_by = scale_factor; - scale_factor *= (payload_name_ == "VP9") ? 1.0 : 2.0; + scale_factor *= SupportsSpatialLayers(payload_name_) ? 1.0 : 2.0; } SetEncoderSpecific(encoder_config, codec_type, configs_.size()); } @@ -458,6 +473,67 @@ TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest, RunBaseTest(&test); } +TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest, + OneStreamLimitsAppliedForAv1OneSpatialLayer) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-GetEncoderInfoOverride/" + "frame_size_pixels:921600," + "min_start_bitrate_bps:0," + "min_bitrate_bps:32000," + "max_bitrate_bps:133000/"); + + InitEncodeTest test( + "AV1", {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}}, + // Expectations: + {{.pixels = 1280 * 720, + .eq_bitrate = {DataRate::KilobitsPerSec(32), + DataRate::KilobitsPerSec(133)}}}); + RunBaseTest(&test); +} + +TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest, + LimitsAppliedForAv1Simulcast) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-GetEncoderInfoOverride/" + "frame_size_pixels:230400|921600," + "min_start_bitrate_bps:0|0," + "min_bitrate_bps:25000|80000," + "max_bitrate_bps:400000|1200000/"); + + InitEncodeTest test( + "AV1", + {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}, + {.active = false}}, + // Expectations: + {{.pixels = 1280 * 720, + .eq_bitrate = {DataRate::KilobitsPerSec(80), + DataRate::KilobitsPerSec(1200)}}}); + RunBaseTest(&test); +} + +TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest, + LimitsNotAppliedForAv1MultipleSpatialLayers) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-GetEncoderInfoOverride/" + "frame_size_pixels:230400|921600," + "min_start_bitrate_bps:0|0," + "min_bitrate_bps:20000|25000," + "max_bitrate_bps:900000|1333000/"); + + InitEncodeTest test( + "AV1", + {{.active = true, .scalability_mode = ScalabilityMode::kL2T1}, + {.active = false}}, + // Expectations: + {{.pixels = 640 * 360, + .ne_bitrate = {DataRate::KilobitsPerSec(20), + DataRate::KilobitsPerSec(900)}}, + {.pixels = 1280 * 720, + .ne_bitrate = {DataRate::KilobitsPerSec(25), + DataRate::KilobitsPerSec(1333)}}}); + RunBaseTest(&test); +} + TEST_P(ResolutionBitrateLimitsTest, LimitsNotAppliedSimulcast) { webrtc::test::ScopedFieldTrials field_trials( "WebRTC-GetEncoderInfoOverride/" diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index f34388e56a..5fa2af398d 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -52,6 +52,9 @@ constexpr TimeDelta kEncoderTimeOut = TimeDelta::Seconds(2); constexpr double kVideoHysteresis = 1.2; constexpr double kScreenshareHysteresis = 1.35; +constexpr int kMinDefaultAv1BitrateBps = + 15000; // This value acts as an absolute minimum AV1 bitrate limit. + // When send-side BWE is used a stricter 1.1x pacing factor is used, rather than // the 2.5x which is used with receive-side BWE. Provides a more careful // bandwidth rampup with less risk of overshoots causing adverse effects like @@ -194,6 +197,13 @@ uint32_t GetInitialEncoderMaxBitrate(int initial_encoder_max_bitrate) { return kFallbackMaxBitrateBps; } +int GetDefaultMinVideoBitrateBps(VideoCodecType codec_type) { + if (codec_type == VideoCodecType::kVideoCodecAV1) { + return kMinDefaultAv1BitrateBps; + } + return kDefaultMinVideoBitrateBps; +} + } // namespace PacingConfig::PacingConfig(const FieldTrialsView& field_trials) @@ -485,7 +495,8 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( encoder_min_bitrate_bps_ = experimental_min_bitrate ? experimental_min_bitrate->bps() - : std::max(streams[0].min_bitrate_bps, kDefaultMinVideoBitrateBps); + : std::max(streams[0].min_bitrate_bps, + GetDefaultMinVideoBitrateBps(codec_type)); encoder_max_bitrate_bps_ = 0; double stream_bitrate_priority_sum = 0; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 34fa3621f7..39f46b930b 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -37,6 +37,7 @@ #include "call/adaptation/video_stream_adapter.h" #include "media/base/media_channel.h" #include "modules/video_coding/include/video_codec_initializer.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/svc_rate_allocator.h" #include "modules/video_coding/utility/vp8_constants.h" #include "rtc_base/arraysize.h" @@ -82,6 +83,17 @@ constexpr int kMaxAnimationPixels = 1280 * 720; constexpr int kDefaultMinScreenSharebps = 1200000; +int GetNumSpatialLayers(const VideoCodec& codec) { + if (codec.codecType == kVideoCodecVP9) { + return codec.VP9().numberOfSpatialLayers; + } else if (codec.codecType == kVideoCodecAV1 && + codec.GetScalabilityMode().has_value()) { + return ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode())); + } else { + return 0; + } +} + bool RequiresEncoderReset(const VideoCodec& prev_send_codec, const VideoCodec& new_send_codec, bool was_encode_called_since_last_initialization) { @@ -383,13 +395,18 @@ int NumActiveStreams(const std::vector& streams) { return num_active; } -void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info, - const VideoEncoderConfig& encoder_config, - VideoCodec* codec) { - if (codec->codecType != VideoCodecType::kVideoCodecVP9 || - encoder_config.simulcast_layers.size() <= 1 || - VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers( - encoder_config)) { +void ApplySpatialLayerBitrateLimits( + const VideoEncoder::EncoderInfo& encoder_info, + const VideoEncoderConfig& encoder_config, + VideoCodec* codec) { + if (!(GetNumSpatialLayers(*codec) > 0)) { + // ApplySpatialLayerBitrateLimits() supports VP9 and AV1 (the latter with + // scalability mode set) only. + return; + } + if (VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers( + encoder_config) || + encoder_config.simulcast_layers.size() <= 1) { // Resolution bitrate limits usage is restricted to singlecast. return; } @@ -405,7 +422,6 @@ void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info, if (!bitrate_limits.has_value()) { return; } - // Index for the active stream. absl::optional index; for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) { @@ -415,7 +431,6 @@ void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info, if (!index.has_value()) { return; } - int min_bitrate_bps; if (encoder_config.simulcast_layers[*index].min_bitrate_bps <= 0) { min_bitrate_bps = bitrate_limits->min_bitrate_bps; @@ -439,7 +454,7 @@ void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info, return; } - for (int i = 0; i < codec->VP9()->numberOfSpatialLayers; ++i) { + for (int i = 0; i < GetNumSpatialLayers(*codec); ++i) { if (codec->spatialLayers[i].active) { codec->spatialLayers[i].minBitrate = min_bitrate_bps / 1000; codec->spatialLayers[i].maxBitrate = max_bitrate_bps / 1000; @@ -1141,15 +1156,17 @@ void VideoStreamEncoder::ReconfigureEncoder() { RTC_LOG(LS_ERROR) << "Failed to create encoder configuration."; } - if (encoder_config_.codec_type == kVideoCodecVP9) { + if (encoder_config_.codec_type == kVideoCodecVP9 || + encoder_config_.codec_type == kVideoCodecAV1) { // Spatial layers configuration might impose some parity restrictions, // thus some cropping might be needed. crop_width_ = last_frame_info_->width - codec.width; crop_height_ = last_frame_info_->height - codec.height; - ApplyVp9BitrateLimits(GetEncoderInfoWithBitrateLimitUpdate( - encoder_->GetEncoderInfo(), encoder_config_, - default_limits_allowed_), - encoder_config_, &codec); + ApplySpatialLayerBitrateLimits( + GetEncoderInfoWithBitrateLimitUpdate(encoder_->GetEncoderInfo(), + encoder_config_, + default_limits_allowed_), + encoder_config_, &codec); } char log_stream_buf[4 * 1024]; @@ -1168,10 +1185,10 @@ void VideoStreamEncoder::ReconfigureEncoder() { << " active: " << (codec.simulcastStream[i].active ? "true" : "false") << "\n"; } - if (encoder_config_.codec_type == kVideoCodecVP9) { - size_t num_spatial_layers = codec.VP9()->numberOfSpatialLayers; + if (encoder_config_.codec_type == kVideoCodecVP9 || + encoder_config_.codec_type == kVideoCodecAV1) { log_stream << "Spatial layers:\n"; - for (size_t i = 0; i < num_spatial_layers; ++i) { + for (int i = 0; i < GetNumSpatialLayers(codec); ++i) { log_stream << i << ": " << codec.spatialLayers[i].width << "x" << codec.spatialLayers[i].height << " min_kbps: " << codec.spatialLayers[i].minBitrate @@ -1331,6 +1348,10 @@ void VideoStreamEncoder::ReconfigureEncoder() { num_layers = codec.VP8()->numberOfTemporalLayers; } else if (codec.codecType == kVideoCodecVP9) { num_layers = codec.VP9()->numberOfTemporalLayers; + } else if (codec.codecType == kVideoCodecAV1 && + codec.GetScalabilityMode().has_value()) { + num_layers = + ScalabilityModeToNumTemporalLayers(*(codec.GetScalabilityMode())); } else if (codec.codecType == kVideoCodecH264) { num_layers = codec.H264()->numberOfTemporalLayers; } else if (codec.codecType == kVideoCodecGeneric && @@ -1375,8 +1396,9 @@ void VideoStreamEncoder::ReconfigureEncoder() { bool is_svc = false; // Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9 - // and leave only one stream containing all necessary information. - if (encoder_config_.codec_type == kVideoCodecVP9 && + // and AV1 and leave only one stream containing all necessary information. + if ((encoder_config_.codec_type == kVideoCodecVP9 || + encoder_config_.codec_type == kVideoCodecAV1) && encoder_config_.number_of_streams == 1) { // Lower max bitrate to the level codec actually can produce. streams[0].max_bitrate_bps = @@ -1388,7 +1410,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { SvcRateAllocator::GetPaddingBitrate(codec).bps(); streams[0].width = streams.back().width; streams[0].height = streams.back().height; - is_svc = codec.VP9()->numberOfSpatialLayers > 1; + is_svc = GetNumSpatialLayers(codec) > 1; streams.resize(1); }