diff --git a/media/engine/simulcast.cc b/media/engine/simulcast.cc index 375572f374..6b3ed71553 100644 --- a/media/engine/simulcast.cc +++ b/media/engine/simulcast.cc @@ -15,13 +15,13 @@ #include #include +#include #include "absl/strings/match.h" #include "absl/types/optional.h" #include "api/video/video_codec_constants.h" #include "media/base/media_constants.h" #include "modules/video_coding/utility/simulcast_rate_allocator.h" -#include "rtc_base/arraysize.h" #include "rtc_base/checks.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/experiments/min_video_bitrate_experiment.h" @@ -42,6 +42,13 @@ constexpr webrtc::DataRate Interpolate(const webrtc::DataRate& a, constexpr char kUseLegacySimulcastLayerLimitFieldTrial[] = "WebRTC-LegacySimulcastLayerLimit"; +// TODO(webrtc:12415): Flip this to a kill switch when this feature launches. +bool EnableLowresBitrateInterpolation( + const webrtc::WebRtcKeyValueConfig& trials) { + return absl::StartsWith( + trials.Lookup("WebRTC-LowresSimulcastBitrateInterpolation"), "Enabled"); +} + // Limits for legacy conference screensharing mode. Currently used for the // lower of the two simulcast streams. constexpr webrtc::DataRate kScreenshareDefaultTl0Bitrate = @@ -97,10 +104,29 @@ constexpr const SimulcastFormat kSimulcastFormats[] = { {320, 180, 1, webrtc::DataRate::KilobitsPerSec(200), webrtc::DataRate::KilobitsPerSec(150), webrtc::DataRate::KilobitsPerSec(30)}, - {0, 0, 1, webrtc::DataRate::KilobitsPerSec(200), - webrtc::DataRate::KilobitsPerSec(150), + // As the resolution goes down, interpolate the target and max bitrates down + // towards zero. The min bitrate is still limited at 30 kbps and the target + // and the max will be capped from below accordingly. + {0, 0, 1, webrtc::DataRate::KilobitsPerSec(0), + webrtc::DataRate::KilobitsPerSec(0), webrtc::DataRate::KilobitsPerSec(30)}}; +std::vector GetSimulcastFormats( + bool enable_lowres_bitrate_interpolation) { + std::vector formats; + formats.insert(formats.begin(), std::begin(kSimulcastFormats), + std::end(kSimulcastFormats)); + if (!enable_lowres_bitrate_interpolation) { + RTC_CHECK_GE(formats.size(), 2u); + SimulcastFormat& format0x0 = formats[formats.size() - 1]; + const SimulcastFormat& format_prev = formats[formats.size() - 2]; + format0x0.max_bitrate = format_prev.max_bitrate; + format0x0.target_bitrate = format_prev.target_bitrate; + format0x0.min_bitrate = format_prev.min_bitrate; + } + return formats; +} + const int kMaxScreenshareSimulcastLayers = 2; // Multiway: Number of temporal layers for each simulcast stream. @@ -136,12 +162,14 @@ int DefaultNumberOfTemporalLayers(int simulcast_id, return default_num_temporal_layers; } -int FindSimulcastFormatIndex(int width, int height) { +int FindSimulcastFormatIndex(int width, + int height, + bool enable_lowres_bitrate_interpolation) { RTC_DCHECK_GE(width, 0); RTC_DCHECK_GE(height, 0); - for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) { - if (width * height >= - kSimulcastFormats[i].width * kSimulcastFormats[i].height) { + const auto formats = GetSimulcastFormats(enable_lowres_bitrate_interpolation); + for (uint32_t i = 0; i < formats.size(); ++i) { + if (width * height >= formats[i].width * formats[i].height) { return i; } } @@ -166,49 +194,67 @@ int NormalizeSimulcastSize(int size, size_t simulcast_layers) { SimulcastFormat InterpolateSimulcastFormat( int width, int height, - absl::optional max_roundup_rate) { - const int index = FindSimulcastFormatIndex(width, height); + absl::optional max_roundup_rate, + bool enable_lowres_bitrate_interpolation) { + const auto formats = GetSimulcastFormats(enable_lowres_bitrate_interpolation); + const int index = FindSimulcastFormatIndex( + width, height, enable_lowres_bitrate_interpolation); if (index == 0) - return kSimulcastFormats[index]; + return formats[index]; const int total_pixels_up = - kSimulcastFormats[index - 1].width * kSimulcastFormats[index - 1].height; - const int total_pixels_down = - kSimulcastFormats[index].width * kSimulcastFormats[index].height; + formats[index - 1].width * formats[index - 1].height; + const int total_pixels_down = formats[index].width * formats[index].height; const int total_pixels = width * height; const float rate = (total_pixels_up - total_pixels) / static_cast(total_pixels_up - total_pixels_down); // Use upper resolution if |rate| is below the configured threshold. size_t max_layers = (max_roundup_rate && rate < max_roundup_rate.value()) - ? kSimulcastFormats[index - 1].max_layers - : kSimulcastFormats[index].max_layers; - webrtc::DataRate max_bitrate = - Interpolate(kSimulcastFormats[index - 1].max_bitrate, - kSimulcastFormats[index].max_bitrate, rate); - webrtc::DataRate target_bitrate = - Interpolate(kSimulcastFormats[index - 1].target_bitrate, - kSimulcastFormats[index].target_bitrate, rate); - webrtc::DataRate min_bitrate = - Interpolate(kSimulcastFormats[index - 1].min_bitrate, - kSimulcastFormats[index].min_bitrate, rate); + ? formats[index - 1].max_layers + : formats[index].max_layers; + webrtc::DataRate max_bitrate = Interpolate(formats[index - 1].max_bitrate, + formats[index].max_bitrate, rate); + webrtc::DataRate target_bitrate = Interpolate( + formats[index - 1].target_bitrate, formats[index].target_bitrate, rate); + webrtc::DataRate min_bitrate = Interpolate(formats[index - 1].min_bitrate, + formats[index].min_bitrate, rate); return {width, height, max_layers, max_bitrate, target_bitrate, min_bitrate}; } -SimulcastFormat InterpolateSimulcastFormat(int width, int height) { - return InterpolateSimulcastFormat(width, height, absl::nullopt); +SimulcastFormat InterpolateSimulcastFormat( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, absl::nullopt, + enable_lowres_bitrate_interpolation); } -webrtc::DataRate FindSimulcastMaxBitrate(int width, int height) { - return InterpolateSimulcastFormat(width, height).max_bitrate; +webrtc::DataRate FindSimulcastMaxBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .max_bitrate; } -webrtc::DataRate FindSimulcastTargetBitrate(int width, int height) { - return InterpolateSimulcastFormat(width, height).target_bitrate; +webrtc::DataRate FindSimulcastTargetBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .target_bitrate; } -webrtc::DataRate FindSimulcastMinBitrate(int width, int height) { - return InterpolateSimulcastFormat(width, height).min_bitrate; +webrtc::DataRate FindSimulcastMinBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .min_bitrate; } void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate, @@ -254,9 +300,12 @@ size_t LimitSimulcastLayerCount(int width, webrtc::ParseFieldTrial({&max_ratio}, trials.Lookup("WebRTC-SimulcastLayerLimitRoundUp")); + const bool enable_lowres_bitrate_interpolation = + EnableLowresBitrateInterpolation(trials); size_t adaptive_layer_count = std::max( need_layers, - InterpolateSimulcastFormat(width, height, max_ratio.GetOptional()) + InterpolateSimulcastFormat(width, height, max_ratio.GetOptional(), + enable_lowres_bitrate_interpolation) .max_layers); if (layer_count > adaptive_layer_count) { RTC_LOG(LS_WARNING) << "Reducing simulcast layer count from " @@ -311,6 +360,9 @@ std::vector GetNormalSimulcastLayers( const webrtc::WebRtcKeyValueConfig& trials) { std::vector layers(layer_count); + const bool enable_lowres_bitrate_interpolation = + EnableLowresBitrateInterpolation(trials); + // Format width and height has to be divisible by |2 ^ num_simulcast_layers - // 1|. width = NormalizeSimulcastSize(width, layer_count); @@ -326,9 +378,14 @@ std::vector GetNormalSimulcastLayers( temporal_layers_supported ? DefaultNumberOfTemporalLayers(s, false, trials) : 1; - layers[s].max_bitrate_bps = FindSimulcastMaxBitrate(width, height).bps(); + layers[s].max_bitrate_bps = + FindSimulcastMaxBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); layers[s].target_bitrate_bps = - FindSimulcastTargetBitrate(width, height).bps(); + FindSimulcastTargetBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); int num_temporal_layers = DefaultNumberOfTemporalLayers(s, false, trials); if (s == 0) { // If alternative temporal rate allocation is selected, adjust the @@ -355,7 +412,17 @@ std::vector GetNormalSimulcastLayers( layers[s].target_bitrate_bps = static_cast(layers[s].target_bitrate_bps * rate_factor); } - layers[s].min_bitrate_bps = FindSimulcastMinBitrate(width, height).bps(); + layers[s].min_bitrate_bps = + FindSimulcastMinBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); + + // Ensure consistency. + layers[s].max_bitrate_bps = + std::max(layers[s].min_bitrate_bps, layers[s].max_bitrate_bps); + layers[s].target_bitrate_bps = + std::max(layers[s].min_bitrate_bps, layers[s].target_bitrate_bps); + layers[s].max_framerate = kDefaultVideoMaxFramerate; width /= 2; diff --git a/media/engine/simulcast_unittest.cc b/media/engine/simulcast_unittest.cc index 193f8c0259..1635055286 100644 --- a/media/engine/simulcast_unittest.cc +++ b/media/engine/simulcast_unittest.cc @@ -440,4 +440,63 @@ TEST(SimulcastTest, MaxLayersWithFieldTrial) { EXPECT_EQ(1u, streams.size()); } +TEST(SimulcastTest, BitratesInterpolatedForResBelow180p) { + // TODO(webrtc:12415): Remove when feature launches. + test::ScopedFieldTrials field_trials( + "WebRTC-LowresSimulcastBitrateInterpolation/Enabled/"); + + const size_t kMaxLayers = 3; + FieldTrialBasedConfig trials; + + std::vector streams = cricket::GetSimulcastConfig( + /* min_layers = */ 1, kMaxLayers, /* width = */ 960, /* height = */ 540, + kBitratePriority, kQpMax, !kScreenshare, true, trials); + + ASSERT_EQ(streams.size(), kMaxLayers); + EXPECT_EQ(240u, streams[0].width); + EXPECT_EQ(135u, streams[0].height); + EXPECT_EQ(streams[0].max_bitrate_bps, 112500); + EXPECT_EQ(streams[0].target_bitrate_bps, 84375); + EXPECT_EQ(streams[0].min_bitrate_bps, 30000); +} + +TEST(SimulcastTest, BitratesConsistentForVerySmallRes) { + // TODO(webrtc:12415): Remove when feature launches. + test::ScopedFieldTrials field_trials( + "WebRTC-LowresSimulcastBitrateInterpolation/Enabled/"); + + FieldTrialBasedConfig trials; + + std::vector streams = cricket::GetSimulcastConfig( + /* min_layers = */ 1, /* max_layers = */ 3, /* width = */ 1, + /* height = */ 1, kBitratePriority, kQpMax, !kScreenshare, true, trials); + + ASSERT_TRUE(!streams.empty()); + EXPECT_EQ(1u, streams[0].width); + EXPECT_EQ(1u, streams[0].height); + EXPECT_EQ(streams[0].max_bitrate_bps, 30000); + EXPECT_EQ(streams[0].target_bitrate_bps, 30000); + EXPECT_EQ(streams[0].min_bitrate_bps, 30000); +} + +TEST(SimulcastTest, + BitratesNotInterpolatedForResBelow180pWhenDisabledTrialSet) { + test::ScopedFieldTrials field_trials( + "WebRTC-LowresSimulcastBitrateInterpolation/Disabled/"); + + const size_t kMaxLayers = 3; + FieldTrialBasedConfig trials; + + std::vector streams = cricket::GetSimulcastConfig( + /* min_layers = */ 1, kMaxLayers, /* width = */ 960, /* height = */ 540, + kBitratePriority, kQpMax, !kScreenshare, true, trials); + + ASSERT_EQ(streams.size(), kMaxLayers); + EXPECT_EQ(240u, streams[0].width); + EXPECT_EQ(135u, streams[0].height); + EXPECT_EQ(streams[0].max_bitrate_bps, 200000); + EXPECT_EQ(streams[0].target_bitrate_bps, 150000); + EXPECT_EQ(streams[0].min_bitrate_bps, 30000); +} + } // namespace webrtc