diff --git a/media/engine/simulcast.cc b/media/engine/simulcast.cc index 9308db35d0..94609ff6ea 100644 --- a/media/engine/simulcast.cc +++ b/media/engine/simulcast.cc @@ -33,18 +33,28 @@ namespace cricket { namespace { +constexpr webrtc::DataRate Interpolate(const webrtc::DataRate& a, + const webrtc::DataRate& b, + float rate) { + return a * (1.0 - rate) + b * rate; +} + constexpr char kUseLegacySimulcastLayerLimitFieldTrial[] = "WebRTC-LegacySimulcastLayerLimit"; // Limits for legacy conference screensharing mode. Currently used for the // lower of the two simulcast streams. -constexpr int kScreenshareDefaultTl0BitrateKbps = 200; -constexpr int kScreenshareDefaultTl1BitrateKbps = 1000; +constexpr webrtc::DataRate kScreenshareDefaultTl0Bitrate = + webrtc::DataRate::kbps(200); +constexpr webrtc::DataRate kScreenshareDefaultTl1Bitrate = + webrtc::DataRate::kbps(1000); // Min/max bitrate for the higher one of the two simulcast stream used for // screen content. -constexpr int kScreenshareHighStreamMinBitrateBps = 600000; -constexpr int kScreenshareHighStreamMaxBitrateBps = 1250000; +constexpr webrtc::DataRate kScreenshareHighStreamMinBitrate = + webrtc::DataRate::kbps(600); +constexpr webrtc::DataRate kScreenshareHighStreamMaxBitrate = + webrtc::DataRate::kbps(1250); } // namespace @@ -56,27 +66,34 @@ struct SimulcastFormat { size_t max_layers; // The maximum bitrate for encoding stream at |widthxheight|, when we are // not sending the next higher spatial stream. - int max_bitrate_kbps; + webrtc::DataRate max_bitrate; // The target bitrate for encoding stream at |widthxheight|, when this layer // is not the highest layer (i.e., when we are sending another higher spatial // stream). - int target_bitrate_kbps; + webrtc::DataRate target_bitrate; // The minimum bitrate needed for encoding stream at |widthxheight|. - int min_bitrate_kbps; + webrtc::DataRate min_bitrate; }; // These tables describe from which resolution we can use how many // simulcast layers at what bitrates (maximum, target, and minimum). // Important!! Keep this table from high resolution to low resolution. // clang-format off -const SimulcastFormat kSimulcastFormats[] = { - {1920, 1080, 3, 5000, 4000, 800}, - {1280, 720, 3, 2500, 2500, 600}, - {960, 540, 3, 1200, 1200, 350}, - {640, 360, 2, 700, 500, 150}, - {480, 270, 2, 450, 350, 150}, - {320, 180, 1, 200, 150, 30}, - {0, 0, 1, 200, 150, 30} +constexpr const SimulcastFormat kSimulcastFormats[] = { + {1920, 1080, 3, webrtc::DataRate::kbps(5000), webrtc::DataRate::kbps(4000), + webrtc::DataRate::kbps(800)}, + {1280, 720, 3, webrtc::DataRate::kbps(2500), webrtc::DataRate::kbps(2500), + webrtc::DataRate::kbps(600)}, + {960, 540, 3, webrtc::DataRate::kbps(1200), webrtc::DataRate::kbps(1200), + webrtc::DataRate::kbps(350)}, + {640, 360, 2, webrtc::DataRate::kbps(700), webrtc::DataRate::kbps(500), + webrtc::DataRate::kbps(150)}, + {480, 270, 2, webrtc::DataRate::kbps(450), webrtc::DataRate::kbps(350), + webrtc::DataRate::kbps(150)}, + {320, 180, 1, webrtc::DataRate::kbps(200), webrtc::DataRate::kbps(150), + webrtc::DataRate::kbps(30)}, + {0, 0, 1, webrtc::DataRate::kbps(200), webrtc::DataRate::kbps(150), + webrtc::DataRate::kbps(30)} }; // clang-format on @@ -128,6 +145,7 @@ int FindSimulcastFormatIndex(int width, int height) { return -1; } +// Round size to nearest simulcast-friendly size. // Simulcast stream width and height must both be dividable by // |2 ^ (simulcast_layers - 1)|. int NormalizeSimulcastSize(int size, size_t simulcast_layers) { @@ -152,59 +170,59 @@ SimulcastFormat InterpolateSimulcastFormat(int width, int height) { const int total_pixels = width * height; const float rate = (total_pixels_up - total_pixels) / static_cast(total_pixels_up - total_pixels_down); - SimulcastFormat res; - res.width = width; - res.height = height; - res.max_layers = kSimulcastFormats[index].max_layers; - res.max_bitrate_kbps = - kSimulcastFormats[index - 1].max_bitrate_kbps * (1.0 - rate) + - kSimulcastFormats[index].max_bitrate_kbps * rate; - res.target_bitrate_kbps = - kSimulcastFormats[index - 1].target_bitrate_kbps * (1.0 - rate) + - kSimulcastFormats[index].target_bitrate_kbps * rate; - res.min_bitrate_kbps = - kSimulcastFormats[index - 1].min_bitrate_kbps * (1.0 - rate) + - kSimulcastFormats[index].min_bitrate_kbps * rate; - return res; + + size_t max_layers = kSimulcastFormats[index].max_layers; + webrtc::DataRate max_bitrate = + Interpolate(kSimulcastFormats[index - 1].max_bitrate, + kSimulcastFormats[index].max_bitrate, rate); + webrtc::DataRate target_bitrate = + Interpolate(kSimulcastFormats[index - 1].target_bitrate, + kSimulcastFormats[index].target_bitrate, rate); + webrtc::DataRate min_bitrate = + Interpolate(kSimulcastFormats[index - 1].min_bitrate, + kSimulcastFormats[index].min_bitrate, rate); + + return {width, height, max_layers, max_bitrate, target_bitrate, min_bitrate}; } -int FindSimulcastMaxBitrateBps(int width, int height) { - const SimulcastFormat format = InterpolateSimulcastFormat(width, height); - return format.max_bitrate_kbps * 1000; +webrtc::DataRate FindSimulcastMaxBitrate(int width, int height) { + return InterpolateSimulcastFormat(width, height).max_bitrate; } -int FindSimulcastTargetBitrateBps(int width, int height) { - const SimulcastFormat format = InterpolateSimulcastFormat(width, height); - return format.target_bitrate_kbps * 1000; +webrtc::DataRate FindSimulcastTargetBitrate(int width, int height) { + return InterpolateSimulcastFormat(width, height).target_bitrate; } -int FindSimulcastMinBitrateBps(int width, int height) { - const SimulcastFormat format = InterpolateSimulcastFormat(width, height); - return format.min_bitrate_kbps * 1000; +webrtc::DataRate FindSimulcastMinBitrate(int width, int height) { + return InterpolateSimulcastFormat(width, height).min_bitrate; } -void BoostMaxSimulcastLayer(int max_bitrate_bps, +void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate, std::vector* layers) { if (layers->empty()) return; - // Spend additional bits to boost the max layer. - int bitrate_left_bps = max_bitrate_bps - GetTotalMaxBitrateBps(*layers); - if (bitrate_left_bps > 0) { - layers->back().max_bitrate_bps += bitrate_left_bps; + const webrtc::DataRate total_bitrate = GetTotalMaxBitrate(*layers); + + // We're still not using all available bits. + if (total_bitrate < max_bitrate) { + // Spend additional bits to boost the max layer. + const webrtc::DataRate bitrate_left = max_bitrate - total_bitrate; + layers->back().max_bitrate_bps += bitrate_left.bps(); } } -int GetTotalMaxBitrateBps(const std::vector& layers) { +webrtc::DataRate GetTotalMaxBitrate( + const std::vector& layers) { if (layers.empty()) - return 0; + return webrtc::DataRate::Zero(); int total_max_bitrate_bps = 0; for (size_t s = 0; s < layers.size() - 1; ++s) { total_max_bitrate_bps += layers[s].target_bitrate_bps; } total_max_bitrate_bps += layers.back().max_bitrate_bps; - return total_max_bitrate_bps; + return webrtc::DataRate::bps(total_max_bitrate_bps); } size_t LimitSimulcastLayerCount(int width, @@ -280,8 +298,9 @@ std::vector GetNormalSimulcastLayers( layers[s].max_qp = max_qp; layers[s].num_temporal_layers = temporal_layers_supported ? DefaultNumberOfTemporalLayers(s, false) : 1; - layers[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height); - layers[s].target_bitrate_bps = FindSimulcastTargetBitrateBps(width, height); + layers[s].max_bitrate_bps = FindSimulcastMaxBitrate(width, height).bps(); + layers[s].target_bitrate_bps = + FindSimulcastTargetBitrate(width, height).bps(); int num_temporal_layers = DefaultNumberOfTemporalLayers(s, false); if (s == 0) { // If alternative temporal rate allocation is selected, adjust the @@ -308,7 +327,7 @@ std::vector GetNormalSimulcastLayers( layers[s].target_bitrate_bps = static_cast(layers[s].target_bitrate_bps * rate_factor); } - layers[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height); + layers[s].min_bitrate_bps = FindSimulcastMinBitrate(width, height).bps(); layers[s].max_framerate = kDefaultVideoMaxFramerate; width /= 2; @@ -348,8 +367,8 @@ std::vector GetScreenshareLayers( layers[0].max_qp = max_qp; layers[0].max_framerate = 5; layers[0].min_bitrate_bps = webrtc::kDefaultMinVideoBitrateBps; - layers[0].target_bitrate_bps = kScreenshareDefaultTl0BitrateKbps * 1000; - layers[0].max_bitrate_bps = kScreenshareDefaultTl1BitrateKbps * 1000; + layers[0].target_bitrate_bps = kScreenshareDefaultTl0Bitrate.bps(); + layers[0].max_bitrate_bps = kScreenshareDefaultTl1Bitrate.bps(); layers[0].num_temporal_layers = temporal_layers_supported ? 2 : 1; // With simulcast enabled, add another spatial layer. This one will have a @@ -371,14 +390,14 @@ std::vector GetScreenshareLayers( // Set the max bitrate to where the base layer would have been if temporal // layers were enabled. max_bitrate_bps = static_cast( - kScreenshareHighStreamMaxBitrateBps * + kScreenshareHighStreamMaxBitrate.bps() * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( num_temporal_layers, 0, base_heavy_tl3_rate_alloc)); } else if (DefaultNumberOfTemporalLayers(1, true) != 3 || base_heavy_tl3_rate_alloc) { // Experimental temporal layer mode used, use increased max bitrate. max_bitrate_bps = experimental_settings.TopLayerMaxBitrate().value_or( - kScreenshareHighStreamMaxBitrateBps); + kScreenshareHighStreamMaxBitrate.bps()); using_boosted_bitrate = true; } else { // Keep current bitrates with default 3tl/8 frame settings. @@ -397,12 +416,13 @@ std::vector GetScreenshareLayers( layers[1].num_temporal_layers = temporal_layers_supported ? DefaultNumberOfTemporalLayers(1, true) : 1; layers[1].min_bitrate_bps = using_boosted_bitrate - ? kScreenshareHighStreamMinBitrateBps + ? kScreenshareHighStreamMinBitrate.bps() : layers[0].target_bitrate_bps * 2; // Cap max bitrate so it isn't overly high for the given resolution. - int resolution_limited_bitrate = std::max( - FindSimulcastMaxBitrateBps(width, height), layers[1].min_bitrate_bps); + int resolution_limited_bitrate = + std::max(FindSimulcastMaxBitrate(width, height).bps(), + layers[1].min_bitrate_bps); max_bitrate_bps = std::min(max_bitrate_bps, resolution_limited_bitrate); diff --git a/media/engine/simulcast.h b/media/engine/simulcast.h index 6af13c10f3..28b08560aa 100644 --- a/media/engine/simulcast.h +++ b/media/engine/simulcast.h @@ -15,16 +15,18 @@ #include +#include "api/units/data_rate.h" #include "api/video_codecs/video_encoder_config.h" namespace cricket { // Gets the total maximum bitrate for the |streams|. -int GetTotalMaxBitrateBps(const std::vector& streams); +webrtc::DataRate GetTotalMaxBitrate( + const std::vector& streams); -// Adds any bitrate of |max_bitrate_bps| that is above the total maximum bitrate -// for the |layers| to the highest quality layer. -void BoostMaxSimulcastLayer(int max_bitrate_bps, +// Adds any bitrate of |max_bitrate| that is above the total maximum bitrate for +// the |layers| to the highest quality layer. +void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate, std::vector* layers); // Round size to nearest simulcast-friendly size diff --git a/media/engine/simulcast_unittest.cc b/media/engine/simulcast_unittest.cc index c8db8a32ef..e5c4c89a42 100644 --- a/media/engine/simulcast_unittest.cc +++ b/media/engine/simulcast_unittest.cc @@ -40,13 +40,13 @@ const std::vector GetSimulcastBitrates720p() { TEST(SimulcastTest, TotalMaxBitrateIsZeroForNoStreams) { std::vector streams; - EXPECT_EQ(0, cricket::GetTotalMaxBitrateBps(streams)); + EXPECT_EQ(0, cricket::GetTotalMaxBitrate(streams).bps()); } TEST(SimulcastTest, GetTotalMaxBitrateForSingleStream) { std::vector streams(1); streams[0].max_bitrate_bps = 100000; - EXPECT_EQ(100000, cricket::GetTotalMaxBitrateBps(streams)); + EXPECT_EQ(100000, cricket::GetTotalMaxBitrate(streams).bps()); } TEST(SimulcastTest, GetTotalMaxBitrateForMultipleStreams) { @@ -54,7 +54,7 @@ TEST(SimulcastTest, GetTotalMaxBitrateForMultipleStreams) { streams[0].target_bitrate_bps = 100000; streams[1].target_bitrate_bps = 200000; streams[2].max_bitrate_bps = 400000; - EXPECT_EQ(700000, cricket::GetTotalMaxBitrateBps(streams)); + EXPECT_EQ(700000, cricket::GetTotalMaxBitrate(streams).bps()); } TEST(SimulcastTest, BandwidthAboveTotalMaxBitrateGivenToHighestStream) { @@ -63,16 +63,19 @@ TEST(SimulcastTest, BandwidthAboveTotalMaxBitrateGivenToHighestStream) { streams[1].target_bitrate_bps = 200000; streams[2].max_bitrate_bps = 400000; + const webrtc::DataRate one_bps = webrtc::DataRate::bps(1); + // No bitrate above the total max to give to the highest stream. - const int kMaxTotalBps = cricket::GetTotalMaxBitrateBps(streams); - cricket::BoostMaxSimulcastLayer(kMaxTotalBps, &streams); + const webrtc::DataRate max_total_bitrate = + cricket::GetTotalMaxBitrate(streams); + cricket::BoostMaxSimulcastLayer(max_total_bitrate, &streams); EXPECT_EQ(400000, streams[2].max_bitrate_bps); - EXPECT_EQ(kMaxTotalBps, cricket::GetTotalMaxBitrateBps(streams)); + EXPECT_EQ(max_total_bitrate, cricket::GetTotalMaxBitrate(streams)); // The bitrate above the total max should be given to the highest stream. - cricket::BoostMaxSimulcastLayer(kMaxTotalBps + 1, &streams); + cricket::BoostMaxSimulcastLayer(max_total_bitrate + one_bps, &streams); EXPECT_EQ(400000 + 1, streams[2].max_bitrate_bps); - EXPECT_EQ(kMaxTotalBps + 1, cricket::GetTotalMaxBitrateBps(streams)); + EXPECT_EQ(max_total_bitrate + one_bps, cricket::GetTotalMaxBitrate(streams)); } TEST(SimulcastTest, GetConfig) { diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 73893589aa..b1c2828de8 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -3341,10 +3341,12 @@ EncoderStreamFactory::CreateSimulcastOrConfereceModeScreenshareStreams( encoder_config.simulcast_layers[i].max_bitrate_bps > 0; } } - if (!is_screenshare_ && !is_highest_layer_max_bitrate_configured) { + if (!is_screenshare_ && !is_highest_layer_max_bitrate_configured && + encoder_config.max_bitrate_bps > 0) { // No application-configured maximum for the largest layer. // If there is bitrate leftover, give it to the largest layer. - BoostMaxSimulcastLayer(encoder_config.max_bitrate_bps, &layers); + BoostMaxSimulcastLayer( + webrtc::DataRate::bps(encoder_config.max_bitrate_bps), &layers); } return layers; } diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index 8b5b26c91f..a04f99ac75 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -4588,7 +4588,7 @@ TEST_F(WebRtcVideoChannelTest, std::vector streams = stream->GetVideoStreams(); ASSERT_GT(streams.size(), 1u) << "Without simulcast this test doesn't make sense."; - int initial_max_bitrate_bps = GetTotalMaxBitrateBps(streams); + int initial_max_bitrate_bps = GetTotalMaxBitrate(streams).bps(); EXPECT_GT(initial_max_bitrate_bps, 0); parameters.max_bandwidth_bps = initial_max_bitrate_bps * 2; @@ -4596,7 +4596,7 @@ TEST_F(WebRtcVideoChannelTest, // Insert a frame to update the encoder config. frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); streams = stream->GetVideoStreams(); - int increased_max_bitrate_bps = GetTotalMaxBitrateBps(streams); + int increased_max_bitrate_bps = GetTotalMaxBitrate(streams).bps(); EXPECT_EQ(initial_max_bitrate_bps * 2, increased_max_bitrate_bps); EXPECT_TRUE(channel_->SetVideoSend(kSsrcs3[0], nullptr, nullptr)); @@ -7034,7 +7034,7 @@ TEST_F(WebRtcVideoChannelTest, BandwidthAboveTotalMaxBitrateGivenToMaxLayer) { // Set max bandwidth equal to total max bitrate. send_parameters_.max_bandwidth_bps = - GetTotalMaxBitrateBps(stream->GetVideoStreams()); + GetTotalMaxBitrate(stream->GetVideoStreams()).bps(); ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); @@ -7045,7 +7045,7 @@ TEST_F(WebRtcVideoChannelTest, BandwidthAboveTotalMaxBitrateGivenToMaxLayer) { // Set max bandwidth above the total max bitrate. send_parameters_.max_bandwidth_bps = - GetTotalMaxBitrateBps(stream->GetVideoStreams()) + 1; + GetTotalMaxBitrate(stream->GetVideoStreams()).bps() + 1; ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); ASSERT_TRUE(channel_->SetSendParameters(send_parameters_)); @@ -7053,7 +7053,7 @@ TEST_F(WebRtcVideoChannelTest, BandwidthAboveTotalMaxBitrateGivenToMaxLayer) { // max should be given to the highest layer. EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); EXPECT_EQ(send_parameters_.max_bandwidth_bps, - GetTotalMaxBitrateBps(stream->GetVideoStreams())); + GetTotalMaxBitrate(stream->GetVideoStreams()).bps()); EXPECT_EQ(kDefault[2].max_bitrate_bps + 1, stream->GetVideoStreams()[2].max_bitrate_bps); @@ -7082,7 +7082,7 @@ TEST_F(WebRtcVideoChannelTest, // Set max bandwidth above the total max bitrate. send_parameters_.max_bandwidth_bps = - GetTotalMaxBitrateBps(stream->GetVideoStreams()) + 1; + GetTotalMaxBitrate(stream->GetVideoStreams()).bps() + 1; ExpectSetMaxBitrate(send_parameters_.max_bandwidth_bps); ASSERT_TRUE(channel_->SetSendParameters(send_parameters_));