From b23b3dd9b1dcbc9cceb5a5c34972fef7ecafedc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Fri, 4 Oct 2024 14:10:02 +0200 Subject: [PATCH] Improve simulcast CPU adaptation when requested_resolution API is used. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In simulcast, BW adaptation causes layers to be disabled rather than downscaling layers. But CPU adaptation restricts the resolution of all layers, this means that a 540p restriction on 180p:360p:720p results in 180p:360p:540p, which is fine but a) it's inconsistent with BW adaptation and b) it's not ideal for performance, because non power of two scaling factors means we can't use a single encoder instance to produce all layers (the CPU adaptation could actually result in even more CPU usage and further adaptation as a result). This CL disables top layers by limiting `max_num_layers` based on `restrictions_` and the layers' `requested_resolution`, the end result is 180p:360p:- when CPU adaptation kicks in. Note that the problem described (and therefore the solution) is specific to the `requested_resolution` API. If instead the `scale_resolution_down_by` API is used, all scaling is relative and we get 135p:270p:540p, which is problematic for other reasons (180p and 360p no longer sent, middle layer no longer HW accelerated). Bug: webrtc:366415118 Change-Id: I2e238b1b87470413c21623b21d0ce20eadf6c8c7 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/364660 Commit-Queue: Henrik Boström Reviewed-by: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/main@{#43172} --- video/config/encoder_stream_factory.cc | 35 +++++++ .../config/encoder_stream_factory_unittest.cc | 99 +++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/video/config/encoder_stream_factory.cc b/video/config/encoder_stream_factory.cc index c58f464fc0..ad1ee4daf1 100644 --- a/video/config/encoder_stream_factory.cc +++ b/video/config/encoder_stream_factory.cc @@ -535,6 +535,41 @@ std::vector EncoderStreamFactory::GetStreamResolutions( : encoder_config.number_of_streams; RTC_DCHECK_LE(max_num_layers, encoder_config.number_of_streams); + // When the `requested_resolution` API is used, disable upper layers that + // are bigger than what adaptation restrictions allow. For example if + // restrictions are 540p, simulcast 180p:360p:720p becomes 180p:360p:- as + // opposed to 180p:360p:540p. This makes CPU adaptation consistent with BW + // adaptation (bitrate allocator disabling layers rather than downscaling) + // and means we don't have to break power of two optimization paths (i.e. + // S-modes based simulcast). Note that the lowest layer is never disabled. + if (encoder_config.HasRequestedResolution() && restrictions_.has_value() && + restrictions_->max_pixels_per_frame().has_value()) { + int max_pixels = rtc::dchecked_cast( + restrictions_->max_pixels_per_frame().value()); + int prev_pixel_count = + encoder_config.simulcast_layers[0] + .requested_resolution.value_or(webrtc::Resolution()) + .PixelCount(); + std::optional restricted_num_layers; + for (size_t i = 1; i < max_num_layers; ++i) { + int pixel_count = + encoder_config.simulcast_layers[i] + .requested_resolution.value_or(webrtc::Resolution()) + .PixelCount(); + if (!restricted_num_layers.has_value() && max_pixels < pixel_count) { + // Current layer is the highest layer allowed by restrictions. + restricted_num_layers = i; + } + if (pixel_count < prev_pixel_count) { + // Cannot limit layers because config is not lower-to-higher. + restricted_num_layers = std::nullopt; + break; + } + prev_pixel_count = pixel_count; + } + max_num_layers = restricted_num_layers.value_or(max_num_layers); + } + const bool has_scale_resolution_down_by = absl::c_any_of( encoder_config.simulcast_layers, [](const webrtc::VideoStream& layer) { return layer.scale_resolution_down_by != -1.; diff --git a/video/config/encoder_stream_factory_unittest.cc b/video/config/encoder_stream_factory_unittest.cc index aa33367e8a..1cf901dbbf 100644 --- a/video/config/encoder_stream_factory_unittest.cc +++ b/video/config/encoder_stream_factory_unittest.cc @@ -116,6 +116,105 @@ TEST(EncoderStreamFactory, SinglecastRequestedResolutionWithAdaptation) { })); } +TEST(EncoderStreamFactory, SimulcastRequestedResolutionUnrestricted) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].requested_resolution = {.width = 320, + .height = 180}; + encoder_config.simulcast_layers[1].requested_resolution = {.width = 640, + .height = 360}; + encoder_config.simulcast_layers[2].requested_resolution = {.width = 1280, + .height = 720}; + auto streams = CreateEncoderStreams( + field_trials, {.width = 1280, .height = 720}, encoder_config); + std::vector stream_resolutions = GetStreamResolutions(streams); + ASSERT_THAT(stream_resolutions, SizeIs(3)); + EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 320, .height = 180})); + EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360})); + EXPECT_EQ(stream_resolutions[2], (Resolution{.width = 1280, .height = 720})); +} + +TEST(EncoderStreamFactory, SimulcastRequestedResolutionWith360pRestriction) { + ExplicitKeyValueConfig field_trials(""); + VideoSourceRestrictions restrictions( + /* max_pixels_per_frame= */ (640 * 360), + /* target_pixels_per_frame= */ std::nullopt, + /* max_frame_rate= */ std::nullopt); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].requested_resolution = {.width = 320, + .height = 180}; + encoder_config.simulcast_layers[1].requested_resolution = {.width = 640, + .height = 360}; + encoder_config.simulcast_layers[2].requested_resolution = {.width = 1280, + .height = 720}; + auto streams = + CreateEncoderStreams(field_trials, {.width = 1280, .height = 720}, + encoder_config, restrictions); + std::vector stream_resolutions = GetStreamResolutions(streams); + // 720p layer is dropped due to 360p restrictions. + ASSERT_THAT(stream_resolutions, SizeIs(2)); + EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 320, .height = 180})); + EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360})); +} + +TEST(EncoderStreamFactory, SimulcastRequestedResolutionWith90pRestriction) { + ExplicitKeyValueConfig field_trials(""); + VideoSourceRestrictions restrictions( + /* max_pixels_per_frame= */ (160 * 90), + /* target_pixels_per_frame= */ std::nullopt, + /* max_frame_rate= */ std::nullopt); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].requested_resolution = {.width = 320, + .height = 180}; + encoder_config.simulcast_layers[1].requested_resolution = {.width = 640, + .height = 360}; + encoder_config.simulcast_layers[2].requested_resolution = {.width = 1280, + .height = 720}; + auto streams = + CreateEncoderStreams(field_trials, {.width = 1280, .height = 720}, + encoder_config, restrictions); + std::vector stream_resolutions = GetStreamResolutions(streams); + ASSERT_THAT(stream_resolutions, SizeIs(1)); + // 90p restriction means all but the first layer (180p) is dropped. The one + // and only layer is downsized to 90p. + EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 160, .height = 90})); +} + +TEST(EncoderStreamFactory, ReverseSimulcastRequestedResolutionWithRestriction) { + ExplicitKeyValueConfig field_trials(""); + VideoSourceRestrictions restrictions( + /* max_pixels_per_frame= */ (640 * 360), + /* target_pixels_per_frame= */ std::nullopt, + /* max_frame_rate= */ std::nullopt); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + // 720p, 360p, 180p (instead of the usual 180p, 360p, 720p). + encoder_config.simulcast_layers[0].requested_resolution = {.width = 1280, + .height = 720}; + encoder_config.simulcast_layers[1].requested_resolution = {.width = 640, + .height = 360}; + encoder_config.simulcast_layers[2].requested_resolution = {.width = 320, + .height = 180}; + auto streams = + CreateEncoderStreams(field_trials, {.width = 1280, .height = 720}, + encoder_config, restrictions); + std::vector stream_resolutions = GetStreamResolutions(streams); + // The layer dropping that is performed for lower-to-higher ordered simulcast + // streams is not applicable when higher-to-lower order is used. In this case + // the 360p restriction is applied to all layers. + ASSERT_THAT(stream_resolutions, SizeIs(3)); + EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 640, .height = 360})); + EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360})); + EXPECT_EQ(stream_resolutions[2], (Resolution{.width = 320, .height = 180})); +} + TEST(EncoderStreamFactory, BitratePriority) { constexpr double kBitratePriority = 0.123; VideoEncoderConfig encoder_config;