admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve simulcast CPU adaptation when requested_resolution API is used.

Browse Source 
In simulcast, BW adaptation causes layers to be disabled rather than
downscaling layers. But CPU adaptation restricts the resolution of all
layers, this means that a 540p restriction on 180p:360p:720p results in
180p:360p:540p, which is fine but a) it's inconsistent with BW
adaptation and b) it's not ideal for performance, because non power of
two scaling factors means we can't use a single encoder instance to
produce all layers (the CPU adaptation could actually result in even
more CPU usage and further adaptation as a result).

This CL disables top layers by limiting `max_num_layers` based on
`restrictions_` and the layers' `requested_resolution`, the end result
is 180p:360p:- when CPU adaptation kicks in.

Note that the problem described (and therefore the solution) is
specific to the `requested_resolution` API. If instead the
`scale_resolution_down_by` API is used, all scaling is relative and we
get 135p:270p:540p, which is problematic for other reasons (180p and
360p no longer sent, middle layer no longer HW accelerated).

Bug: webrtc:366415118
Change-Id: I2e238b1b87470413c21623b21d0ce20eadf6c8c7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/364660
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43172}
This commit is contained in:
Henrik Boström 2024-10-04 14:10:02 +02:00 committed by WebRTC LUCI CQ
parent d9b04adbdb
commit b23b3dd9b1
2 changed files with 134 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
video/config/encoder_stream_factory.cc
View File

@ -535,6 +535,41 @@ std::vector<webrtc::Resolution> EncoderStreamFactory::GetStreamResolutions(
: encoder_config.number_of_streams;
RTC_DCHECK_LE(max_num_layers, encoder_config.number_of_streams);
// When the `requested_resolution` API is used, disable upper layers that
// are bigger than what adaptation restrictions allow. For example if
// restrictions are 540p, simulcast 180p:360p:720p becomes 180p:360p:- as
// opposed to 180p:360p:540p. This makes CPU adaptation consistent with BW
// adaptation (bitrate allocator disabling layers rather than downscaling)
// and means we don't have to break power of two optimization paths (i.e.
// S-modes based simulcast). Note that the lowest layer is never disabled.
if (encoder_config.HasRequestedResolution() && restrictions_.has_value() &&
restrictions_->max_pixels_per_frame().has_value()) {
int max_pixels = rtc::dchecked_cast<int>(
restrictions_->max_pixels_per_frame().value());
int prev_pixel_count =
encoder_config.simulcast_layers[0]
.requested_resolution.value_or(webrtc::Resolution())
.PixelCount();
std::optional<size_t> restricted_num_layers;
for (size_t i = 1; i < max_num_layers; ++i) {
int pixel_count =
encoder_config.simulcast_layers[i]
.requested_resolution.value_or(webrtc::Resolution())
.PixelCount();
if (!restricted_num_layers.has_value() && max_pixels < pixel_count) {
// Current layer is the highest layer allowed by restrictions.
restricted_num_layers = i;
}
if (pixel_count < prev_pixel_count) {
// Cannot limit layers because config is not lower-to-higher.
restricted_num_layers = std::nullopt;
break;
}
prev_pixel_count = pixel_count;
}
max_num_layers = restricted_num_layers.value_or(max_num_layers);
}
const bool has_scale_resolution_down_by = absl::c_any_of(
encoder_config.simulcast_layers, [](const webrtc::VideoStream& layer) {
return layer.scale_resolution_down_by != -1.;

99
video/config/encoder_stream_factory_unittest.cc
View File

@ -116,6 +116,105 @@ TEST(EncoderStreamFactory, SinglecastRequestedResolutionWithAdaptation) {
}));
}
TEST(EncoderStreamFactory, SimulcastRequestedResolutionUnrestricted) {
ExplicitKeyValueConfig field_trials("");
VideoEncoderConfig encoder_config;
encoder_config.number_of_streams = 3;
encoder_config.simulcast_layers.resize(3);
encoder_config.simulcast_layers[0].requested_resolution = {.width = 320,
.height = 180};
encoder_config.simulcast_layers[1].requested_resolution = {.width = 640,
.height = 360};
encoder_config.simulcast_layers[2].requested_resolution = {.width = 1280,
.height = 720};
auto streams = CreateEncoderStreams(
field_trials, {.width = 1280, .height = 720}, encoder_config);
std::vector<Resolution> stream_resolutions = GetStreamResolutions(streams);
ASSERT_THAT(stream_resolutions, SizeIs(3));
EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 320, .height = 180}));
EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360}));
EXPECT_EQ(stream_resolutions[2], (Resolution{.width = 1280, .height = 720}));
}
TEST(EncoderStreamFactory, SimulcastRequestedResolutionWith360pRestriction) {
ExplicitKeyValueConfig field_trials("");
VideoSourceRestrictions restrictions(
/* max_pixels_per_frame= */ (640 * 360),
/* target_pixels_per_frame= */ std::nullopt,
/* max_frame_rate= */ std::nullopt);
VideoEncoderConfig encoder_config;
encoder_config.number_of_streams = 3;
encoder_config.simulcast_layers.resize(3);
encoder_config.simulcast_layers[0].requested_resolution = {.width = 320,
.height = 180};
encoder_config.simulcast_layers[1].requested_resolution = {.width = 640,
.height = 360};
encoder_config.simulcast_layers[2].requested_resolution = {.width = 1280,
.height = 720};
auto streams =
CreateEncoderStreams(field_trials, {.width = 1280, .height = 720},
encoder_config, restrictions);
std::vector<Resolution> stream_resolutions = GetStreamResolutions(streams);
// 720p layer is dropped due to 360p restrictions.
ASSERT_THAT(stream_resolutions, SizeIs(2));
EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 320, .height = 180}));
EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360}));
}
TEST(EncoderStreamFactory, SimulcastRequestedResolutionWith90pRestriction) {
ExplicitKeyValueConfig field_trials("");
VideoSourceRestrictions restrictions(
/* max_pixels_per_frame= */ (160 * 90),
/* target_pixels_per_frame= */ std::nullopt,
/* max_frame_rate= */ std::nullopt);
VideoEncoderConfig encoder_config;
encoder_config.number_of_streams = 3;
encoder_config.simulcast_layers.resize(3);
encoder_config.simulcast_layers[0].requested_resolution = {.width = 320,
.height = 180};
encoder_config.simulcast_layers[1].requested_resolution = {.width = 640,
.height = 360};
encoder_config.simulcast_layers[2].requested_resolution = {.width = 1280,
.height = 720};
auto streams =
CreateEncoderStreams(field_trials, {.width = 1280, .height = 720},
encoder_config, restrictions);
std::vector<Resolution> stream_resolutions = GetStreamResolutions(streams);
ASSERT_THAT(stream_resolutions, SizeIs(1));
// 90p restriction means all but the first layer (180p) is dropped. The one
// and only layer is downsized to 90p.
EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 160, .height = 90}));
}
TEST(EncoderStreamFactory, ReverseSimulcastRequestedResolutionWithRestriction) {
ExplicitKeyValueConfig field_trials("");
VideoSourceRestrictions restrictions(
/* max_pixels_per_frame= */ (640 * 360),
/* target_pixels_per_frame= */ std::nullopt,
/* max_frame_rate= */ std::nullopt);
VideoEncoderConfig encoder_config;
encoder_config.number_of_streams = 3;
encoder_config.simulcast_layers.resize(3);
// 720p, 360p, 180p (instead of the usual 180p, 360p, 720p).
encoder_config.simulcast_layers[0].requested_resolution = {.width = 1280,
.height = 720};
encoder_config.simulcast_layers[1].requested_resolution = {.width = 640,
.height = 360};
encoder_config.simulcast_layers[2].requested_resolution = {.width = 320,
.height = 180};
auto streams =
CreateEncoderStreams(field_trials, {.width = 1280, .height = 720},
encoder_config, restrictions);
std::vector<Resolution> stream_resolutions = GetStreamResolutions(streams);
// The layer dropping that is performed for lower-to-higher ordered simulcast
// streams is not applicable when higher-to-lower order is used. In this case
// the 360p restriction is applied to all layers.
ASSERT_THAT(stream_resolutions, SizeIs(3));
EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 640, .height = 360}));
EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360}));
EXPECT_EQ(stream_resolutions[2], (Resolution{.width = 320, .height = 180}));
}
TEST(EncoderStreamFactory, BitratePriority) {
constexpr double kBitratePriority = 0.123;
VideoEncoderConfig encoder_config;