From 8c007fffead0f29650d6e34c10c85605e4dd33df Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Fri, 22 Jan 2021 18:59:59 +0100 Subject: [PATCH] Restrict usage of resolution bitrate limits to singlecast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: none Change-Id: I4d0726d45a517b51eae124dc23e533910ede7cc7 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/203262 Commit-Queue: Sergey Silkin Reviewed-by: Åsa Persson Cr-Commit-Position: refs/heads/master@{#33061} --- video/adaptation/BUILD.gn | 1 + video/adaptation/bitrate_constraint.cc | 57 ++++- .../adaptation/bitrate_constraint_unittest.cc | 223 ++++++++++++++++++ .../video_stream_encoder_resource_manager.cc | 53 +++-- .../video_stream_encoder_resource_manager.h | 3 + 5 files changed, 301 insertions(+), 36 deletions(-) create mode 100644 video/adaptation/bitrate_constraint_unittest.cc diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index c5afb02c83..b908ac3425 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -75,6 +75,7 @@ if (rtc_include_tests) { defines = [] sources = [ + "bitrate_constraint_unittest.cc", "overuse_frame_detector_unittest.cc", "pixel_limit_resource_unittest.cc", "quality_scaler_resource_unittest.cc", diff --git a/video/adaptation/bitrate_constraint.cc b/video/adaptation/bitrate_constraint.cc index 1061c4557f..28b5058747 100644 --- a/video/adaptation/bitrate_constraint.cc +++ b/video/adaptation/bitrate_constraint.cc @@ -10,13 +10,32 @@ #include #include +#include #include "call/adaptation/video_stream_adapter.h" #include "rtc_base/synchronization/sequence_checker.h" #include "video/adaptation/bitrate_constraint.h" +#include "video/adaptation/video_stream_encoder_resource_manager.h" namespace webrtc { +namespace { +bool IsSimulcast(const VideoEncoderConfig& encoder_config) { + const std::vector& simulcast_layers = + encoder_config.simulcast_layers; + + bool is_simulcast = simulcast_layers.size() > 1; + bool is_lowest_layer_active = simulcast_layers[0].active; + int num_active_layers = + std::count_if(simulcast_layers.begin(), simulcast_layers.end(), + [](const VideoStream& layer) { return layer.active; }); + + // We can't distinguish between simulcast and singlecast when only the + // lowest spatial layer is active. Treat this case as simulcast. + return is_simulcast && (num_active_layers > 1 || is_lowest_layer_active); +} +} // namespace + BitrateConstraint::BitrateConstraint() : encoder_settings_(absl::nullopt), encoder_target_bitrate_bps_(absl::nullopt) { @@ -42,19 +61,35 @@ bool BitrateConstraint::IsAdaptationUpAllowed( RTC_DCHECK_RUN_ON(&sequence_checker_); // Make sure bitrate limits are not violated. if (DidIncreaseResolution(restrictions_before, restrictions_after)) { + if (!encoder_settings_.has_value()) { + return true; + } + uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0); + if (bitrate_bps == 0) { + return true; + } + + if (IsSimulcast(encoder_settings_->encoder_config())) { + // Resolution bitrate limits usage is restricted to singlecast. + return true; + } + + absl::optional current_frame_size_px = + VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels( + encoder_settings_->video_codec()); + if (!current_frame_size_px.has_value()) { + return true; + } + absl::optional bitrate_limits = - encoder_settings_.has_value() - ? encoder_settings_->encoder_info() - .GetEncoderBitrateLimitsForResolution( - // Need some sort of expected resulting pixels to be used - // instead of unrestricted. - GetHigherResolutionThan( - input_state.frame_size_pixels().value())) - : absl::nullopt; - if (bitrate_limits.has_value() && bitrate_bps != 0) { - RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, - input_state.frame_size_pixels().value()); + encoder_settings_->encoder_info().GetEncoderBitrateLimitsForResolution( + // Need some sort of expected resulting pixels to be used + // instead of unrestricted. + GetHigherResolutionThan(*current_frame_size_px)); + + if (bitrate_limits.has_value()) { + RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, *current_frame_size_px); return bitrate_bps >= static_cast(bitrate_limits->min_start_bitrate_bps); } diff --git a/video/adaptation/bitrate_constraint_unittest.cc b/video/adaptation/bitrate_constraint_unittest.cc new file mode 100644 index 0000000000..e60418f266 --- /dev/null +++ b/video/adaptation/bitrate_constraint_unittest.cc @@ -0,0 +1,223 @@ +/* + * Copyright 2021 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/bitrate_constraint.h" + +#include +#include +#include + +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" +#include "test/gtest.h" + +namespace webrtc { + +using ResolutionBitrateLimits = VideoEncoder::ResolutionBitrateLimits; + +namespace { + +void FillCodecConfig(VideoCodec* video_codec, + VideoEncoderConfig* encoder_config, + int width_px, + int height_px, + std::vector active_flags) { + size_t num_layers = active_flags.size(); + video_codec->codecType = kVideoCodecVP8; + video_codec->numberOfSimulcastStreams = num_layers; + + encoder_config->number_of_streams = num_layers; + encoder_config->simulcast_layers.resize(num_layers); + + for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) { + int layer_width_px = width_px >> (num_layers - 1 - layer_idx); + int layer_height_px = height_px >> (num_layers - 1 - layer_idx); + + video_codec->simulcastStream[layer_idx].active = active_flags[layer_idx]; + video_codec->simulcastStream[layer_idx].width = layer_width_px; + video_codec->simulcastStream[layer_idx].height = layer_height_px; + + encoder_config->simulcast_layers[layer_idx].active = + active_flags[layer_idx]; + encoder_config->simulcast_layers[layer_idx].width = layer_width_px; + encoder_config->simulcast_layers[layer_idx].height = layer_height_px; + } +} + +VideoEncoder::EncoderInfo MakeEncoderInfo() { + VideoEncoder::EncoderInfo encoder_info; + encoder_info.resolution_bitrate_limits = std::vector( + {ResolutionBitrateLimits(640 * 360, 500000, 0, 5000000), + ResolutionBitrateLimits(1280 * 720, 1000000, 0, 5000000), + ResolutionBitrateLimits(1920 * 1080, 2000000, 0, 5000000)}); + return encoder_info; +} +} // namespace + +TEST(BitrateConstraintTest, AdaptUpAllowedAtSinglecastIfBitrateIsEnough) { + VideoCodec video_codec; + VideoEncoderConfig encoder_config; + FillCodecConfig(&video_codec, &encoder_config, + /*width_px=*/640, /*height_px=*/360, + /*active_flags=*/{true}); + + EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config), + video_codec); + + BitrateConstraint bitrate_constraint; + bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings); + bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000); + + VideoSourceRestrictions restrictions_before( + /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360, + /*max_frame_rate=*/30); + VideoSourceRestrictions restrictions_after( + /*max_pixels_per_frame=*/1280 * 720, + /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30); + + EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed( + VideoStreamInputState(), restrictions_before, restrictions_after)); +} + +TEST(BitrateConstraintTest, AdaptUpDisallowedAtSinglecastIfBitrateIsNotEnough) { + VideoCodec video_codec; + VideoEncoderConfig encoder_config; + FillCodecConfig(&video_codec, &encoder_config, + /*width_px=*/640, /*height_px=*/360, + /*active_flags=*/{true}); + + EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config), + video_codec); + + BitrateConstraint bitrate_constraint; + bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings); + // 1 bps less than needed for 720p. + bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1); + + VideoSourceRestrictions restrictions_before( + /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360, + /*max_frame_rate=*/30); + VideoSourceRestrictions restrictions_after( + /*max_pixels_per_frame=*/1280 * 720, + /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30); + + EXPECT_FALSE(bitrate_constraint.IsAdaptationUpAllowed( + VideoStreamInputState(), restrictions_before, restrictions_after)); +} + +TEST(BitrateConstraintTest, + AdaptUpAllowedAtSinglecastUpperLayerActiveIfBitrateIsEnough) { + VideoCodec video_codec; + VideoEncoderConfig encoder_config; + FillCodecConfig(&video_codec, &encoder_config, + /*width_px=*/640, /*height_px=*/360, + /*active_flags=*/{false, true}); + + EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config), + video_codec); + + BitrateConstraint bitrate_constraint; + bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings); + bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000); + + VideoSourceRestrictions restrictions_before( + /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360, + /*max_frame_rate=*/30); + VideoSourceRestrictions restrictions_after( + /*max_pixels_per_frame=*/1280 * 720, + /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30); + + EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed( + VideoStreamInputState(), restrictions_before, restrictions_after)); +} + +TEST(BitrateConstraintTest, + AdaptUpDisallowedAtSinglecastUpperLayerActiveIfBitrateIsNotEnough) { + VideoCodec video_codec; + VideoEncoderConfig encoder_config; + FillCodecConfig(&video_codec, &encoder_config, + /*width_px=*/640, /*height_px=*/360, + /*active_flags=*/{false, true}); + + EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config), + video_codec); + + BitrateConstraint bitrate_constraint; + bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings); + // 1 bps less than needed for 720p. + bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1); + + VideoSourceRestrictions restrictions_before( + /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360, + /*max_frame_rate=*/30); + VideoSourceRestrictions restrictions_after( + /*max_pixels_per_frame=*/1280 * 720, + /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30); + + EXPECT_FALSE(bitrate_constraint.IsAdaptationUpAllowed( + VideoStreamInputState(), restrictions_before, restrictions_after)); +} + +TEST(BitrateConstraintTest, + AdaptUpAllowedAtSinglecastLowestLayerActiveIfBitrateIsNotEnough) { + VideoCodec video_codec; + VideoEncoderConfig encoder_config; + FillCodecConfig(&video_codec, &encoder_config, + /*width_px=*/640, /*height_px=*/360, + /*active_flags=*/{true, false}); + + EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config), + video_codec); + + BitrateConstraint bitrate_constraint; + bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings); + // 1 bps less than needed for 720p. + bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1); + + VideoSourceRestrictions restrictions_before( + /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360, + /*max_frame_rate=*/30); + VideoSourceRestrictions restrictions_after( + /*max_pixels_per_frame=*/1280 * 720, + /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30); + + EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed( + VideoStreamInputState(), restrictions_before, restrictions_after)); +} + +TEST(BitrateConstraintTest, AdaptUpAllowedAtSimulcastIfBitrateIsNotEnough) { + VideoCodec video_codec; + VideoEncoderConfig encoder_config; + FillCodecConfig(&video_codec, &encoder_config, + /*width_px=*/640, /*height_px=*/360, + /*active_flags=*/{true, true}); + + EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config), + video_codec); + + BitrateConstraint bitrate_constraint; + bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings); + // 1 bps less than needed for 720p. + bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1); + + VideoSourceRestrictions restrictions_before( + /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360, + /*max_frame_rate=*/30); + VideoSourceRestrictions restrictions_after( + /*max_pixels_per_frame=*/1280 * 720, + /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30); + + EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed( + VideoStreamInputState(), restrictions_before, restrictions_after)); +} + +} // namespace webrtc diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 81d23a124b..96f888d4f6 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -64,30 +64,6 @@ std::string ToString(VideoAdaptationReason reason) { RTC_CHECK_NOTREACHED(); } -absl::optional GetSingleActiveStreamPixels(const VideoCodec& codec) { - int num_active = 0; - absl::optional pixels; - if (codec.codecType == VideoCodecType::kVideoCodecVP9) { - for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { - if (codec.spatialLayers[i].active) { - ++num_active; - pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height; - } - } - } else { - for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { - if (codec.simulcastStream[i].active) { - ++num_active; - pixels = - codec.simulcastStream[i].width * codec.simulcastStream[i].height; - } - } - } - if (num_active > 1) - return absl::nullopt; - return pixels; -} - std::vector GetActiveLayersFlags(const VideoCodec& codec) { std::vector flags; if (codec.codecType == VideoCodecType::kVideoCodecVP9) { @@ -188,7 +164,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { last_active_flags_ = active_flags; last_input_width_ = codec.width; last_input_height_ = codec.height; - single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec); + single_active_stream_pixels_ = GetSingleActiveLayerPixels(codec); } void OnFrameDroppedDueToSize() { ++initial_framedrop_; } @@ -703,4 +679,31 @@ void VideoStreamEncoderResourceManager::OnQualityRampUp() { stream_adapter_->ClearRestrictions(); quality_rampup_experiment_.reset(); } + +absl::optional +VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels( + const VideoCodec& codec) { + int num_active = 0; + absl::optional pixels; + if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { + if (codec.spatialLayers[i].active) { + ++num_active; + pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height; + } + } + } else { + for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { + if (codec.simulcastStream[i].active) { + ++num_active; + pixels = + codec.simulcastStream[i].width * codec.simulcastStream[i].height; + } + } + } + if (num_active > 1) + return absl::nullopt; + return pixels; +} + } // namespace webrtc diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 7e458a92b3..d6b9dd1910 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -146,6 +146,9 @@ class VideoStreamEncoderResourceManager // QualityRampUpExperimentListener implementation. void OnQualityRampUp() override; + static absl::optional GetSingleActiveLayerPixels( + const VideoCodec& codec); + private: class InitialFrameDropper;