From a7e34d33fe70363acdabd41fe14f91469cdc1619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Wed, 20 Jan 2021 15:36:13 +0100 Subject: [PATCH] Add resolution_bitrate_limits to EncoderInfo field trial. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added class EncoderInfoSettings for parsing settings. Added use of class to SimulcastEncoderAdapter. Bug: none Change-Id: I8182b2ab43f0c330ebdf077e9f7cbc79247da90e Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/202246 Commit-Queue: Åsa Persson Reviewed-by: Sergey Silkin Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#33050} --- media/BUILD.gn | 2 +- media/engine/simulcast_encoder_adapter.cc | 22 +- media/engine/simulcast_encoder_adapter.h | 10 +- .../simulcast_encoder_adapter_unittest.cc | 17 +- rtc_base/experiments/BUILD.gn | 16 ++ rtc_base/experiments/encoder_info_settings.cc | 78 +++++++ rtc_base/experiments/encoder_info_settings.h | 62 ++++++ .../encoder_info_settings_unittest.cc | 91 ++++++++ video/video_stream_encoder.cc | 151 +++++++++---- video/video_stream_encoder_unittest.cc | 199 +++++++++++++++++- 10 files changed, 586 insertions(+), 62 deletions(-) create mode 100644 rtc_base/experiments/encoder_info_settings.cc create mode 100644 rtc_base/experiments/encoder_info_settings.h create mode 100644 rtc_base/experiments/encoder_info_settings_unittest.cc diff --git a/media/BUILD.gn b/media/BUILD.gn index 84f938b2d2..4843b892cc 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -193,7 +193,7 @@ rtc_library("rtc_simulcast_encoder_adapter") { "../modules/video_coding:video_coding_utility", "../rtc_base:checks", "../rtc_base:rtc_base_approved", - "../rtc_base/experiments:field_trial_parser", + "../rtc_base/experiments:encoder_info_settings", "../rtc_base/experiments:rate_control_settings", "../rtc_base/synchronization:sequence_checker", "../rtc_base/system:no_unique_address", diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 10cf686329..52790f9af0 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -228,11 +228,6 @@ SimulcastEncoderAdapter::SimulcastEncoderAdapter( "WebRTC-Video-PreferTemporalSupportOnBaseLayer")) { RTC_DCHECK(primary_factory); - ParseFieldTrial({&requested_resolution_alignment_override_, - &apply_alignment_to_all_simulcast_layers_override_}, - field_trial::FindFullName( - "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride")); - // The adapter is typically created on the worker thread, but operated on // the encoder task queue. encoder_queue_.Detach(); @@ -430,8 +425,9 @@ int SimulcastEncoderAdapter::Encode( return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - if (requested_resolution_alignment_override_) { - const int alignment = *requested_resolution_alignment_override_; + if (encoder_info_override_.requested_resolution_alignment()) { + const int alignment = + *encoder_info_override_.requested_resolution_alignment(); if (input_image.width() % alignment != 0 || input_image.height() % alignment != 0) { RTC_LOG(LS_WARNING) << "Frame " << input_image.width() << "x" @@ -439,7 +435,7 @@ int SimulcastEncoderAdapter::Encode( << alignment; return WEBRTC_VIDEO_CODEC_ERROR; } - if (apply_alignment_to_all_simulcast_layers_override_.Get()) { + if (encoder_info_override_.apply_alignment_to_all_simulcast_layers()) { for (const auto& layer : encoder_contexts_) { if (layer.width() % alignment != 0 || layer.height() % alignment != 0) { RTC_LOG(LS_WARNING) @@ -741,11 +737,15 @@ void SimulcastEncoderAdapter::DestroyStoredEncoders() { void SimulcastEncoderAdapter::OverrideFromFieldTrial( VideoEncoder::EncoderInfo* info) const { - if (requested_resolution_alignment_override_) { + if (encoder_info_override_.requested_resolution_alignment()) { info->requested_resolution_alignment = - *requested_resolution_alignment_override_; + *encoder_info_override_.requested_resolution_alignment(); info->apply_alignment_to_all_simulcast_layers = - apply_alignment_to_all_simulcast_layers_override_.Get(); + encoder_info_override_.apply_alignment_to_all_simulcast_layers(); + } + if (!encoder_info_override_.resolution_bitrate_limits().empty()) { + info->resolution_bitrate_limits = + encoder_info_override_.resolution_bitrate_limits(); } } diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h index d3d5d17d66..c127fee37a 100644 --- a/media/engine/simulcast_encoder_adapter.h +++ b/media/engine/simulcast_encoder_adapter.h @@ -26,7 +26,7 @@ #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/utility/framerate_controller.h" #include "rtc_base/atomic_ops.h" -#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/experiments/encoder_info_settings.h" #include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/system/rtc_export.h" @@ -162,13 +162,7 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { const bool boost_base_layer_quality_; const bool prefer_temporal_support_on_base_layer_; - // Overrides from field trial. - // EncoderInfo::requested_resolution_alignment. - FieldTrialOptional requested_resolution_alignment_override_{ - "requested_resolution_alignment"}; - // EncoderInfo::apply_alignment_to_all_simulcast_layers. - FieldTrialFlag apply_alignment_to_all_simulcast_layers_override_{ - "apply_alignment_to_all_simulcast_layers"}; + const SimulcastEncoderAdapterEncoderInfoSettings encoder_info_override_; }; } // namespace webrtc diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index 510db6fb5d..8a64ba4ddd 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -1292,7 +1292,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, adapter_->GetEncoderInfo().apply_alignment_to_all_simulcast_layers); } -TEST_F(TestSimulcastEncoderAdapterFake, AlignmentFromFieldTrial) { +TEST_F(TestSimulcastEncoderAdapterFake, EncoderInfoFromFieldTrial) { test::ScopedFieldTrials field_trials( "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride/" "requested_resolution_alignment:8," @@ -1308,13 +1308,18 @@ TEST_F(TestSimulcastEncoderAdapterFake, AlignmentFromFieldTrial) { EXPECT_EQ(8, adapter_->GetEncoderInfo().requested_resolution_alignment); EXPECT_TRUE( adapter_->GetEncoderInfo().apply_alignment_to_all_simulcast_layers); + EXPECT_TRUE(adapter_->GetEncoderInfo().resolution_bitrate_limits.empty()); } TEST_F(TestSimulcastEncoderAdapterFake, - AlignmentFromFieldTrialForSingleStream) { + EncoderInfoFromFieldTrialForSingleStream) { test::ScopedFieldTrials field_trials( "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride/" - "requested_resolution_alignment:9/"); + "requested_resolution_alignment:9," + "frame_size_pixels:123|456|789," + "min_start_bitrate_bps:11000|22000|33000," + "min_bitrate_bps:44000|55000|66000," + "max_bitrate_bps:77000|88000|99000/"); SetUp(); SimulcastTestFixtureImpl::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile), @@ -1326,6 +1331,12 @@ TEST_F(TestSimulcastEncoderAdapterFake, EXPECT_EQ(9, adapter_->GetEncoderInfo().requested_resolution_alignment); EXPECT_FALSE( adapter_->GetEncoderInfo().apply_alignment_to_all_simulcast_layers); + EXPECT_THAT( + adapter_->GetEncoderInfo().resolution_bitrate_limits, + ::testing::ElementsAre( + VideoEncoder::ResolutionBitrateLimits{123, 11000, 44000, 77000}, + VideoEncoder::ResolutionBitrateLimits{456, 22000, 55000, 88000}, + VideoEncoder::ResolutionBitrateLimits{789, 33000, 66000, 99000})); } TEST_F(TestSimulcastEncoderAdapterFake, ReportsInternalSource) { diff --git a/rtc_base/experiments/BUILD.gn b/rtc_base/experiments/BUILD.gn index a40c9e0d80..3ea815d36d 100644 --- a/rtc_base/experiments/BUILD.gn +++ b/rtc_base/experiments/BUILD.gn @@ -130,6 +130,20 @@ rtc_library("cpu_speed_experiment") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } +rtc_library("encoder_info_settings") { + sources = [ + "encoder_info_settings.cc", + "encoder_info_settings.h", + ] + deps = [ + ":field_trial_parser", + "../:rtc_base_approved", + "../../api/video_codecs:video_codecs_api", + "../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_library("rtt_mult_experiment") { sources = [ "rtt_mult_experiment.cc", @@ -224,6 +238,7 @@ if (rtc_include_tests) { sources = [ "balanced_degradation_settings_unittest.cc", "cpu_speed_experiment_unittest.cc", + "encoder_info_settings_unittest.cc", "field_trial_list_unittest.cc", "field_trial_parser_unittest.cc", "field_trial_units_unittest.cc", @@ -241,6 +256,7 @@ if (rtc_include_tests) { deps = [ ":balanced_degradation_settings", ":cpu_speed_experiment", + ":encoder_info_settings", ":field_trial_parser", ":keyframe_interval_settings_experiment", ":min_video_bitrate_experiment", diff --git a/rtc_base/experiments/encoder_info_settings.cc b/rtc_base/experiments/encoder_info_settings.cc new file mode 100644 index 0000000000..fa6b843fb9 --- /dev/null +++ b/rtc_base/experiments/encoder_info_settings.cc @@ -0,0 +1,78 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/experiments/encoder_info_settings.h" + +#include + +#include "rtc_base/experiments/field_trial_list.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace { + +std::vector ToResolutionBitrateLimits( + const std::vector& limits) { + std::vector result; + for (const auto& limit : limits) { + result.push_back(VideoEncoder::ResolutionBitrateLimits( + limit.frame_size_pixels, limit.min_start_bitrate_bps, + limit.min_bitrate_bps, limit.max_bitrate_bps)); + } + return result; +} + +} // namespace + +EncoderInfoSettings::EncoderInfoSettings(std::string name) + : requested_resolution_alignment_("requested_resolution_alignment"), + apply_alignment_to_all_simulcast_layers_( + "apply_alignment_to_all_simulcast_layers") { + FieldTrialStructList bitrate_limits( + {FieldTrialStructMember( + "frame_size_pixels", + [](BitrateLimit* b) { return &b->frame_size_pixels; }), + FieldTrialStructMember( + "min_start_bitrate_bps", + [](BitrateLimit* b) { return &b->min_start_bitrate_bps; }), + FieldTrialStructMember( + "min_bitrate_bps", + [](BitrateLimit* b) { return &b->min_bitrate_bps; }), + FieldTrialStructMember( + "max_bitrate_bps", + [](BitrateLimit* b) { return &b->max_bitrate_bps; })}, + {}); + + ParseFieldTrial({&bitrate_limits, &requested_resolution_alignment_, + &apply_alignment_to_all_simulcast_layers_}, + field_trial::FindFullName(name)); + + resolution_bitrate_limits_ = ToResolutionBitrateLimits(bitrate_limits.Get()); +} + +absl::optional EncoderInfoSettings::requested_resolution_alignment() + const { + if (requested_resolution_alignment_ && + requested_resolution_alignment_.Value() < 1) { + RTC_LOG(LS_WARNING) << "Unsupported alignment value, ignored."; + return absl::nullopt; + } + return requested_resolution_alignment_.GetOptional(); +} + +EncoderInfoSettings::~EncoderInfoSettings() {} + +SimulcastEncoderAdapterEncoderInfoSettings:: + SimulcastEncoderAdapterEncoderInfoSettings() + : EncoderInfoSettings( + "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride") {} + +} // namespace webrtc diff --git a/rtc_base/experiments/encoder_info_settings.h b/rtc_base/experiments/encoder_info_settings.h new file mode 100644 index 0000000000..9cacdbd2a5 --- /dev/null +++ b/rtc_base/experiments/encoder_info_settings.h @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_EXPERIMENTS_ENCODER_INFO_SETTINGS_H_ +#define RTC_BASE_EXPERIMENTS_ENCODER_INFO_SETTINGS_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/experiments/field_trial_parser.h" + +namespace webrtc { + +class EncoderInfoSettings { + public: + virtual ~EncoderInfoSettings(); + + // Bitrate limits per resolution. + struct BitrateLimit { + int frame_size_pixels = 0; // The video frame size. + int min_start_bitrate_bps = 0; // The minimum bitrate to start encoding. + int min_bitrate_bps = 0; // The minimum bitrate. + int max_bitrate_bps = 0; // The maximum bitrate. + }; + + absl::optional requested_resolution_alignment() const; + bool apply_alignment_to_all_simulcast_layers() const { + return apply_alignment_to_all_simulcast_layers_.Get(); + } + std::vector resolution_bitrate_limits() + const { + return resolution_bitrate_limits_; + } + + protected: + explicit EncoderInfoSettings(std::string name); + + private: + FieldTrialOptional requested_resolution_alignment_; + FieldTrialFlag apply_alignment_to_all_simulcast_layers_; + std::vector resolution_bitrate_limits_; +}; + +// EncoderInfo settings for SimulcastEncoderAdapter. +class SimulcastEncoderAdapterEncoderInfoSettings : public EncoderInfoSettings { + public: + SimulcastEncoderAdapterEncoderInfoSettings(); + ~SimulcastEncoderAdapterEncoderInfoSettings() override {} +}; + +} // namespace webrtc + +#endif // RTC_BASE_EXPERIMENTS_ENCODER_INFO_SETTINGS_H_ diff --git a/rtc_base/experiments/encoder_info_settings_unittest.cc b/rtc_base/experiments/encoder_info_settings_unittest.cc new file mode 100644 index 0000000000..0208c0dd66 --- /dev/null +++ b/rtc_base/experiments/encoder_info_settings_unittest.cc @@ -0,0 +1,91 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/experiments/encoder_info_settings.h" + +#include "rtc_base/gunit.h" +#include "test/field_trial.h" +#include "test/gmock.h" + +namespace webrtc { + +TEST(SimulcastEncoderAdapterSettingsTest, NoValuesWithoutFieldTrial) { + SimulcastEncoderAdapterEncoderInfoSettings settings; + EXPECT_EQ(absl::nullopt, settings.requested_resolution_alignment()); + EXPECT_FALSE(settings.apply_alignment_to_all_simulcast_layers()); + EXPECT_TRUE(settings.resolution_bitrate_limits().empty()); +} + +TEST(SimulcastEncoderAdapterSettingsTest, NoValueForInvalidAlignment) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride/" + "requested_resolution_alignment:0/"); + + SimulcastEncoderAdapterEncoderInfoSettings settings; + EXPECT_EQ(absl::nullopt, settings.requested_resolution_alignment()); +} + +TEST(SimulcastEncoderAdapterSettingsTest, GetResolutionAlignment) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride/" + "requested_resolution_alignment:2/"); + + SimulcastEncoderAdapterEncoderInfoSettings settings; + EXPECT_EQ(2, settings.requested_resolution_alignment()); + EXPECT_FALSE(settings.apply_alignment_to_all_simulcast_layers()); + EXPECT_TRUE(settings.resolution_bitrate_limits().empty()); +} + +TEST(SimulcastEncoderAdapterSettingsTest, GetApplyAlignment) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride/" + "requested_resolution_alignment:3," + "apply_alignment_to_all_simulcast_layers/"); + + SimulcastEncoderAdapterEncoderInfoSettings settings; + EXPECT_EQ(3, settings.requested_resolution_alignment()); + EXPECT_TRUE(settings.apply_alignment_to_all_simulcast_layers()); + EXPECT_TRUE(settings.resolution_bitrate_limits().empty()); +} + +TEST(SimulcastEncoderAdapterSettingsTest, GetResolutionBitrateLimits) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride/" + "frame_size_pixels:123," + "min_start_bitrate_bps:11000," + "min_bitrate_bps:44000," + "max_bitrate_bps:77000/"); + + SimulcastEncoderAdapterEncoderInfoSettings settings; + EXPECT_EQ(absl::nullopt, settings.requested_resolution_alignment()); + EXPECT_FALSE(settings.apply_alignment_to_all_simulcast_layers()); + EXPECT_THAT(settings.resolution_bitrate_limits(), + ::testing::ElementsAre(VideoEncoder::ResolutionBitrateLimits{ + 123, 11000, 44000, 77000})); +} + +TEST(SimulcastEncoderAdapterSettingsTest, GetResolutionBitrateLimitsWithList) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride/" + "frame_size_pixels:123|456|789," + "min_start_bitrate_bps:11000|22000|33000," + "min_bitrate_bps:44000|55000|66000," + "max_bitrate_bps:77000|88000|99000/"); + + SimulcastEncoderAdapterEncoderInfoSettings settings; + EXPECT_THAT( + settings.resolution_bitrate_limits(), + ::testing::ElementsAre( + VideoEncoder::ResolutionBitrateLimits{123, 11000, 44000, 77000}, + VideoEncoder::ResolutionBitrateLimits{456, 22000, 55000, 88000}, + VideoEncoder::ResolutionBitrateLimits{789, 33000, 66000, 99000})); +} + +} // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 3d87379749..9a05e47eda 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -337,6 +337,78 @@ VideoLayersAllocation CreateVideoLayersAllocation( return layers_allocation; } +int NumActiveStreams(const std::vector& streams) { + int num_active = 0; + for (const auto& stream : streams) { + if (stream.active) + ++num_active; + } + return num_active; +} + +void ApplyEncoderBitrateLimitsIfSingleActiveStream( + const VideoEncoder::EncoderInfo& encoder_info, + const std::vector& encoder_config_layers, + std::vector* streams) { + // Apply limits if simulcast with one active stream (expect lowest). + bool single_active_stream = + streams->size() > 1 && NumActiveStreams(*streams) == 1 && + !streams->front().active && NumActiveStreams(encoder_config_layers) == 1; + if (!single_active_stream) { + return; + } + + // Index for the active stream. + size_t index = 0; + for (size_t i = 0; i < encoder_config_layers.size(); ++i) { + if (encoder_config_layers[i].active) + index = i; + } + if (streams->size() < (index + 1) || !(*streams)[index].active) { + return; + } + + // Get bitrate limits for active stream. + absl::optional encoder_bitrate_limits = + encoder_info.GetEncoderBitrateLimitsForResolution( + (*streams)[index].width * (*streams)[index].height); + if (!encoder_bitrate_limits) { + return; + } + + // If bitrate limits are set by RtpEncodingParameters, use intersection. + int min_bitrate_bps; + if (encoder_config_layers[index].min_bitrate_bps <= 0) { + min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps; + } else { + min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps, + (*streams)[index].min_bitrate_bps); + } + int max_bitrate_bps; + if (encoder_config_layers[index].max_bitrate_bps <= 0) { + max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps; + } else { + max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps, + (*streams)[index].max_bitrate_bps); + } + if (min_bitrate_bps >= max_bitrate_bps) { + RTC_LOG(LS_WARNING) << "Encoder bitrate limits" + << " (min=" << encoder_bitrate_limits->min_bitrate_bps + << ", max=" << encoder_bitrate_limits->max_bitrate_bps + << ") do not intersect with stream limits" + << " (min=" << (*streams)[index].min_bitrate_bps + << ", max=" << (*streams)[index].max_bitrate_bps + << "). Encoder bitrate limits not used."; + return; + } + + (*streams)[index].min_bitrate_bps = min_bitrate_bps; + (*streams)[index].max_bitrate_bps = max_bitrate_bps; + (*streams)[index].target_bitrate_bps = + std::min((*streams)[index].target_bitrate_bps, + encoder_bitrate_limits->max_bitrate_bps); +} + } // namespace VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings() @@ -769,46 +841,51 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( last_frame_info_->width * last_frame_info_->height); - if (streams.size() == 1 && encoder_bitrate_limits_) { - // Bitrate limits can be set by app (in SDP or RtpEncodingParameters) or/and - // can be provided by encoder. In presence of both set of limits, the final - // set is derived as their intersection. - int min_bitrate_bps; - if (encoder_config_.simulcast_layers.empty() || - encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) { - min_bitrate_bps = encoder_bitrate_limits_->min_bitrate_bps; - } else { - min_bitrate_bps = std::max(encoder_bitrate_limits_->min_bitrate_bps, - streams.back().min_bitrate_bps); - } + if (encoder_bitrate_limits_) { + if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) { + // Bitrate limits can be set by app (in SDP or RtpEncodingParameters) + // or/and can be provided by encoder. In presence of both set of limits, + // the final set is derived as their intersection. + int min_bitrate_bps; + if (encoder_config_.simulcast_layers.empty() || + encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) { + min_bitrate_bps = encoder_bitrate_limits_->min_bitrate_bps; + } else { + min_bitrate_bps = std::max(encoder_bitrate_limits_->min_bitrate_bps, + streams.back().min_bitrate_bps); + } - int max_bitrate_bps; - // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps - // here since encoder_config_.max_bitrate_bps is derived from it (as - // well as from other inputs). - if (encoder_config_.max_bitrate_bps <= 0) { - max_bitrate_bps = encoder_bitrate_limits_->max_bitrate_bps; - } else { - max_bitrate_bps = std::min(encoder_bitrate_limits_->max_bitrate_bps, - streams.back().max_bitrate_bps); - } + int max_bitrate_bps; + // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps + // here since encoder_config_.max_bitrate_bps is derived from it (as + // well as from other inputs). + if (encoder_config_.max_bitrate_bps <= 0) { + max_bitrate_bps = encoder_bitrate_limits_->max_bitrate_bps; + } else { + max_bitrate_bps = std::min(encoder_bitrate_limits_->max_bitrate_bps, + streams.back().max_bitrate_bps); + } - if (min_bitrate_bps < max_bitrate_bps) { - streams.back().min_bitrate_bps = min_bitrate_bps; - streams.back().max_bitrate_bps = max_bitrate_bps; - streams.back().target_bitrate_bps = - std::min(streams.back().target_bitrate_bps, - encoder_bitrate_limits_->max_bitrate_bps); + if (min_bitrate_bps < max_bitrate_bps) { + streams.back().min_bitrate_bps = min_bitrate_bps; + streams.back().max_bitrate_bps = max_bitrate_bps; + streams.back().target_bitrate_bps = + std::min(streams.back().target_bitrate_bps, + encoder_bitrate_limits_->max_bitrate_bps); + } else { + RTC_LOG(LS_WARNING) + << "Bitrate limits provided by encoder" + << " (min=" << encoder_bitrate_limits_->min_bitrate_bps + << ", max=" << encoder_bitrate_limits_->min_bitrate_bps + << ") do not intersect with limits set by app" + << " (min=" << streams.back().min_bitrate_bps + << ", max=" << encoder_config_.max_bitrate_bps + << "). The app bitrate limits will be used."; + } } else { - RTC_LOG(LS_WARNING) << "Bitrate limits provided by encoder" - << " (min=" - << encoder_bitrate_limits_->min_bitrate_bps - << ", max=" - << encoder_bitrate_limits_->min_bitrate_bps - << ") do not intersect with limits set by app" - << " (min=" << streams.back().min_bitrate_bps - << ", max=" << encoder_config_.max_bitrate_bps - << "). The app bitrate limits will be used."; + ApplyEncoderBitrateLimitsIfSingleActiveStream( + encoder_->GetEncoderInfo(), encoder_config_.simulcast_layers, + &streams); } } diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 85be6951e5..dea22de1b3 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -2009,6 +2009,201 @@ TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) { + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p( + 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p( + 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000); + fake_encoder_.SetResolutionBitrateLimits( + {kEncoderLimits270p, kEncoderLimits360p}); + + // Two streams, highest stream active. + VideoEncoderConfig config; + const int kNumStreams = 2; + test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); + config.max_bitrate_bps = 0; + config.simulcast_layers[0].active = false; + config.simulcast_layers[1].active = true; + config.video_stream_factory = + new rtc::RefCountedObject( + "VP8", /*max qp*/ 56, /*screencast*/ false, + /*screenshare enabled*/ false); + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + + // The encoder bitrate limits for 270p should be used. + video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams); + EXPECT_EQ(static_cast(kEncoderLimits270p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits270p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + // The encoder bitrate limits for 360p should be used. + video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(static_cast(kEncoderLimits360p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits360p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + // Resolution b/w 270p and 360p. The encoder limits for 360p should be used. + video_source_.IncomingCapturedFrame( + CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(static_cast(kEncoderLimits360p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits360p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + // Resolution higher than 360p. Encoder limits should be ignored. + video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_NE(static_cast(kEncoderLimits270p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_NE(static_cast(kEncoderLimits270p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + EXPECT_NE(static_cast(kEncoderLimits360p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_NE(static_cast(kEncoderLimits360p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + // Resolution lower than 270p. The encoder limits for 270p should be used. + video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(static_cast(kEncoderLimits270p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits270p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) { + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p( + 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p( + 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p( + 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000); + fake_encoder_.SetResolutionBitrateLimits( + {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p}); + + // Three streams, middle stream active. + VideoEncoderConfig config; + const int kNumStreams = 3; + test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); + config.simulcast_layers[0].active = false; + config.simulcast_layers[1].active = true; + config.simulcast_layers[2].active = false; + config.video_stream_factory = + new rtc::RefCountedObject( + "VP8", /*max qp*/ 56, /*screencast*/ false, + /*screenshare enabled*/ false); + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + + // The encoder bitrate limits for 360p should be used. + video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams); + EXPECT_EQ(static_cast(kEncoderLimits360p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits360p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + // The encoder bitrate limits for 270p should be used. + video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(static_cast(kEncoderLimits270p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits270p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) { + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p( + 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p( + 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p( + 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000); + fake_encoder_.SetResolutionBitrateLimits( + {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p}); + + // Three streams, lowest stream active. + VideoEncoderConfig config; + const int kNumStreams = 3; + test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); + config.simulcast_layers[0].active = true; + config.simulcast_layers[1].active = false; + config.simulcast_layers[2].active = false; + config.video_stream_factory = + new rtc::RefCountedObject( + "VP8", /*max qp*/ 56, /*screencast*/ false, + /*screenshare enabled*/ false); + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + + // Resolution on lowest stream lower than 270p. The encoder limits not applied + // on lowest stream, limits for 270p should not be used + video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams); + EXPECT_NE(static_cast(kEncoderLimits270p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_NE(static_cast(kEncoderLimits270p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) { + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p( + 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p( + 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000); + fake_encoder_.SetResolutionBitrateLimits( + {kEncoderLimits270p, kEncoderLimits360p}); + const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000; + + // Two streams, highest stream active. + VideoEncoderConfig config; + const int kNumStreams = 2; + test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); + config.simulcast_layers[0].active = false; + config.simulcast_layers[1].active = true; + config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps; + config.video_stream_factory = + new rtc::RefCountedObject( + "VP8", /*max qp*/ 56, /*screencast*/ false, + /*screenshare enabled*/ false); + video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); + + // The encoder bitrate limits for 270p should be used. + video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams); + EXPECT_EQ(static_cast(kEncoderLimits270p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits270p.max_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + // The max configured bitrate is less than the encoder limit for 360p. + video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(static_cast(kEncoderLimits360p.min_bitrate_bps), + fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000); + EXPECT_EQ(static_cast(kMaxBitrateBps), + fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) { EXPECT_TRUE(video_source_.has_sinks()); test::FrameForwarder new_video_source; @@ -4091,7 +4286,7 @@ TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) { } TEST_F(VideoStreamEncoderTest, - ReportsVideoLayersAllocationForVP8WithMidleLayerDisabled) { + ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) { fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true); fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true); fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true); @@ -4136,7 +4331,7 @@ TEST_F(VideoStreamEncoderTest, } TEST_F(VideoStreamEncoderTest, - ReportsVideoLayersAllocationForVP8WithMidleAndHighestLayerDisabled) { + ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) { fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true); fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true); fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);