From 8efd93dd761135e58621f0a4204f8f87bf10f41e Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Fri, 4 Aug 2023 13:59:43 +0200 Subject: [PATCH] Encoder type agnostic resolution based fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WebRTC-Video-EncoderFallbackSettings/resolution_threshold_px:X sets resolution threshold to switch from primary to fallback encoder. When the field trial is present, VP8-specific resolution based fallback settings, provided by WebRTC-VP8-Forced-Fallback-Encoder-v2, are ignored. Bug: none Change-Id: I8f2e28438547f3896c7fc288ed6634720328f3a0 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/314760 Reviewed-by: Mirko Bonadei Reviewed-by: Rasmus Brandt Commit-Queue: Sergey Silkin Reviewed-by: Åsa Persson Cr-Commit-Position: refs/heads/main@{#40526} --- api/video_codecs/BUILD.gn | 2 + api/video_codecs/test/BUILD.gn | 1 + ...oder_software_fallback_wrapper_unittest.cc | 58 +++++++++++++++++++ ...video_encoder_software_fallback_wrapper.cc | 51 ++++++++++------ test/BUILD.gn | 5 +- test/fake_encoder.cc | 9 ++- test/fake_encoder.h | 7 ++- 7 files changed, 111 insertions(+), 22 deletions(-) diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index 214c161fab..20c54d541e 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -291,6 +291,7 @@ rtc_library("rtc_software_fallback_wrappers") { deps = [ ":video_codecs_api", "..:fec_controller_api", + "../../api/transport:field_trial_based_config", "../../api/video:video_frame", "../../media:rtc_media_base", "../../modules/video_coding:video_codec_interface", @@ -298,6 +299,7 @@ rtc_library("rtc_software_fallback_wrappers") { "../../rtc_base:checks", "../../rtc_base:event_tracer", "../../rtc_base:logging", + "../../rtc_base/experiments:field_trial_parser", "../../rtc_base/system:rtc_export", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", diff --git a/api/video_codecs/test/BUILD.gn b/api/video_codecs/test/BUILD.gn index 47d5ff9aa3..eda8c568af 100644 --- a/api/video_codecs/test/BUILD.gn +++ b/api/video_codecs/test/BUILD.gn @@ -35,6 +35,7 @@ if (rtc_include_tests) { "../../../modules/video_coding:webrtc_vp8", "../../../rtc_base:checks", "../../../rtc_base:rtc_base_tests_utils", + "../../../test:fake_video_codecs", "../../../test:field_trial", "../../../test:test_support", "../../../test:video_test_common", diff --git a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc index 08afbfbb01..b3fadcbecf 100644 --- a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc +++ b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc @@ -34,6 +34,7 @@ #include "modules/video_coding/include/video_error_codes.h" #include "modules/video_coding/utility/simulcast_rate_allocator.h" #include "rtc_base/fake_clock.h" +#include "test/fake_encoder.h" #include "test/fake_texture_frame.h" #include "test/field_trial.h" #include "test/gmock.h" @@ -42,6 +43,7 @@ namespace webrtc { using ::testing::_; using ::testing::Return; +using ::testing::ValuesIn; namespace { const int kWidth = 320; @@ -1083,4 +1085,60 @@ TEST_F(PreferTemporalLayersFallbackTest, PrimesEncoderOnSwitch) { EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); } +struct ResolutionBasedFallbackTestParams { + std::string test_name; + std::string field_trials = ""; + VideoCodecType codec_type = kVideoCodecGeneric; + int width = 16; + int height = 16; + std::string expect_implementation_name; +}; + +using ResolutionBasedFallbackTest = + ::testing::TestWithParam; + +INSTANTIATE_TEST_SUITE_P( + VideoEncoderFallbackTest, + ResolutionBasedFallbackTest, + ValuesIn( + {{.test_name = "FallbackNotConfigured", + .expect_implementation_name = "primary"}, + {.test_name = "ResolutionAboveFallbackThreshold", + .field_trials = "WebRTC-Video-EncoderFallbackSettings/" + "resolution_threshold_px:255/", + .expect_implementation_name = "primary"}, + {.test_name = "ResolutionEqualFallbackThreshold", + .field_trials = "WebRTC-Video-EncoderFallbackSettings/" + "resolution_threshold_px:256/", + .expect_implementation_name = "fallback"}, + {.test_name = "GenericFallbackSettingsTakePrecedence", + .field_trials = + "WebRTC-Video-EncoderFallbackSettings/" + "resolution_threshold_px:255/" + "WebRTC-VP8-Forced-Fallback-Encoder-v2/Enabled-1,256,1/", + .codec_type = kVideoCodecVP8, + .expect_implementation_name = "primary"}}), + [](const testing::TestParamInfo& + info) { return info.param.test_name; }); + +TEST_P(ResolutionBasedFallbackTest, VerifyForcedEncoderFallback) { + const ResolutionBasedFallbackTestParams& params = GetParam(); + test::ScopedFieldTrials field_trials(params.field_trials); + auto primary = new test::FakeEncoder(Clock::GetRealTimeClock()); + auto fallback = new test::FakeEncoder(Clock::GetRealTimeClock()); + auto encoder = CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr(fallback), + std::unique_ptr(primary), + /*prefer_temporal_support=*/false); + primary->SetImplementationName("primary"); + fallback->SetImplementationName("fallback"); + VideoCodec codec; + codec.codecType = params.codec_type; + codec.width = params.width; + codec.height = params.height; + encoder->InitEncode(&codec, kSettings); + EXPECT_EQ(encoder->GetEncoderInfo().implementation_name, + params.expect_implementation_name); +} + } // namespace webrtc diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc index 39c52a0081..d35c9f9950 100644 --- a/api/video_codecs/video_encoder_software_fallback_wrapper.cc +++ b/api/video_codecs/video_encoder_software_fallback_wrapper.cc @@ -20,6 +20,7 @@ #include "absl/strings/match.h" #include "absl/types/optional.h" #include "api/fec_controller_override.h" +#include "api/transport/field_trial_based_config.h" #include "api/video/i420_buffer.h" #include "api/video/video_bitrate_allocation.h" #include "api/video/video_frame.h" @@ -29,6 +30,7 @@ #include "modules/video_coding/include/video_error_codes.h" #include "modules/video_coding/utility/simulcast_utility.h" #include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/logging.h" #include "system_wrappers/include/field_trial.h" @@ -48,10 +50,18 @@ namespace { struct ForcedFallbackParams { public: bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const { - return enable_resolution_based_switch && - codec.codecType == kVideoCodecVP8 && - codec.numberOfSimulcastStreams <= 1 && - codec.width * codec.height <= max_pixels; + if (!enable_resolution_based_switch || + codec.width * codec.height > max_pixels) { + return false; + } + + if (vp8_specific_resolution_switch && + (codec.codecType != kVideoCodecVP8 || + codec.numberOfSimulcastStreams > 1)) { + return false; + } + + return true; } bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const { @@ -61,7 +71,8 @@ struct ForcedFallbackParams { bool enable_temporal_based_switch = false; bool enable_resolution_based_switch = false; - int min_pixels = 320 * 180; + bool vp8_specific_resolution_switch = false; + int min_pixels = kDefaultMinPixelsPerFrame; int max_pixels = 320 * 240; }; @@ -70,6 +81,19 @@ const char kVp8ForceFallbackEncoderFieldTrial[] = absl::optional ParseFallbackParamsFromFieldTrials( const VideoEncoder& main_encoder) { + // Ignore WebRTC-VP8-Forced-Fallback-Encoder-v2 if + // WebRTC-Video-EncoderFallbackSettings is present. + FieldTrialOptional resolution_threshold_px("resolution_threshold_px"); + ParseFieldTrial( + {&resolution_threshold_px}, + FieldTrialBasedConfig().Lookup("WebRTC-Video-EncoderFallbackSettings")); + if (resolution_threshold_px) { + ForcedFallbackParams params; + params.enable_resolution_based_switch = true; + params.max_pixels = resolution_threshold_px.Value(); + return params; + } + const std::string field_trial = webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial); if (!absl::StartsWith(field_trial, "Enabled")) { @@ -95,6 +119,7 @@ absl::optional ParseFallbackParamsFromFieldTrials( return absl::nullopt; } + params.vp8_specific_resolution_switch = true; return params; } @@ -107,7 +132,7 @@ absl::optional GetForcedFallbackParams( if (!params.has_value()) { params.emplace(); } - params->enable_temporal_based_switch = prefer_temporal_support; + params->enable_temporal_based_switch = true; } return params; } @@ -421,18 +446,8 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() fallback_encoder_info.apply_alignment_to_all_simulcast_layers || default_encoder_info.apply_alignment_to_all_simulcast_layers; - if (fallback_params_.has_value()) { - const auto settings = (encoder_state_ == EncoderState::kForcedFallback) - ? fallback_encoder_info.scaling_settings - : default_encoder_info.scaling_settings; - info.scaling_settings = - settings.thresholds - ? VideoEncoder::ScalingSettings(settings.thresholds->low, - settings.thresholds->high, - fallback_params_->min_pixels) - : VideoEncoder::ScalingSettings::kOff; - } else { - info.scaling_settings = default_encoder_info.scaling_settings; + if (fallback_params_ && fallback_params_->vp8_specific_resolution_switch) { + info.scaling_settings.min_pixels_per_frame = fallback_params_->min_pixels; } return info; diff --git a/test/BUILD.gn b/test/BUILD.gn index e60febb0a9..89b934e596 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -1022,7 +1022,10 @@ rtc_library("fake_video_codecs") { "../rtc_base/synchronization:mutex", "../system_wrappers", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] } rtc_library("null_transport") { diff --git a/test/fake_encoder.cc b/test/fake_encoder.cc index 79fda96638..f6a998f4e0 100644 --- a/test/fake_encoder.cc +++ b/test/fake_encoder.cc @@ -77,6 +77,11 @@ void FakeEncoder::SetQp(int qp) { qp_ = qp; } +void FakeEncoder::SetImplementationName(absl::string_view implementation_name) { + MutexLock lock(&mutex_); + implementation_name_ = std::string(implementation_name); +} + int32_t FakeEncoder::InitEncode(const VideoCodec* config, const Settings& settings) { MutexLock lock(&mutex_); @@ -274,9 +279,9 @@ void FakeEncoder::SetRatesLocked(const RateControlParameters& parameters) { const char* FakeEncoder::kImplementationName = "fake_encoder"; VideoEncoder::EncoderInfo FakeEncoder::GetEncoderInfo() const { EncoderInfo info; - info.implementation_name = kImplementationName; - info.is_hardware_accelerated = true; MutexLock lock(&mutex_); + info.implementation_name = implementation_name_.value_or(kImplementationName); + info.is_hardware_accelerated = true; for (int sid = 0; sid < config_.numberOfSimulcastStreams; ++sid) { int number_of_temporal_layers = config_.simulcastStream[sid].numberOfTemporalLayers; diff --git a/test/fake_encoder.h b/test/fake_encoder.h index 65e03155d7..02eeee0633 100644 --- a/test/fake_encoder.h +++ b/test/fake_encoder.h @@ -17,6 +17,7 @@ #include #include +#include "absl/strings/string_view.h" #include "api/fec_controller_override.h" #include "api/sequence_checker.h" #include "api/task_queue/task_queue_factory.h" @@ -42,6 +43,9 @@ class FakeEncoder : public VideoEncoder { void SetMaxBitrate(int max_kbps) RTC_LOCKS_EXCLUDED(mutex_); void SetQp(int qp) RTC_LOCKS_EXCLUDED(mutex_); + void SetImplementationName(absl::string_view implementation_name) + RTC_LOCKS_EXCLUDED(mutex_); + void SetFecControllerOverride( FecControllerOverride* fec_controller_override) override; @@ -55,7 +59,7 @@ class FakeEncoder : public VideoEncoder { int32_t Release() override; void SetRates(const RateControlParameters& parameters) RTC_LOCKS_EXCLUDED(mutex_) override; - EncoderInfo GetEncoderInfo() const override; + EncoderInfo GetEncoderInfo() const RTC_LOCKS_EXCLUDED(mutex_) override; int GetConfiguredInputFramerate() const RTC_LOCKS_EXCLUDED(mutex_); int GetNumInitializations() const RTC_LOCKS_EXCLUDED(mutex_); @@ -108,6 +112,7 @@ class FakeEncoder : public VideoEncoder { mutable Mutex mutex_; bool used_layers_[kMaxSimulcastStreams]; absl::optional qp_ RTC_GUARDED_BY(mutex_); + absl::optional implementation_name_ RTC_GUARDED_BY(mutex_); // Current byte debt to be payed over a number of frames. // The debt is acquired by keyframes overshooting the bitrate target.