Encoder type agnostic resolution based fallback

WebRTC-Video-EncoderFallbackSettings/resolution_threshold_px:X sets resolution threshold to switch from primary to fallback encoder. When the field trial is present, VP8-specific resolution based fallback settings, provided by WebRTC-VP8-Forced-Fallback-Encoder-v2, are ignored.

Bug: none
Change-Id: I8f2e28438547f3896c7fc288ed6634720328f3a0
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/314760
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#40526}

api/video_codecs/BUILD.gn

@@ -291,6 +291,7 @@ rtc_library("rtc_software_fallback_wrappers") {
   deps = [
     ":video_codecs_api",
     "..:fec_controller_api",
+    "../../api/transport:field_trial_based_config",
     "../../api/video:video_frame",
     "../../media:rtc_media_base",
     "../../modules/video_coding:video_codec_interface",
@@ -298,6 +299,7 @@ rtc_library("rtc_software_fallback_wrappers") {
     "../../rtc_base:checks",
     "../../rtc_base:event_tracer",
     "../../rtc_base:logging",
+    "../../rtc_base/experiments:field_trial_parser",
     "../../rtc_base/system:rtc_export",
     "../../system_wrappers:field_trial",
     "../../system_wrappers:metrics",

api/video_codecs/test/BUILD.gn

@@ -35,6 +35,7 @@ if (rtc_include_tests) {
       "../../../modules/video_coding:webrtc_vp8",
       "../../../rtc_base:checks",
       "../../../rtc_base:rtc_base_tests_utils",
+      "../../../test:fake_video_codecs",
       "../../../test:field_trial",
       "../../../test:test_support",
       "../../../test:video_test_common",

api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc

@@ -34,6 +34,7 @@
 #include "modules/video_coding/include/video_error_codes.h"
 #include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "rtc_base/fake_clock.h"
+#include "test/fake_encoder.h"
 #include "test/fake_texture_frame.h"
 #include "test/field_trial.h"
 #include "test/gmock.h"
@@ -42,6 +43,7 @@
 namespace webrtc {
 using ::testing::_;
 using ::testing::Return;
+using ::testing::ValuesIn;
 
 namespace {
 const int kWidth = 320;
@@ -1083,4 +1085,60 @@ TEST_F(PreferTemporalLayersFallbackTest, PrimesEncoderOnSwitch) {
   EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
 }
 
+struct ResolutionBasedFallbackTestParams {
+  std::string test_name;
+  std::string field_trials = "";
+  VideoCodecType codec_type = kVideoCodecGeneric;
+  int width = 16;
+  int height = 16;
+  std::string expect_implementation_name;
+};
+
+using ResolutionBasedFallbackTest =
+    ::testing::TestWithParam<ResolutionBasedFallbackTestParams>;
+
+INSTANTIATE_TEST_SUITE_P(
+    VideoEncoderFallbackTest,
+    ResolutionBasedFallbackTest,
+    ValuesIn<ResolutionBasedFallbackTestParams>(
+        {{.test_name = "FallbackNotConfigured",
+          .expect_implementation_name = "primary"},
+         {.test_name = "ResolutionAboveFallbackThreshold",
+          .field_trials = "WebRTC-Video-EncoderFallbackSettings/"
+                          "resolution_threshold_px:255/",
+          .expect_implementation_name = "primary"},
+         {.test_name = "ResolutionEqualFallbackThreshold",
+          .field_trials = "WebRTC-Video-EncoderFallbackSettings/"
+                          "resolution_threshold_px:256/",
+          .expect_implementation_name = "fallback"},
+         {.test_name = "GenericFallbackSettingsTakePrecedence",
+          .field_trials =
+              "WebRTC-Video-EncoderFallbackSettings/"
+              "resolution_threshold_px:255/"
+              "WebRTC-VP8-Forced-Fallback-Encoder-v2/Enabled-1,256,1/",
+          .codec_type = kVideoCodecVP8,
+          .expect_implementation_name = "primary"}}),
+    [](const testing::TestParamInfo<ResolutionBasedFallbackTest::ParamType>&
+           info) { return info.param.test_name; });
+
+TEST_P(ResolutionBasedFallbackTest, VerifyForcedEncoderFallback) {
+  const ResolutionBasedFallbackTestParams& params = GetParam();
+  test::ScopedFieldTrials field_trials(params.field_trials);
+  auto primary = new test::FakeEncoder(Clock::GetRealTimeClock());
+  auto fallback = new test::FakeEncoder(Clock::GetRealTimeClock());
+  auto encoder = CreateVideoEncoderSoftwareFallbackWrapper(
+      std::unique_ptr<VideoEncoder>(fallback),
+      std::unique_ptr<VideoEncoder>(primary),
+      /*prefer_temporal_support=*/false);
+  primary->SetImplementationName("primary");
+  fallback->SetImplementationName("fallback");
+  VideoCodec codec;
+  codec.codecType = params.codec_type;
+  codec.width = params.width;
+  codec.height = params.height;
+  encoder->InitEncode(&codec, kSettings);
+  EXPECT_EQ(encoder->GetEncoderInfo().implementation_name,
+            params.expect_implementation_name);
+}
+
 }  // namespace webrtc

api/video_codecs/video_encoder_software_fallback_wrapper.cc

@@ -20,6 +20,7 @@
 #include "absl/strings/match.h"
 #include "absl/types/optional.h"
 #include "api/fec_controller_override.h"
+#include "api/transport/field_trial_based_config.h"
 #include "api/video/i420_buffer.h"
 #include "api/video/video_bitrate_allocation.h"
 #include "api/video/video_frame.h"
@@ -29,6 +30,7 @@
 #include "modules/video_coding/include/video_error_codes.h"
 #include "modules/video_coding/utility/simulcast_utility.h"
 #include "rtc_base/checks.h"
+#include "rtc_base/experiments/field_trial_parser.h"
 #include "rtc_base/logging.h"
 #include "system_wrappers/include/field_trial.h"
 
@@ -48,10 +50,18 @@ namespace {
 struct ForcedFallbackParams {
  public:
   bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const {
-    return enable_resolution_based_switch &&
-           codec.codecType == kVideoCodecVP8 &&
-           codec.numberOfSimulcastStreams <= 1 &&
-           codec.width * codec.height <= max_pixels;
+    if (!enable_resolution_based_switch ||
+        codec.width * codec.height > max_pixels) {
+      return false;
+    }
+
+    if (vp8_specific_resolution_switch &&
+        (codec.codecType != kVideoCodecVP8 ||
+         codec.numberOfSimulcastStreams > 1)) {
+      return false;
+    }
+
+    return true;
   }
 
   bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const {
@@ -61,7 +71,8 @@ struct ForcedFallbackParams {
 
   bool enable_temporal_based_switch = false;
   bool enable_resolution_based_switch = false;
-  int min_pixels = 320 * 180;
+  bool vp8_specific_resolution_switch = false;
+  int min_pixels = kDefaultMinPixelsPerFrame;
   int max_pixels = 320 * 240;
 };
 
@@ -70,6 +81,19 @@ const char kVp8ForceFallbackEncoderFieldTrial[] =
 
 absl::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials(
     const VideoEncoder& main_encoder) {
+  // Ignore WebRTC-VP8-Forced-Fallback-Encoder-v2 if
+  // WebRTC-Video-EncoderFallbackSettings is present.
+  FieldTrialOptional<int> resolution_threshold_px("resolution_threshold_px");
+  ParseFieldTrial(
+      {&resolution_threshold_px},
+      FieldTrialBasedConfig().Lookup("WebRTC-Video-EncoderFallbackSettings"));
+  if (resolution_threshold_px) {
+    ForcedFallbackParams params;
+    params.enable_resolution_based_switch = true;
+    params.max_pixels = resolution_threshold_px.Value();
+    return params;
+  }
+
   const std::string field_trial =
       webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial);
   if (!absl::StartsWith(field_trial, "Enabled")) {
@@ -95,6 +119,7 @@ absl::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials(
     return absl::nullopt;
   }
 
+  params.vp8_specific_resolution_switch = true;
   return params;
 }
 
@@ -107,7 +132,7 @@ absl::optional<ForcedFallbackParams> GetForcedFallbackParams(
     if (!params.has_value()) {
       params.emplace();
     }
-    params->enable_temporal_based_switch = prefer_temporal_support;
+    params->enable_temporal_based_switch = true;
   }
   return params;
 }
@@ -421,18 +446,8 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
       fallback_encoder_info.apply_alignment_to_all_simulcast_layers ||
       default_encoder_info.apply_alignment_to_all_simulcast_layers;
 
-  if (fallback_params_.has_value()) {
-    const auto settings = (encoder_state_ == EncoderState::kForcedFallback)
-                              ? fallback_encoder_info.scaling_settings
-                              : default_encoder_info.scaling_settings;
-    info.scaling_settings =
-        settings.thresholds
-            ? VideoEncoder::ScalingSettings(settings.thresholds->low,
-                                            settings.thresholds->high,
-                                            fallback_params_->min_pixels)
-            : VideoEncoder::ScalingSettings::kOff;
-  } else {
-    info.scaling_settings = default_encoder_info.scaling_settings;
+  if (fallback_params_ && fallback_params_->vp8_specific_resolution_switch) {
+    info.scaling_settings.min_pixels_per_frame = fallback_params_->min_pixels;
   }
 
   return info;

test/BUILD.gn

@@ -1022,7 +1022,10 @@ rtc_library("fake_video_codecs") {
     "../rtc_base/synchronization:mutex",
     "../system_wrappers",
   ]
-  absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+  absl_deps = [
+    "//third_party/abseil-cpp/absl/strings",
+    "//third_party/abseil-cpp/absl/types:optional",
+  ]
 }
 
 rtc_library("null_transport") {

test/fake_encoder.cc

@@ -77,6 +77,11 @@ void FakeEncoder::SetQp(int qp) {
   qp_ = qp;
 }
 
+void FakeEncoder::SetImplementationName(absl::string_view implementation_name) {
+  MutexLock lock(&mutex_);
+  implementation_name_ = std::string(implementation_name);
+}
+
 int32_t FakeEncoder::InitEncode(const VideoCodec* config,
                                 const Settings& settings) {
   MutexLock lock(&mutex_);
@@ -274,9 +279,9 @@ void FakeEncoder::SetRatesLocked(const RateControlParameters& parameters) {
 const char* FakeEncoder::kImplementationName = "fake_encoder";
 VideoEncoder::EncoderInfo FakeEncoder::GetEncoderInfo() const {
   EncoderInfo info;
-  info.implementation_name = kImplementationName;
-  info.is_hardware_accelerated = true;
   MutexLock lock(&mutex_);
+  info.implementation_name = implementation_name_.value_or(kImplementationName);
+  info.is_hardware_accelerated = true;
   for (int sid = 0; sid < config_.numberOfSimulcastStreams; ++sid) {
     int number_of_temporal_layers =
         config_.simulcastStream[sid].numberOfTemporalLayers;

test/fake_encoder.h

@@ -17,6 +17,7 @@
 #include <memory>
 #include <vector>
 
+#include "absl/strings/string_view.h"
 #include "api/fec_controller_override.h"
 #include "api/sequence_checker.h"
 #include "api/task_queue/task_queue_factory.h"
@@ -42,6 +43,9 @@ class FakeEncoder : public VideoEncoder {
   void SetMaxBitrate(int max_kbps) RTC_LOCKS_EXCLUDED(mutex_);
   void SetQp(int qp) RTC_LOCKS_EXCLUDED(mutex_);
 
+  void SetImplementationName(absl::string_view implementation_name)
+      RTC_LOCKS_EXCLUDED(mutex_);
+
   void SetFecControllerOverride(
       FecControllerOverride* fec_controller_override) override;
 
@@ -55,7 +59,7 @@ class FakeEncoder : public VideoEncoder {
   int32_t Release() override;
   void SetRates(const RateControlParameters& parameters)
       RTC_LOCKS_EXCLUDED(mutex_) override;
-  EncoderInfo GetEncoderInfo() const override;
+  EncoderInfo GetEncoderInfo() const RTC_LOCKS_EXCLUDED(mutex_) override;
 
   int GetConfiguredInputFramerate() const RTC_LOCKS_EXCLUDED(mutex_);
   int GetNumInitializations() const RTC_LOCKS_EXCLUDED(mutex_);
@@ -108,6 +112,7 @@ class FakeEncoder : public VideoEncoder {
   mutable Mutex mutex_;
   bool used_layers_[kMaxSimulcastStreams];
   absl::optional<int> qp_ RTC_GUARDED_BY(mutex_);
+  absl::optional<std::string> implementation_name_ RTC_GUARDED_BY(mutex_);
 
   // Current byte debt to be payed over a number of frames.
   // The debt is acquired by keyframes overshooting the bitrate target.