Added OnResolutionChange to EncoderSelectorInterface.

Bug: webrtc:12406
Change-Id: I0160636d93ad0a33caf7ae7443cefe321a191406
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/258442
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#36517}

api/video_codecs/video_encoder_factory.h

@@ -17,6 +17,7 @@
 
 #include "absl/types/optional.h"
 #include "api/units/data_rate.h"
+#include "api/video/render_resolution.h"
 #include "api/video_codecs/sdp_video_format.h"
 
 namespace webrtc {
@@ -47,6 +48,13 @@ class VideoEncoderFactory {
     virtual absl::optional<SdpVideoFormat> OnAvailableBitrate(
         const DataRate& rate) = 0;
 
+    // Called every time the encoder input resolution change. Should return a
+    // non-empty if an encoder switch should be performed.
+    virtual absl::optional<SdpVideoFormat> OnResolutionChange(
+        const RenderResolution& resolution) {
+      return absl::nullopt;
+    }
+
     // Called if the currently used encoder reports itself as broken. Should
     // return a non-empty if an encoder switch should be performed.
     virtual absl::optional<SdpVideoFormat> OnEncoderBroken() = 0;

sdk/android/api/org/webrtc/VideoEncoderFactory.java

@@ -24,6 +24,16 @@ public interface VideoEncoderFactory {
      */
     @Nullable @CalledByNative("VideoEncoderSelector") VideoCodecInfo onAvailableBitrate(int kbps);
 
+    /**
+     * Called every time the encoder input resolution change. Returns null if the encoder selector
+     * prefers to keep the current encoder or a VideoCodecInfo if a new encoder is preferred.
+     */
+    @Nullable
+    @CalledByNative("VideoEncoderSelector")
+    default VideoCodecInfo onResolutionChange(int widht, int height) {
+      return null;
+    }
+
     /**
      * Called when the currently used encoder signal itself as broken. Returns null if the encoder
      * selector prefers to keep the current encoder or a VideoCodecInfo if a new encoder is

sdk/android/src/jni/video_encoder_factory_wrapper.cc

@@ -10,6 +10,7 @@
 
 #include "sdk/android/src/jni/video_encoder_factory_wrapper.h"
 
+#include "api/video/render_resolution.h"
 #include "api/video_codecs/video_encoder.h"
 #include "rtc_base/logging.h"
 #include "sdk/android/generated_video_jni/VideoEncoderFactory_jni.h"
@@ -48,6 +49,18 @@ class VideoEncoderSelectorWrapper
     return VideoCodecInfoToSdpVideoFormat(jni, codec_info);
   }
 
+  absl::optional<SdpVideoFormat> OnResolutionChange(
+      const RenderResolution& resolution) override {
+    JNIEnv* jni = AttachCurrentThreadIfNeeded();
+    ScopedJavaLocalRef<jobject> codec_info =
+        Java_VideoEncoderSelector_onResolutionChange(
+            jni, encoder_selector_, resolution.Width(), resolution.Height());
+    if (codec_info.is_null()) {
+      return absl::nullopt;
+    }
+    return VideoCodecInfoToSdpVideoFormat(jni, codec_info);
+  }
+
   absl::optional<SdpVideoFormat> OnEncoderBroken() override {
     JNIEnv* jni = AttachCurrentThreadIfNeeded();
     ScopedJavaLocalRef<jobject> codec_info =

test/video_encoder_proxy_factory.h

@@ -132,6 +132,11 @@ class VideoEncoderProxyFactory : public VideoEncoderFactory {
       return encoder_selector_->OnAvailableBitrate(rate);
     }
 
+    absl::optional<SdpVideoFormat> OnResolutionChange(
+        const RenderResolution& resolution) override {
+      return encoder_selector_->OnResolutionChange(resolution);
+    }
+
     absl::optional<SdpVideoFormat> OnEncoderBroken() override {
       return encoder_selector_->OnEncoderBroken();
     }

video/BUILD.gn

@@ -453,6 +453,7 @@ rtc_library("video_stream_encoder_impl") {
     "../api/task_queue:task_queue",
     "../api/units:data_rate",
     "../api/video:encoded_image",
+    "../api/video:render_resolution",
     "../api/video:video_adaptation",
     "../api/video:video_bitrate_allocation",
     "../api/video:video_bitrate_allocator",

video/video_stream_encoder.cc

@@ -25,6 +25,7 @@
 #include "api/task_queue/task_queue_base.h"
 #include "api/video/encoded_image.h"
 #include "api/video/i420_buffer.h"
+#include "api/video/render_resolution.h"
 #include "api/video/video_adaptation_reason.h"
 #include "api/video/video_bitrate_allocator_factory.h"
 #include "api/video/video_codec_constants.h"
@@ -1609,6 +1610,16 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
   if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
       video_frame.height() != last_frame_info_->height ||
       video_frame.is_texture() != last_frame_info_->is_texture) {
+    if ((!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
+         video_frame.height() != last_frame_info_->height) &&
+        settings_.encoder_switch_request_callback && encoder_selector_) {
+      if (auto encoder = encoder_selector_->OnResolutionChange(
+              {video_frame.width(), video_frame.height()})) {
+        settings_.encoder_switch_request_callback->RequestEncoderSwitch(
+            *encoder, /*allow_default_fallback=*/false);
+      }
+    }
+
     pending_encoder_reconfiguration_ = true;
     last_frame_info_ = VideoFrameInfo(video_frame.width(), video_frame.height(),
                                       video_frame.is_texture());

video/video_stream_encoder_unittest.cc

@@ -788,6 +788,10 @@ class MockEncoderSelector
               OnAvailableBitrate,
               (const DataRate& rate),
               (override));
+  MOCK_METHOD(absl::optional<SdpVideoFormat>,
+              OnResolutionChange,
+              (const RenderResolution& resolution),
+              (override));
   MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
 };
 
@@ -7563,6 +7567,43 @@ TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
   video_stream_encoder_->Stop();
 }
 
+TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
+  NiceMock<MockEncoderSelector> encoder_selector;
+  StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
+  video_send_config_.encoder_settings.encoder_switch_request_callback =
+      &switch_callback;
+  auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
+      &fake_encoder_, &encoder_selector);
+  video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
+
+  // Reset encoder for new configuration to take effect.
+  ConfigureEncoder(video_encoder_config_.Copy());
+
+  EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
+      .WillOnce(Return(absl::nullopt));
+  EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
+      .WillOnce(Return(SdpVideoFormat("AV1")));
+  EXPECT_CALL(switch_callback,
+              RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
+                                   /*allow_default_fallback=*/false));
+
+  video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+      /*target_bitrate=*/DataRate::KilobitsPerSec(800),
+      /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
+      /*link_allocation=*/DataRate::KilobitsPerSec(1000),
+      /*fraction_lost=*/0,
+      /*round_trip_time_ms=*/0,
+      /*cwnd_reduce_ratio=*/0);
+
+  video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
+  video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
+  video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
+
+  AdvanceTime(TimeDelta::Zero());
+
+  video_stream_encoder_->Stop();
+}
+
 TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
   constexpr int kSufficientBitrateToNotDrop = 1000;
   constexpr int kDontCare = 100;