diff --git a/api/video_codecs/video_encoder_factory.h b/api/video_codecs/video_encoder_factory.h index d7cea47909..2914a41518 100644 --- a/api/video_codecs/video_encoder_factory.h +++ b/api/video_codecs/video_encoder_factory.h @@ -17,6 +17,7 @@ #include "absl/types/optional.h" #include "api/units/data_rate.h" +#include "api/video/render_resolution.h" #include "api/video_codecs/sdp_video_format.h" namespace webrtc { @@ -47,6 +48,13 @@ class VideoEncoderFactory { virtual absl::optional OnAvailableBitrate( const DataRate& rate) = 0; + // Called every time the encoder input resolution change. Should return a + // non-empty if an encoder switch should be performed. + virtual absl::optional OnResolutionChange( + const RenderResolution& resolution) { + return absl::nullopt; + } + // Called if the currently used encoder reports itself as broken. Should // return a non-empty if an encoder switch should be performed. virtual absl::optional OnEncoderBroken() = 0; diff --git a/sdk/android/api/org/webrtc/VideoEncoderFactory.java b/sdk/android/api/org/webrtc/VideoEncoderFactory.java index e69e967f24..2a46662d14 100644 --- a/sdk/android/api/org/webrtc/VideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/VideoEncoderFactory.java @@ -24,6 +24,16 @@ public interface VideoEncoderFactory { */ @Nullable @CalledByNative("VideoEncoderSelector") VideoCodecInfo onAvailableBitrate(int kbps); + /** + * Called every time the encoder input resolution change. Returns null if the encoder selector + * prefers to keep the current encoder or a VideoCodecInfo if a new encoder is preferred. + */ + @Nullable + @CalledByNative("VideoEncoderSelector") + default VideoCodecInfo onResolutionChange(int widht, int height) { + return null; + } + /** * Called when the currently used encoder signal itself as broken. Returns null if the encoder * selector prefers to keep the current encoder or a VideoCodecInfo if a new encoder is diff --git a/sdk/android/src/jni/video_encoder_factory_wrapper.cc b/sdk/android/src/jni/video_encoder_factory_wrapper.cc index 8ab4191db2..7df129b360 100644 --- a/sdk/android/src/jni/video_encoder_factory_wrapper.cc +++ b/sdk/android/src/jni/video_encoder_factory_wrapper.cc @@ -10,6 +10,7 @@ #include "sdk/android/src/jni/video_encoder_factory_wrapper.h" +#include "api/video/render_resolution.h" #include "api/video_codecs/video_encoder.h" #include "rtc_base/logging.h" #include "sdk/android/generated_video_jni/VideoEncoderFactory_jni.h" @@ -48,6 +49,18 @@ class VideoEncoderSelectorWrapper return VideoCodecInfoToSdpVideoFormat(jni, codec_info); } + absl::optional OnResolutionChange( + const RenderResolution& resolution) override { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + ScopedJavaLocalRef codec_info = + Java_VideoEncoderSelector_onResolutionChange( + jni, encoder_selector_, resolution.Width(), resolution.Height()); + if (codec_info.is_null()) { + return absl::nullopt; + } + return VideoCodecInfoToSdpVideoFormat(jni, codec_info); + } + absl::optional OnEncoderBroken() override { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedJavaLocalRef codec_info = diff --git a/test/video_encoder_proxy_factory.h b/test/video_encoder_proxy_factory.h index ba7d1e3fde..cc485e993a 100644 --- a/test/video_encoder_proxy_factory.h +++ b/test/video_encoder_proxy_factory.h @@ -132,6 +132,11 @@ class VideoEncoderProxyFactory : public VideoEncoderFactory { return encoder_selector_->OnAvailableBitrate(rate); } + absl::optional OnResolutionChange( + const RenderResolution& resolution) override { + return encoder_selector_->OnResolutionChange(resolution); + } + absl::optional OnEncoderBroken() override { return encoder_selector_->OnEncoderBroken(); } diff --git a/video/BUILD.gn b/video/BUILD.gn index b770963a8b..6a86553a07 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -453,6 +453,7 @@ rtc_library("video_stream_encoder_impl") { "../api/task_queue:task_queue", "../api/units:data_rate", "../api/video:encoded_image", + "../api/video:render_resolution", "../api/video:video_adaptation", "../api/video:video_bitrate_allocation", "../api/video:video_bitrate_allocator", diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 916819ed33..a77600fa38 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -25,6 +25,7 @@ #include "api/task_queue/task_queue_base.h" #include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" +#include "api/video/render_resolution.h" #include "api/video/video_adaptation_reason.h" #include "api/video/video_bitrate_allocator_factory.h" #include "api/video/video_codec_constants.h" @@ -1609,6 +1610,16 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || video_frame.height() != last_frame_info_->height || video_frame.is_texture() != last_frame_info_->is_texture) { + if ((!last_frame_info_ || video_frame.width() != last_frame_info_->width || + video_frame.height() != last_frame_info_->height) && + settings_.encoder_switch_request_callback && encoder_selector_) { + if (auto encoder = encoder_selector_->OnResolutionChange( + {video_frame.width(), video_frame.height()})) { + settings_.encoder_switch_request_callback->RequestEncoderSwitch( + *encoder, /*allow_default_fallback=*/false); + } + } + pending_encoder_reconfiguration_ = true; last_frame_info_ = VideoFrameInfo(video_frame.width(), video_frame.height(), video_frame.is_texture()); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index dc164e7b61..c0fa26cd06 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -788,6 +788,10 @@ class MockEncoderSelector OnAvailableBitrate, (const DataRate& rate), (override)); + MOCK_METHOD(absl::optional, + OnResolutionChange, + (const RenderResolution& resolution), + (override)); MOCK_METHOD(absl::optional, OnEncoderBroken, (), (override)); }; @@ -7563,6 +7567,43 @@ TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) { + NiceMock encoder_selector; + StrictMock switch_callback; + video_send_config_.encoder_settings.encoder_switch_request_callback = + &switch_callback; + auto encoder_factory = std::make_unique( + &fake_encoder_, &encoder_selector); + video_send_config_.encoder_settings.encoder_factory = encoder_factory.get(); + + // Reset encoder for new configuration to take effect. + ConfigureEncoder(video_encoder_config_.Copy()); + + EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480))) + .WillOnce(Return(absl::nullopt)); + EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240))) + .WillOnce(Return(SdpVideoFormat("AV1"))); + EXPECT_CALL(switch_callback, + RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"), + /*allow_default_fallback=*/false)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + /*target_bitrate=*/DataRate::KilobitsPerSec(800), + /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000), + /*link_allocation=*/DataRate::KilobitsPerSec(1000), + /*fraction_lost=*/0, + /*round_trip_time_ms=*/0, + /*cwnd_reduce_ratio=*/0); + + video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480)); + video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480)); + video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240)); + + AdvanceTime(TimeDelta::Zero()); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) { constexpr int kSufficientBitrateToNotDrop = 1000; constexpr int kDontCare = 100;