From 261f792f836a26fc62cb5835635883a762134338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Spr=C3=A5ng?= Date: Fri, 31 Jan 2020 13:51:12 +0100 Subject: [PATCH] Allow software fallback on lowest simulcast stream for temporal support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:11324 Change-Id: Ie505be0cda74c0444065d86c3727671c62bd4842 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167527 Reviewed-by: Sebastian Jansson Reviewed-by: Evan Shrubsole Reviewed-by: Åsa Persson Commit-Queue: Erik Språng Cr-Commit-Position: refs/heads/master@{#30437} --- api/test/mock_video_encoder.h | 4 + api/video_codecs/BUILD.gn | 1 + ...oder_software_fallback_wrapper_unittest.cc | 378 +++++++++++++-- ...video_encoder_software_fallback_wrapper.cc | 445 +++++++++++------- .../video_encoder_software_fallback_wrapper.h | 18 +- media/engine/simulcast_encoder_adapter.cc | 8 +- media/engine/simulcast_encoder_adapter.h | 1 + 7 files changed, 632 insertions(+), 223 deletions(-) diff --git a/api/test/mock_video_encoder.h b/api/test/mock_video_encoder.h index 65de14f98b..34c038a2e0 100644 --- a/api/test/mock_video_encoder.h +++ b/api/test/mock_video_encoder.h @@ -51,6 +51,10 @@ class MockVideoEncoder : public VideoEncoder { MOCK_METHOD0(Release, int32_t()); MOCK_METHOD0(Reset, int32_t()); MOCK_METHOD1(SetRates, void(const RateControlParameters& parameters)); + MOCK_METHOD1(OnPacketLossRateUpdate, void(float packet_loss_rate)); + MOCK_METHOD1(OnRttUpdate, void(int64_t rtt_ms)); + MOCK_METHOD1(OnLossNotification, + void(const LossNotification& loss_notification)); MOCK_CONST_METHOD0(GetEncoderInfo, EncoderInfo(void)); }; diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index 5a16e6bc13..a3072eb31b 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -137,6 +137,7 @@ rtc_library("rtc_software_fallback_wrappers") { "../../media:rtc_h264_profile_id", "../../media:rtc_media_base", "../../modules/video_coding:video_codec_interface", + "../../modules/video_coding:video_coding_utility", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", "../../rtc_base/system:rtc_export", diff --git a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc index 574bc6fd5a..ba13d92b72 100644 --- a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc +++ b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc @@ -40,6 +40,7 @@ #include "test/gtest.h" namespace webrtc { +using ::testing::_; using ::testing::Return; namespace { @@ -76,6 +77,17 @@ VideoEncoder::EncoderInfo GetEncoderInfoWithInternalSource( info.has_internal_source = internal_source; return info; } + +class FakeEncodedImageCallback : public EncodedImageCallback { + public: + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override { + ++callback_count_; + return Result(Result::OK, callback_count_); + } + int callback_count_ = 0; +}; } // namespace class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { @@ -86,9 +98,11 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { const std::string& field_trials) : override_field_trials_(field_trials), fake_encoder_(new CountingFakeEncoder()), + wrapper_initialized_(false), fallback_wrapper_(CreateVideoEncoderSoftwareFallbackWrapper( std::unique_ptr(VP8Encoder::Create()), - std::unique_ptr(fake_encoder_))) {} + std::unique_ptr(fake_encoder_), + false)) {} class CountingFakeEncoder : public VideoEncoder { public: @@ -125,9 +139,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { return WEBRTC_VIDEO_CODEC_OK; } - void SetRates(const RateControlParameters& parameters) override { - ++set_rates_count_; - } + void SetRates(const RateControlParameters& parameters) override {} EncoderInfo GetEncoderInfo() const override { ++supports_native_handle_count_; @@ -144,23 +156,11 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { int encode_count_ = 0; EncodedImageCallback* encode_complete_callback_ = nullptr; int release_count_ = 0; - int set_rates_count_ = 0; mutable int supports_native_handle_count_ = 0; bool supports_native_handle_ = false; }; - class FakeEncodedImageCallback : public EncodedImageCallback { - public: - Result OnEncodedImage( - const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info, - const RTPFragmentationHeader* fragmentation) override { - ++callback_count_; - return Result(Result::OK, callback_count_); - } - int callback_count_ = 0; - }; - + void InitEncode(); void UtilizeFallbackEncoder(); void FallbackFromEncodeRequest(); void EncodeFrame(); @@ -174,6 +174,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { FakeEncodedImageCallback callback_; // |fake_encoder_| is owned and released by |fallback_wrapper_|. CountingFakeEncoder* fake_encoder_; + bool wrapper_initialized_; std::unique_ptr fallback_wrapper_; VideoCodec codec_ = {}; std::unique_ptr frame_; @@ -199,9 +200,42 @@ void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame(int expected_ret) { EXPECT_EQ(expected_ret, fallback_wrapper_->Encode(*frame_, &types)); } +void VideoEncoderSoftwareFallbackWrapperTest::InitEncode() { + if (!wrapper_initialized_) { + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); + EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); + } + + // Register fake encoder as main. + codec_.codecType = kVideoCodecVP8; + codec_.maxFramerate = kFramerate; + codec_.width = kWidth; + codec_.height = kHeight; + codec_.VP8()->numberOfTemporalLayers = 1; + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + + if (wrapper_initialized_) { + fallback_wrapper_->Release(); + } + + fake_encoder_->init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_->InitEncode(&codec_, kSettings)); + + if (!wrapper_initialized_) { + fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(300000, kFramerate)), + kFramerate)); + } + wrapper_initialized_ = true; +} + void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { - fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); - EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); + if (!wrapper_initialized_) { + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); + EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); + } // Register with failing fake encoder. Should succeed with VP8 fallback. codec_.codecType = kVideoCodecVP8; @@ -211,6 +245,10 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { codec_.VP8()->numberOfTemporalLayers = 1; rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + if (wrapper_initialized_) { + fallback_wrapper_->Release(); + } + fake_encoder_->init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->InitEncode(&codec_, kSettings)); @@ -234,6 +272,9 @@ void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() { codec_.height = kHeight; codec_.VP8()->numberOfTemporalLayers = 1; rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + if (wrapper_initialized_) { + fallback_wrapper_->Release(); + } fallback_wrapper_->InitEncode(&codec_, kSettings); fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters( rate_allocator_->Allocate( @@ -272,11 +313,24 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanUtilizeFallbackEncoder) { TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InternalEncoderReleasedDuringFallback) { + EXPECT_EQ(0, fake_encoder_->init_encode_count_); EXPECT_EQ(0, fake_encoder_->release_count_); + + InitEncode(); + + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EXPECT_EQ(0, fake_encoder_->release_count_); + UtilizeFallbackEncoder(); + + // One successful InitEncode(), one failed. + EXPECT_EQ(2, fake_encoder_->init_encode_count_); EXPECT_EQ(1, fake_encoder_->release_count_); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); + // No extra release when the fallback is released. + EXPECT_EQ(2, fake_encoder_->init_encode_count_); EXPECT_EQ(1, fake_encoder_->release_count_); } @@ -292,29 +346,30 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanRegisterCallbackWhileUsingFallbackEncoder) { + InitEncode(); + EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); + UtilizeFallbackEncoder(); - // Registering an encode-complete callback should still work when fallback - // encoder is being used. + + // Registering an encode-complete callback will now pass to the fallback + // instead of the main encoder. FakeEncodedImageCallback callback2; fallback_wrapper_->RegisterEncodeCompleteCallback(&callback2); - EXPECT_EQ(&callback2, fake_encoder_->encode_complete_callback_); + EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); // Encoding a frame using the fallback should arrive at the new callback. std::vector types(1, VideoFrameType::kVideoFrameKey); frame_->set_timestamp(frame_->timestamp() + 1000); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Encode(*frame_, &types)); + EXPECT_EQ(callback2.callback_count_, 1); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); -} + // Re-initialize to use the main encoder, the new callback should be in use. + InitEncode(); + EXPECT_EQ(&callback2, fake_encoder_->encode_complete_callback_); -TEST_F(VideoEncoderSoftwareFallbackWrapperTest, - SetRatesForwardedDuringFallback) { - UtilizeFallbackEncoder(); - EXPECT_EQ(1, fake_encoder_->set_rates_count_); - fallback_wrapper_->SetRates( - VideoEncoder::RateControlParameters(VideoBitrateAllocation(), 1)); - EXPECT_EQ(2, fake_encoder_->set_rates_count_); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); + frame_->set_timestamp(frame_->timestamp() + 2000); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Encode(*frame_, &types)); + EXPECT_EQ(callback2.callback_count_, 2); } TEST_F(VideoEncoderSoftwareFallbackWrapperTest, @@ -372,11 +427,12 @@ class ForcedFallbackTest : public VideoEncoderSoftwareFallbackWrapperTest { } void TearDown() override { - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); + if (wrapper_initialized_) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); + } } void ConfigureVp8Codec() { - fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); codec_.codecType = kVideoCodecVP8; codec_.maxFramerate = kFramerate; codec_.width = kWidth; @@ -390,8 +446,13 @@ class ForcedFallbackTest : public VideoEncoderSoftwareFallbackWrapperTest { void InitEncode(int width, int height) { codec_.width = width; codec_.height = height; + if (wrapper_initialized_) { + fallback_wrapper_->Release(); + } EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->InitEncode(&codec_, kSettings)); + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); + wrapper_initialized_ = true; SetRateAllocation(kBitrateKbps); } @@ -494,11 +555,11 @@ TEST_F(ForcedFallbackTestEnabled, FallbackIsEndedForNonValidSettings) { EXPECT_EQ(1, fake_encoder_->init_encode_count_); EncodeFrameAndVerifyLastName("fake-encoder"); - // Re-initialize encoder with valid setting but fallback disabled from now on. + // Re-initialize encoder with valid setting. codec_.VP8()->numberOfTemporalLayers = 1; InitEncode(kWidth, kHeight); - EXPECT_EQ(2, fake_encoder_->init_encode_count_); - EncodeFrameAndVerifyLastName("fake-encoder"); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("libvpx"); } TEST_F(ForcedFallbackTestEnabled, MultipleStartEndFallback) { @@ -689,4 +750,247 @@ TEST(SoftwareFallbackEncoderTest, ReportsInternalSource) { EXPECT_FALSE(wrapper->GetEncoderInfo().has_internal_source); } +class PreferTemporalLayersFallbackTest : public ::testing::Test { + public: + PreferTemporalLayersFallbackTest() {} + void SetUp() override { + sw_ = new ::testing::NiceMock(); + sw_info_.implementation_name = "sw"; + EXPECT_CALL(*sw_, GetEncoderInfo).WillRepeatedly([&]() { + return sw_info_; + }); + EXPECT_CALL(*sw_, InitEncode(_, _, _)) + .WillRepeatedly(Return(WEBRTC_VIDEO_CODEC_OK)); + + hw_ = new ::testing::NiceMock(); + hw_info_.implementation_name = "hw"; + EXPECT_CALL(*hw_, GetEncoderInfo()).WillRepeatedly([&]() { + return hw_info_; + }); + EXPECT_CALL(*hw_, InitEncode(_, _, _)) + .WillRepeatedly(Return(WEBRTC_VIDEO_CODEC_OK)); + + wrapper_ = CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr(sw_), std::unique_ptr(hw_), + /*prefer_temporal_support=*/true); + + codec_settings.codecType = kVideoCodecVP8; + codec_settings.maxFramerate = kFramerate; + codec_settings.width = kWidth; + codec_settings.height = kHeight; + codec_settings.numberOfSimulcastStreams = 1; + codec_settings.VP8()->numberOfTemporalLayers = 1; + } + + protected: + void SetSupportsLayers(VideoEncoder::EncoderInfo* info, bool tl_enabled) { + info->fps_allocation[0].clear(); + int num_layers = 1; + if (tl_enabled) { + num_layers = codec_settings.VP8()->numberOfTemporalLayers; + } + for (int i = 0; i < num_layers; ++i) { + info->fps_allocation[0].push_back( + VideoEncoder::EncoderInfo::kMaxFramerateFraction >> + (num_layers - i - 1)); + } + } + + VideoCodec codec_settings; + ::testing::NiceMock* sw_; + ::testing::NiceMock* hw_; + VideoEncoder::EncoderInfo sw_info_; + VideoEncoder::EncoderInfo hw_info_; + std::unique_ptr wrapper_; +}; + +TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenLayersNotUsed) { + codec_settings.VP8()->numberOfTemporalLayers = 1; + SetSupportsLayers(&hw_info_, true); + SetSupportsLayers(&sw_info_, true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenLayersSupported) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + SetSupportsLayers(&hw_info_, true); + SetSupportsLayers(&sw_info_, true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, + UsesFallbackWhenLayersNotSupportedOnMain) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + SetSupportsLayers(&hw_info_, false); + SetSupportsLayers(&sw_info_, true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenNeitherSupportsTemporal) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + SetSupportsLayers(&hw_info_, false); + SetSupportsLayers(&sw_info_, false); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, PrimesEncoderOnSwitch) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + // Both support temporal layers, will use main one. + SetSupportsLayers(&hw_info_, true); + SetSupportsLayers(&sw_info_, true); + + // On first InitEncode most params have no state and will not be + // called to update. + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0); + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0); + + EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0); + EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0); + + EXPECT_CALL(*hw_, SetRates).Times(0); + EXPECT_CALL(*hw_, SetRates).Times(0); + + EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0); + EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0); + + EXPECT_CALL(*hw_, OnRttUpdate).Times(0); + EXPECT_CALL(*sw_, OnRttUpdate).Times(0); + + EXPECT_CALL(*hw_, OnLossNotification).Times(0); + EXPECT_CALL(*sw_, OnLossNotification).Times(0); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); + + FakeEncodedImageCallback callback1; + class DummyFecControllerOverride : public FecControllerOverride { + public: + void SetFecAllowed(bool fec_allowed) override {} + }; + DummyFecControllerOverride fec_controller_override1; + VideoEncoder::RateControlParameters rate_params1; + float packet_loss1 = 0.1; + int64_t rtt1 = 1; + VideoEncoder::LossNotification lntf1; + + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback(&callback1)); + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0); + wrapper_->RegisterEncodeCompleteCallback(&callback1); + + EXPECT_CALL(*hw_, SetFecControllerOverride(&fec_controller_override1)); + EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0); + wrapper_->SetFecControllerOverride(&fec_controller_override1); + + EXPECT_CALL(*hw_, SetRates(rate_params1)); + EXPECT_CALL(*sw_, SetRates).Times(0); + wrapper_->SetRates(rate_params1); + + EXPECT_CALL(*hw_, OnPacketLossRateUpdate(packet_loss1)); + EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0); + wrapper_->OnPacketLossRateUpdate(packet_loss1); + + EXPECT_CALL(*hw_, OnRttUpdate(rtt1)); + EXPECT_CALL(*sw_, OnRttUpdate).Times(0); + wrapper_->OnRttUpdate(rtt1); + + EXPECT_CALL(*hw_, OnLossNotification).Times(1); + EXPECT_CALL(*sw_, OnLossNotification).Times(0); + wrapper_->OnLossNotification(lntf1); + + // Release and re-init, with fallback to software. This should trigger + // the software encoder to be primed with the current state. + wrapper_->Release(); + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback(&callback1)); + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0); + + EXPECT_CALL(*sw_, SetFecControllerOverride(&fec_controller_override1)); + EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0); + + // Rate control parameters are cleared on InitEncode. + EXPECT_CALL(*sw_, SetRates).Times(0); + EXPECT_CALL(*hw_, SetRates).Times(0); + + EXPECT_CALL(*sw_, OnPacketLossRateUpdate(packet_loss1)); + EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0); + + EXPECT_CALL(*sw_, OnRttUpdate(rtt1)); + EXPECT_CALL(*hw_, OnRttUpdate).Times(0); + + EXPECT_CALL(*sw_, OnLossNotification).Times(1); + EXPECT_CALL(*hw_, OnLossNotification).Times(0); + + SetSupportsLayers(&hw_info_, false); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw"); + + // Update with all-new params for the software encoder. + FakeEncodedImageCallback callback2; + DummyFecControllerOverride fec_controller_override2; + VideoEncoder::RateControlParameters rate_params2; + float packet_loss2 = 0.2; + int64_t rtt2 = 2; + VideoEncoder::LossNotification lntf2; + + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback(&callback2)); + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0); + wrapper_->RegisterEncodeCompleteCallback(&callback2); + + EXPECT_CALL(*sw_, SetFecControllerOverride(&fec_controller_override2)); + EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0); + wrapper_->SetFecControllerOverride(&fec_controller_override2); + + EXPECT_CALL(*sw_, SetRates(rate_params2)); + EXPECT_CALL(*hw_, SetRates).Times(0); + wrapper_->SetRates(rate_params2); + + EXPECT_CALL(*sw_, OnPacketLossRateUpdate(packet_loss2)); + EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0); + wrapper_->OnPacketLossRateUpdate(packet_loss2); + + EXPECT_CALL(*sw_, OnRttUpdate(rtt2)); + EXPECT_CALL(*hw_, OnRttUpdate).Times(0); + wrapper_->OnRttUpdate(rtt2); + + EXPECT_CALL(*sw_, OnLossNotification).Times(1); + EXPECT_CALL(*hw_, OnLossNotification).Times(0); + wrapper_->OnLossNotification(lntf2); + + // Release and re-init, back to main encoder. This should trigger + // the main encoder to be primed with the current state. + wrapper_->Release(); + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback(&callback2)); + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0); + + EXPECT_CALL(*hw_, SetFecControllerOverride(&fec_controller_override2)); + EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0); + + // Rate control parameters are cleared on InitEncode. + EXPECT_CALL(*sw_, SetRates).Times(0); + EXPECT_CALL(*hw_, SetRates).Times(0); + + EXPECT_CALL(*hw_, OnPacketLossRateUpdate(packet_loss2)); + EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0); + + EXPECT_CALL(*hw_, OnRttUpdate(rtt2)); + EXPECT_CALL(*sw_, OnRttUpdate).Times(0); + + EXPECT_CALL(*hw_, OnLossNotification).Times(1); + EXPECT_CALL(*sw_, OnLossNotification).Times(0); + + SetSupportsLayers(&hw_info_, true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); +} + } // namespace webrtc diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc index fe3274128e..9edc9b074b 100644 --- a/api/video_codecs/video_encoder_software_fallback_wrapper.cc +++ b/api/video_codecs/video_encoder_software_fallback_wrapper.cc @@ -15,7 +15,6 @@ #include #include #include -#include #include #include "absl/types/optional.h" @@ -25,6 +24,7 @@ #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/simulcast_utility.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "system_wrappers/include/field_trial.h" @@ -33,52 +33,89 @@ namespace webrtc { namespace { +// If forced fallback is allowed, either: +// +// 1) The forced fallback is requested if the resolution is less than or equal +// to |max_pixels_|. The resolution is allowed to be scaled down to +// |min_pixels_|. +// +// 2) The forced fallback is requested if temporal support is preferred and the +// SW fallback supports temporal layers while the HW encoder does not. + +struct ForcedFallbackParams { + public: + bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const { + return enable_resolution_based_switch && + codec.codecType == kVideoCodecVP8 && + codec.numberOfSimulcastStreams <= 1 && + codec.VP8().numberOfTemporalLayers == 1 && + codec.width * codec.height <= max_pixels; + } + + bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const { + return enable_temporal_based_switch && + SimulcastUtility::NumberOfTemporalLayers(codec, 0) > 1; + } + + bool enable_temporal_based_switch = false; + bool enable_resolution_based_switch = false; + int min_pixels = 320 * 180; + int max_pixels = 320 * 240; +}; + const char kVp8ForceFallbackEncoderFieldTrial[] = "WebRTC-VP8-Forced-Fallback-Encoder-v2"; -bool EnableForcedFallback() { - return field_trial::IsEnabled(kVp8ForceFallbackEncoderFieldTrial); -} - -bool IsForcedFallbackPossible(const VideoCodec& codec_settings) { - return codec_settings.codecType == kVideoCodecVP8 && - codec_settings.numberOfSimulcastStreams <= 1 && - codec_settings.VP8().numberOfTemporalLayers == 1; -} - -void GetForcedFallbackParamsFromFieldTrialGroup(int* param_min_pixels, - int* param_max_pixels, - int minimum_max_pixels) { - RTC_DCHECK(param_min_pixels); - RTC_DCHECK(param_max_pixels); - std::string group = +absl::optional ParseFallbackParamsFromFieldTrials( + const VideoEncoder& main_encoder) { + const std::string field_trial = webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial); - if (group.empty()) - return; + if (field_trial.find("Enabled") != 0) { + return absl::nullopt; + } - int min_pixels; - int max_pixels; - int min_bps; - if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &min_pixels, &max_pixels, - &min_bps) != 3) { + int max_pixels_lower_bound = + main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1; + + ForcedFallbackParams params; + params.enable_resolution_based_switch = true; + + int min_bps = 0; + if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", ¶ms.min_pixels, + ¶ms.max_pixels, &min_bps) != 3) { RTC_LOG(LS_WARNING) << "Invalid number of forced fallback parameters provided."; - return; - } - if (min_pixels <= 0 || max_pixels < minimum_max_pixels || - max_pixels < min_pixels || min_bps <= 0) { + return absl::nullopt; + } else if (params.min_pixels <= 0 || + params.max_pixels < max_pixels_lower_bound || + params.max_pixels < params.min_pixels || min_bps <= 0) { RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided."; - return; + return absl::nullopt; } - *param_min_pixels = min_pixels; - *param_max_pixels = max_pixels; + + return params; +} + +absl::optional GetForcedFallbackParams( + bool prefer_temporal_support, + const VideoEncoder& main_encoder) { + absl::optional params = + ParseFallbackParamsFromFieldTrials(main_encoder); + if (prefer_temporal_support) { + if (!params.has_value()) { + params.emplace(); + } + params->enable_temporal_based_switch = prefer_temporal_support; + } + return params; } class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder { public: VideoEncoderSoftwareFallbackWrapper( std::unique_ptr sw_encoder, - std::unique_ptr hw_encoder); + std::unique_ptr hw_encoder, + bool prefer_temporal_support); ~VideoEncoderSoftwareFallbackWrapper() override; void SetFecControllerOverride( @@ -106,28 +143,28 @@ class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder { EncoderInfo GetEncoderInfo() const override; private: - bool InitFallbackEncoder(); - - // If |forced_fallback_possible_| is true: - // The forced fallback is requested if the resolution is less than or equal to - // |max_pixels_|. The resolution is allowed to be scaled down to - // |min_pixels_|. - class ForcedFallbackParams { - public: - bool IsValid(const VideoCodec& codec) const { - return codec.width * codec.height <= max_pixels_; - } - - bool active_ = false; - int min_pixels_ = 320 * 180; - int max_pixels_ = 320 * 240; - }; - + bool InitFallbackEncoder(bool is_forced); bool TryInitForcedFallbackEncoder(); - bool TryReInitForcedFallbackEncoder(); - void ValidateSettingsForForcedFallback(); - bool IsForcedFallbackActive() const; - void MaybeModifyCodecForFallback(); + bool IsFallbackActive() const; + + VideoEncoder* current_encoder() { + switch (encoder_state_) { + case EncoderState::kUninitialized: + RTC_LOG(LS_WARNING) + << "Trying to access encoder in uninitialized fallback wrapper."; + // Return main encoder to preserve previous behavior. + ABSL_FALLTHROUGH_INTENDED; + case EncoderState::kMainEncoderUsed: + return encoder_.get(); + case EncoderState::kFallbackDueToFailure: + case EncoderState::kForcedFallback: + return fallback_encoder_.get(); + } + } + + // Updates encoder with last observed parameters, such as callbacks, rates, + // etc. + void PrimeEncoder(VideoEncoder* encoder) const; // Settings used in the last InitEncode call and used if a dynamic fallback to // software is required. @@ -137,65 +174,95 @@ class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder { // The last rate control settings, if set. absl::optional rate_control_parameters_; - // The last channel parameters set, and a flag for noting they are set. - bool channel_parameters_set_; - uint32_t packet_loss_; - int64_t rtt_; + // The last channel parameters set. + absl::optional packet_loss_; + absl::optional rtt_; + FecControllerOverride* fec_controller_override_; + absl::optional loss_notification_; - bool use_fallback_encoder_; + enum class EncoderState { + kUninitialized, + kMainEncoderUsed, + kFallbackDueToFailure, + kForcedFallback + }; + + EncoderState encoder_state_; const std::unique_ptr encoder_; - const std::unique_ptr fallback_encoder_; + EncodedImageCallback* callback_; - bool forced_fallback_possible_; - ForcedFallbackParams forced_fallback_; + const absl::optional fallback_params_; }; VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( std::unique_ptr sw_encoder, - std::unique_ptr hw_encoder) - : channel_parameters_set_(false), - packet_loss_(0), - rtt_(0), - use_fallback_encoder_(false), + std::unique_ptr hw_encoder, + bool prefer_temporal_support) + : fec_controller_override_(nullptr), + encoder_state_(EncoderState::kUninitialized), encoder_(std::move(hw_encoder)), fallback_encoder_(std::move(sw_encoder)), callback_(nullptr), - forced_fallback_possible_(EnableForcedFallback()) { + fallback_params_( + GetForcedFallbackParams(prefer_temporal_support, *encoder_)) { RTC_DCHECK(fallback_encoder_); - if (forced_fallback_possible_) { - GetForcedFallbackParamsFromFieldTrialGroup( - &forced_fallback_.min_pixels_, &forced_fallback_.max_pixels_, - encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame - - 1); // No HW below. - } } + VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() = default; -bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() { +void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder( + VideoEncoder* encoder) const { + RTC_DCHECK(encoder); + // Replay callback, rates, and channel parameters. + if (callback_) { + encoder->RegisterEncodeCompleteCallback(callback_); + } + if (rate_control_parameters_) { + encoder->SetRates(*rate_control_parameters_); + } + if (rtt_.has_value()) { + encoder->OnRttUpdate(rtt_.value()); + } + if (packet_loss_.has_value()) { + encoder->OnPacketLossRateUpdate(packet_loss_.value()); + } + if (fec_controller_override_) { + encoder->SetFecControllerOverride(fec_controller_override_); + } + if (loss_notification_.has_value()) { + encoder->OnLossNotification(loss_notification_.value()); + } +} + +bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) { RTC_LOG(LS_WARNING) << "Encoder falling back to software encoding."; RTC_DCHECK(encoder_settings_.has_value()); const int ret = fallback_encoder_->InitEncode(&codec_settings_, encoder_settings_.value()); - use_fallback_encoder_ = (ret == WEBRTC_VIDEO_CODEC_OK); - if (!use_fallback_encoder_) { + + if (ret != WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_ERROR) << "Failed to initialize software-encoder fallback."; fallback_encoder_->Release(); return false; } - // Replay callback, rates, and channel parameters. - if (callback_) - fallback_encoder_->RegisterEncodeCompleteCallback(callback_); - if (rate_control_parameters_) - fallback_encoder_->SetRates(*rate_control_parameters_); - // Since we're switching to the fallback encoder, Release the real encoder. It - // may be re-initialized via InitEncode later, and it will continue to get - // Set calls for rates and channel parameters in the meantime. - encoder_->Release(); + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Since we're switching to the fallback encoder, Release the real encoder. + // It may be re-initialized via InitEncode later, and it will continue to + // get Set calls for rates and channel parameters in the meantime. + encoder_->Release(); + } + + if (is_forced) { + encoder_state_ = EncoderState::kForcedFallback; + } else { + encoder_state_ = EncoderState::kFallbackDueToFailure; + } + return true; } @@ -204,8 +271,9 @@ void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride( // It is important that only one of those would ever interact with the // |fec_controller_override| at a given time. This is the responsibility // of |this| to maintain. - encoder_->SetFecControllerOverride(fec_controller_override); - fallback_encoder_->SetFecControllerOverride(fec_controller_override); + + fec_controller_override_ = fec_controller_override; + current_encoder()->SetFecControllerOverride(fec_controller_override); } int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( @@ -217,93 +285,94 @@ int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( encoder_settings_ = settings; // Clear stored rate/channel parameters. rate_control_parameters_ = absl::nullopt; - ValidateSettingsForForcedFallback(); - // Try to reinit forced software codec if it is in use. - if (TryReInitForcedFallbackEncoder()) { - return WEBRTC_VIDEO_CODEC_OK; - } + RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized) + << "InitEncode() should never be called on an active instance!"; + // Try to init forced software codec if it should be used. if (TryInitForcedFallbackEncoder()) { + PrimeEncoder(current_encoder()); return WEBRTC_VIDEO_CODEC_OK; } - forced_fallback_.active_ = false; int32_t ret = encoder_->InitEncode(codec_settings, settings); if (ret == WEBRTC_VIDEO_CODEC_OK) { - if (use_fallback_encoder_) { - RTC_LOG(LS_WARNING) - << "InitEncode OK, no longer using the software fallback encoder."; - fallback_encoder_->Release(); - use_fallback_encoder_ = false; - } - if (callback_) - encoder_->RegisterEncodeCompleteCallback(callback_); + encoder_state_ = EncoderState::kMainEncoderUsed; + PrimeEncoder(current_encoder()); return ret; } + // Try to instantiate software codec. - if (InitFallbackEncoder()) { + if (InitFallbackEncoder(/*is_forced=*/false)) { + PrimeEncoder(current_encoder()); return WEBRTC_VIDEO_CODEC_OK; } - // Software encoder failed, use original return code. + + // Software encoder failed too, use original return code. + encoder_state_ = EncoderState::kUninitialized; return ret; } int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback( EncodedImageCallback* callback) { callback_ = callback; - int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback); - if (use_fallback_encoder_) - return fallback_encoder_->RegisterEncodeCompleteCallback(callback); - return ret; + return current_encoder()->RegisterEncodeCompleteCallback(callback); } int32_t VideoEncoderSoftwareFallbackWrapper::Release() { - return use_fallback_encoder_ ? fallback_encoder_->Release() - : encoder_->Release(); + if (encoder_state_ == EncoderState::kUninitialized) { + return WEBRTC_VIDEO_CODEC_OK; + } + int32_t ret = current_encoder()->Release(); + encoder_state_ = EncoderState::kUninitialized; + return ret; } int32_t VideoEncoderSoftwareFallbackWrapper::Encode( const VideoFrame& frame, const std::vector* frame_types) { - if (use_fallback_encoder_) - return fallback_encoder_->Encode(frame, frame_types); - int32_t ret = encoder_->Encode(frame, frame_types); - // If requested, try a software fallback. - bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); - if (fallback_requested && InitFallbackEncoder()) { - // Start using the fallback with this frame. - return fallback_encoder_->Encode(frame, frame_types); + switch (encoder_state_) { + case EncoderState::kUninitialized: + return WEBRTC_VIDEO_CODEC_ERROR; + case EncoderState::kMainEncoderUsed: { + int32_t ret = encoder_->Encode(frame, frame_types); + // If requested, try a software fallback. + bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) { + // Start using the fallback with this frame. + PrimeEncoder(current_encoder()); + return fallback_encoder_->Encode(frame, frame_types); + } + // Fallback encoder failed too, return original error code. + return ret; + } + case EncoderState::kFallbackDueToFailure: + case EncoderState::kForcedFallback: + return fallback_encoder_->Encode(frame, frame_types); } - return ret; } void VideoEncoderSoftwareFallbackWrapper::SetRates( const RateControlParameters& parameters) { rate_control_parameters_ = parameters; - encoder_->SetRates(parameters); - if (use_fallback_encoder_) - fallback_encoder_->SetRates(parameters); + return current_encoder()->SetRates(parameters); } void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate( float packet_loss_rate) { - VideoEncoder* encoder = - use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get(); - encoder->OnPacketLossRateUpdate(packet_loss_rate); + packet_loss_ = packet_loss_rate; + current_encoder()->OnPacketLossRateUpdate(packet_loss_rate); } void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) { - VideoEncoder* encoder = - use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get(); - encoder->OnRttUpdate(rtt_ms); + rtt_ = rtt_ms; + current_encoder()->OnRttUpdate(rtt_ms); } void VideoEncoderSoftwareFallbackWrapper::OnLossNotification( const LossNotification& loss_notification) { - VideoEncoder* encoder = - use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get(); - encoder->OnLossNotification(loss_notification); + loss_notification_ = loss_notification; + current_encoder()->OnLossNotification(loss_notification); } VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() @@ -312,17 +381,17 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() EncoderInfo default_encoder_info = encoder_->GetEncoderInfo(); EncoderInfo info = - use_fallback_encoder_ ? fallback_encoder_info : default_encoder_info; + IsFallbackActive() ? fallback_encoder_info : default_encoder_info; - if (forced_fallback_possible_) { - const auto settings = forced_fallback_.active_ + if (fallback_params_.has_value()) { + const auto settings = (encoder_state_ == EncoderState::kForcedFallback) ? fallback_encoder_info.scaling_settings : default_encoder_info.scaling_settings; info.scaling_settings = settings.thresholds ? VideoEncoder::ScalingSettings(settings.thresholds->low, settings.thresholds->high, - forced_fallback_.min_pixels_) + fallback_params_->min_pixels) : VideoEncoder::ScalingSettings::kOff; } else { info.scaling_settings = default_encoder_info.scaling_settings; @@ -331,72 +400,82 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() return info; } -bool VideoEncoderSoftwareFallbackWrapper::IsForcedFallbackActive() const { - return (forced_fallback_possible_ && use_fallback_encoder_ && - forced_fallback_.active_); +bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const { + return encoder_state_ == EncoderState::kForcedFallback || + encoder_state_ == EncoderState::kFallbackDueToFailure; } bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() { - if (!forced_fallback_possible_ || use_fallback_encoder_) { - return false; - } - // Fallback not active. - if (!forced_fallback_.IsValid(codec_settings_)) { - return false; - } - // Settings valid, try to instantiate software codec. - RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: " - << codec_settings_.width << "x" << codec_settings_.height; - if (!InitFallbackEncoder()) { - return false; - } - forced_fallback_.active_ = true; - return true; -} - -bool VideoEncoderSoftwareFallbackWrapper::TryReInitForcedFallbackEncoder() { - if (!IsForcedFallbackActive()) { + if (!fallback_params_) { return false; } - // Forced fallback active. - if (!forced_fallback_.IsValid(codec_settings_)) { - RTC_LOG(LS_INFO) << "Stop forced SW encoder fallback, max pixels exceeded."; - return false; + RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized); + + if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) { + // Settings valid, try to instantiate software codec. + RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: " + << codec_settings_.width << "x" << codec_settings_.height; + return InitFallbackEncoder(/*is_forced=*/true); } - // Settings valid, reinitialize the forced fallback encoder. - RTC_DCHECK(encoder_settings_.has_value()); - if (fallback_encoder_->InitEncode(&codec_settings_, - encoder_settings_.value()) != - WEBRTC_VIDEO_CODEC_OK) { - RTC_LOG(LS_ERROR) << "Failed to init forced SW encoder fallback."; - return false; - } - return true; -} - -void VideoEncoderSoftwareFallbackWrapper::ValidateSettingsForForcedFallback() { - if (!forced_fallback_possible_) - return; - - if (!IsForcedFallbackPossible(codec_settings_)) { - if (IsForcedFallbackActive()) { - fallback_encoder_->Release(); - use_fallback_encoder_ = false; + if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) { + // First init main encoder to see if that supports temporal layers. + if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) == + WEBRTC_VIDEO_CODEC_OK) { + encoder_state_ = EncoderState::kMainEncoderUsed; + } + + if (encoder_state_ == EncoderState::kMainEncoderUsed && + encoder_->GetEncoderInfo().fps_allocation[0].size() > 1) { + // Primary encoder already supports temporal layers, use that instead. + return true; + } + + // Try to initialize fallback and check if it supports temporal layers. + if (fallback_encoder_->InitEncode(&codec_settings_, + encoder_settings_.value()) == + WEBRTC_VIDEO_CODEC_OK) { + if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() > 1) { + // Fallback encoder available and supports temporal layers, use it! + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Main encoder initialized but does not support temporal layers, + // release it again. + encoder_->Release(); + } + encoder_state_ = EncoderState::kForcedFallback; + RTC_LOG(LS_INFO) + << "Forced switch to SW encoder due to temporal support."; + return true; + } else { + // Fallback encoder intialization succeeded, but it does not support + // temporal layers either - release it. + fallback_encoder_->Release(); + } + } + + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Main encoder already initialized - make use of it. + RTC_LOG(LS_INFO) + << "Cannot fall back for temporal support since fallback that " + "supports is not available. Using main encoder instead."; + return true; } - RTC_LOG(LS_INFO) << "Disable forced_fallback_possible_ due to settings."; - forced_fallback_possible_ = false; } + + // Neither forced fallback mode supported. + return false; } } // namespace std::unique_ptr CreateVideoEncoderSoftwareFallbackWrapper( std::unique_ptr sw_fallback_encoder, - std::unique_ptr hw_encoder) { + std::unique_ptr hw_encoder, + bool prefer_temporal_support) { return std::make_unique( - std::move(sw_fallback_encoder), std::move(hw_encoder)); + std::move(sw_fallback_encoder), std::move(hw_encoder), + prefer_temporal_support); } } // namespace webrtc diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.h b/api/video_codecs/video_encoder_software_fallback_wrapper.h index fa93ab82e9..5282dcb0c0 100644 --- a/api/video_codecs/video_encoder_software_fallback_wrapper.h +++ b/api/video_codecs/video_encoder_software_fallback_wrapper.h @@ -12,6 +12,7 @@ #define API_VIDEO_CODECS_VIDEO_ENCODER_SOFTWARE_FALLBACK_WRAPPER_H_ #include +#include #include "api/video_codecs/video_encoder.h" #include "rtc_base/system/rtc_export.h" @@ -21,10 +22,25 @@ namespace webrtc { // Used to wrap external VideoEncoders to provide a fallback option on // software encoding when a hardware encoder fails to encode a stream due to // hardware restrictions, such as max resolution. +// |bool prefer_temporal_support| indicates that if the software fallback +// encoder supports temporal layers but the hardware encoder does not, a +// fallback should be forced even if the encoder otherwise works. RTC_EXPORT std::unique_ptr CreateVideoEncoderSoftwareFallbackWrapper( std::unique_ptr sw_fallback_encoder, - std::unique_ptr hw_encoder); + std::unique_ptr hw_encoder, + bool prefer_temporal_support); + +// Default fallback for call-sites not yet updated with +// |prefer_temporal_support|. +// TODO(sprang): Remove when usage is gone. +RTC_EXPORT inline std::unique_ptr +CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr sw_fallback_encoder, + std::unique_ptr hw_encoder) { + return CreateVideoEncoderSoftwareFallbackWrapper( + std::move(sw_fallback_encoder), std::move(hw_encoder), false); +} } // namespace webrtc diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 300439a1d7..af0d66e5c1 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -150,7 +150,9 @@ SimulcastEncoderAdapter::SimulcastEncoderAdapter( encoded_complete_callback_(nullptr), experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()), boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials() - .Vp8BoostBaseLayerQuality()) { + .Vp8BoostBaseLayerQuality()), + prefer_temporal_support_on_base_layer_(field_trial::IsEnabled( + "WebRTC-Video-PreferTemporalSupportOnBaseLayer")) { RTC_DCHECK(primary_factory); // The adapter is typically created on the worker thread, but operated on @@ -259,7 +261,9 @@ int SimulcastEncoderAdapter::InitEncode( if (fallback_encoder_factory_ != nullptr) { encoder = CreateVideoEncoderSoftwareFallbackWrapper( fallback_encoder_factory_->CreateVideoEncoder(format), - std::move(encoder)); + std::move(encoder), + i == lowest_resolution_stream_index && + prefer_temporal_support_on_base_layer_); } } diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h index 32e798eb5b..a4cf863151 100644 --- a/media/engine/simulcast_encoder_adapter.h +++ b/media/engine/simulcast_encoder_adapter.h @@ -133,6 +133,7 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { const absl::optional experimental_boosted_screenshare_qp_; const bool boost_base_layer_quality_; + const bool prefer_temporal_support_on_base_layer_; }; } // namespace webrtc