diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc index be79c42464..bcce9dcd93 100644 --- a/api/video_codecs/video_encoder_software_fallback_wrapper.cc +++ b/api/video_codecs/video_encoder_software_fallback_wrapper.cc @@ -25,6 +25,7 @@ #include "api/video/video_frame.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "media/base/video_common.h" #include "modules/video_coding/include/video_error_codes.h" #include "modules/video_coding/utility/simulcast_utility.h" #include "rtc_base/checks.h" @@ -417,6 +418,13 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() EncoderInfo info = IsFallbackActive() ? fallback_encoder_info : default_encoder_info; + info.requested_resolution_alignment = cricket::LeastCommonMultiple( + fallback_encoder_info.requested_resolution_alignment, + default_encoder_info.requested_resolution_alignment); + info.apply_alignment_to_all_simulcast_layers = + fallback_encoder_info.apply_alignment_to_all_simulcast_layers || + default_encoder_info.apply_alignment_to_all_simulcast_layers; + if (fallback_params_.has_value()) { const auto settings = (encoder_state_ == EncoderState::kForcedFallback) ? fallback_encoder_info.scaling_settings diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 3af022ab17..116f987aa4 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -149,9 +149,13 @@ namespace webrtc { SimulcastEncoderAdapter::EncoderContext::EncoderContext( std::unique_ptr encoder, - bool prefer_temporal_support) + bool prefer_temporal_support, + VideoEncoder::EncoderInfo primary_info, + VideoEncoder::EncoderInfo fallback_info) : encoder_(std::move(encoder)), - prefer_temporal_support_(prefer_temporal_support) {} + prefer_temporal_support_(prefer_temporal_support), + primary_info_(std::move(primary_info)), + fallback_info_(std::move(fallback_info)) {} void SimulcastEncoderAdapter::EncoderContext::Release() { if (encoder_) { @@ -690,7 +694,7 @@ void SimulcastEncoderAdapter::DestroyStoredEncoders() { std::unique_ptr SimulcastEncoderAdapter::FetchOrCreateEncoderContext( - bool is_lowest_quality_stream) { + bool is_lowest_quality_stream) const { bool prefer_temporal_support = fallback_encoder_factory_ != nullptr && is_lowest_quality_stream && prefer_temporal_support_on_base_layer_; @@ -712,14 +716,20 @@ SimulcastEncoderAdapter::FetchOrCreateEncoderContext( } else { std::unique_ptr encoder = primary_encoder_factory_->CreateVideoEncoder(video_format_); + VideoEncoder::EncoderInfo primary_info = encoder->GetEncoderInfo(); + VideoEncoder::EncoderInfo fallback_info = primary_info; if (fallback_encoder_factory_ != nullptr) { + std::unique_ptr fallback_encoder = + fallback_encoder_factory_->CreateVideoEncoder(video_format_); + fallback_info = fallback_encoder->GetEncoderInfo(); encoder = CreateVideoEncoderSoftwareFallbackWrapper( - fallback_encoder_factory_->CreateVideoEncoder(video_format_), - std::move(encoder), prefer_temporal_support); + std::move(fallback_encoder), std::move(encoder), + prefer_temporal_support); } encoder_context = std::make_unique( - std::move(encoder), prefer_temporal_support); + std::move(encoder), prefer_temporal_support, primary_info, + fallback_info); } encoder_context->encoder().RegisterEncodeCompleteCallback( @@ -789,9 +799,11 @@ webrtc::VideoCodec SimulcastEncoderAdapter::MakeStreamCodec( void SimulcastEncoderAdapter::OverrideFromFieldTrial( VideoEncoder::EncoderInfo* info) const { if (encoder_info_override_.requested_resolution_alignment()) { - info->requested_resolution_alignment = - *encoder_info_override_.requested_resolution_alignment(); + info->requested_resolution_alignment = cricket::LeastCommonMultiple( + info->requested_resolution_alignment, + *encoder_info_override_.requested_resolution_alignment()); info->apply_alignment_to_all_simulcast_layers = + info->apply_alignment_to_all_simulcast_layers || encoder_info_override_.apply_alignment_to_all_simulcast_layers(); } if (!encoder_info_override_.resolution_bitrate_limits().empty()) { @@ -815,7 +827,34 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.apply_alignment_to_all_simulcast_layers = false; encoder_info.supports_native_handle = true; encoder_info.scaling_settings.thresholds = absl::nullopt; + if (stream_contexts_.empty()) { + // GetEncoderInfo queried before InitEncode. Only alignment info is needed + // to be filled. + // Create one encoder and query it. + + std::unique_ptr encoder_context = + FetchOrCreateEncoderContext(true); + + const VideoEncoder::EncoderInfo& primary_info = + encoder_context->PrimaryInfo(); + const VideoEncoder::EncoderInfo& fallback_info = + encoder_context->FallbackInfo(); + + encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple( + primary_info.requested_resolution_alignment, + fallback_info.requested_resolution_alignment); + + encoder_info.apply_alignment_to_all_simulcast_layers = + primary_info.apply_alignment_to_all_simulcast_layers || + fallback_info.apply_alignment_to_all_simulcast_layers; + + if (!primary_info.supports_simulcast || !fallback_info.supports_simulcast) { + encoder_info.apply_alignment_to_all_simulcast_layers = true; + } + + cached_encoder_contexts_.emplace_back(std::move(encoder_context)); + OverrideFromFieldTrial(&encoder_info); return encoder_info; } @@ -825,7 +864,6 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { for (size_t i = 0; i < stream_contexts_.size(); ++i) { VideoEncoder::EncoderInfo encoder_impl_info = stream_contexts_[i].encoder().GetEncoderInfo(); - if (i == 0) { // Encoder name indicates names of all sub-encoders. encoder_info.implementation_name += " ("; @@ -864,7 +902,12 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple( encoder_info.requested_resolution_alignment, encoder_impl_info.requested_resolution_alignment); - if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) { + // request alignment on all layers if any of the encoders may need it, or + // if any non-top layer encoder requests a non-trivial alignment. + if (encoder_impl_info.apply_alignment_to_all_simulcast_layers || + (encoder_impl_info.requested_resolution_alignment > 1 && + (codec_.simulcastStream[i].height < codec_.height || + codec_.simulcastStream[i].width < codec_.width))) { encoder_info.apply_alignment_to_all_simulcast_layers = true; } } diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h index 2cb29edfd6..07e3ccd024 100644 --- a/media/engine/simulcast_encoder_adapter.h +++ b/media/engine/simulcast_encoder_adapter.h @@ -71,16 +71,24 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { class EncoderContext { public: EncoderContext(std::unique_ptr encoder, - bool prefer_temporal_support); + bool prefer_temporal_support, + VideoEncoder::EncoderInfo primary_info, + VideoEncoder::EncoderInfo fallback_info); EncoderContext& operator=(EncoderContext&&) = delete; VideoEncoder& encoder() { return *encoder_; } bool prefer_temporal_support() { return prefer_temporal_support_; } void Release(); + const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; } + + const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; } + private: std::unique_ptr encoder_; bool prefer_temporal_support_; + const VideoEncoder::EncoderInfo primary_info_; + const VideoEncoder::EncoderInfo fallback_info_; }; class StreamContext : public EncodedImageCallback { @@ -138,8 +146,11 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { void DestroyStoredEncoders(); + // This method creates encoder. May reuse previously created encoders from + // |cached_encoder_contexts_|. It's const because it's used from + // const GetEncoderInfo(). std::unique_ptr FetchOrCreateEncoderContext( - bool is_lowest_quality_stream); + bool is_lowest_quality_stream) const; webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec, int stream_idx, @@ -169,9 +180,11 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { // Used for checking the single-threaded access of the encoder interface. RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_; - // Store encoders in between calls to Release and InitEncode, so they don't - // have to be recreated. Remaining encoders are destroyed by the destructor. - std::list> cached_encoder_contexts_; + // Store previously created and released encoders , so they don't have to be + // recreated. Remaining encoders are destroyed by the destructor. + // Marked as |mutable| becuase we may need to temporarily create encoder in + // GetEncoderInfo(), which is const. + mutable std::list> cached_encoder_contexts_; const absl::optional experimental_boosted_screenshare_qp_; const bool boost_base_layer_quality_; diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index a74a2c3785..48e005f1c2 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -900,8 +900,6 @@ TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) { } TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { - EXPECT_EQ("SimulcastEncoderAdapter", - adapter_->GetEncoderInfo().implementation_name); SimulcastTestFixtureImpl::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile), kVideoCodecVP8); @@ -910,6 +908,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { encoder_names.push_back("codec2"); encoder_names.push_back("codec3"); helper_->factory()->SetEncoderNames(encoder_names); + EXPECT_EQ("SimulcastEncoderAdapter", + adapter_->GetEncoderInfo().implementation_name); EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)", adapter_->GetEncoderInfo().implementation_name);