diff --git a/api/video_codecs/video_encoder.cc b/api/video_codecs/video_encoder.cc index 3a848f39ed..43f959ba5a 100644 --- a/api/video_codecs/video_encoder.cc +++ b/api/video_codecs/video_encoder.cc @@ -90,7 +90,8 @@ VideoEncoder::EncoderInfo::EncoderInfo() has_internal_source(false), fps_allocation{absl::InlinedVector( 1, - kMaxFramerateFraction)} {} + kMaxFramerateFraction)}, + supports_simulcast(false) {} VideoEncoder::EncoderInfo::EncoderInfo(const EncoderInfo&) = default; diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h index 766ea75712..fbbd4ed949 100644 --- a/api/video_codecs/video_encoder.h +++ b/api/video_codecs/video_encoder.h @@ -216,6 +216,13 @@ class RTC_EXPORT VideoEncoder { // Recommended bitrate limits for different resolutions. std::vector resolution_bitrate_limits; + + // If true, this encoder has internal support for generating simulcast + // streams. Otherwise, an adapter class will be needed. + // Even if true, the config provided to InitEncode() might not be supported, + // in such case the encoder should return + // WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED. + bool supports_simulcast; }; struct RateControlParameters { diff --git a/media/BUILD.gn b/media/BUILD.gn index 1a8c24e4c7..b451fefbeb 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -169,6 +169,7 @@ rtc_static_library("rtc_simulcast_encoder_adapter") { "../api/video:video_frame", "../api/video:video_frame_i420", "../api/video:video_rtp_headers", + "../api/video_codecs:rtc_software_fallback_wrappers", "../api/video_codecs:video_codecs_api", "../modules/video_coding:video_codec_interface", "../modules/video_coding:video_coding_utility", diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 667b3032e3..6a585c6c7a 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -25,6 +25,7 @@ #include "api/video/video_rotation.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory.h" +#include "api/video_codecs/video_encoder_software_fallback_wrapper.h" #include "modules/video_coding/include/video_error_codes.h" #include "modules/video_coding/utility/simulcast_rate_allocator.h" #include "rtc_base/atomic_ops.h" @@ -70,6 +71,17 @@ int NumberOfStreams(const webrtc::VideoCodec& codec) { return streams; } +int NumActiveStreams(const webrtc::VideoCodec& codec) { + int num_configured_streams = NumberOfStreams(codec); + int num_active_streams = 0; + for (int i = 0; i < num_configured_streams; ++i) { + if (codec.simulcastStream[i].active) { + ++num_active_streams; + } + } + return num_active_streams; +} + int VerifyCodec(const webrtc::VideoCodec* inst) { if (inst == nullptr) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -124,14 +136,21 @@ namespace webrtc { SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory, const SdpVideoFormat& format) + : SimulcastEncoderAdapter(factory, nullptr, format) {} + +SimulcastEncoderAdapter::SimulcastEncoderAdapter( + VideoEncoderFactory* primary_factory, + VideoEncoderFactory* fallback_factory, + const SdpVideoFormat& format) : inited_(0), - factory_(factory), + primary_encoder_factory_(primary_factory), + fallback_encoder_factory_(fallback_factory), video_format_(format), encoded_complete_callback_(nullptr), experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()), boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials() .Vp8BoostBaseLayerQuality()) { - RTC_DCHECK(factory_); + RTC_DCHECK(primary_factory); encoder_info_.implementation_name = "SimulcastEncoderAdapter"; // The adapter is typically created on the worker thread, but operated on @@ -196,7 +215,8 @@ int SimulcastEncoderAdapter::InitEncode( int number_of_streams = NumberOfStreams(*inst); RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams); - const bool doing_simulcast = (number_of_streams > 1); + bool doing_simulcast_using_adapter = (number_of_streams > 1); + int num_active_streams = NumActiveStreams(*inst); codec_ = *inst; SimulcastRateAllocator rate_allocator(codec_); @@ -225,14 +245,48 @@ int SimulcastEncoderAdapter::InitEncode( RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams); RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams); + const SdpVideoFormat format( + codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264"); + for (int i = 0; i < number_of_streams; ++i) { + // If an existing encoder instance exists, reuse it. + // TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here, + // when we start storing that state outside the encoder wrappers. + std::unique_ptr encoder; + if (!stored_encoders_.empty()) { + encoder = std::move(stored_encoders_.top()); + stored_encoders_.pop(); + } else { + encoder = primary_encoder_factory_->CreateVideoEncoder(format); + if (fallback_encoder_factory_ != nullptr) { + encoder = CreateVideoEncoderSoftwareFallbackWrapper( + fallback_encoder_factory_->CreateVideoEncoder(format), + std::move(encoder)); + } + } + + bool encoder_initialized = false; + if (doing_simulcast_using_adapter && i == 0 && + encoder->GetEncoderInfo().supports_simulcast) { + ret = encoder->InitEncode(&codec_, settings); + if (ret < 0) { + encoder->Release(); + } else { + doing_simulcast_using_adapter = false; + number_of_streams = 1; + encoder_initialized = true; + } + } + VideoCodec stream_codec; uint32_t start_bitrate_kbps = start_bitrates[i]; - const bool send_stream = start_bitrate_kbps > 0; - if (!doing_simulcast) { + const bool send_stream = doing_simulcast_using_adapter + ? start_bitrate_kbps > 0 + : num_active_streams > 0; + if (!doing_simulcast_using_adapter) { stream_codec = codec_; - stream_codec.numberOfSimulcastStreams = 1; - + stream_codec.numberOfSimulcastStreams = + std::max(1, stream_codec.numberOfSimulcastStreams); } else { // Cap start bitrate to the min bitrate in order to avoid strange codec // behavior. Since sending will be false, this should not matter. @@ -253,39 +307,32 @@ int SimulcastEncoderAdapter::InitEncode( stream_codec.qpMax = kDefaultMaxQp; } - // If an existing encoder instance exists, reuse it. - // TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here, - // when we start storing that state outside the encoder wrappers. - std::unique_ptr encoder; - if (!stored_encoders_.empty()) { - encoder = std::move(stored_encoders_.top()); - stored_encoders_.pop(); - } else { - encoder = factory_->CreateVideoEncoder(SdpVideoFormat( - codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264")); + if (!encoder_initialized) { + ret = encoder->InitEncode(&stream_codec, settings); + if (ret < 0) { + // Explicitly destroy the current encoder; because we haven't registered + // a StreamInfo for it yet, Release won't do anything about it. + encoder.reset(); + Release(); + return ret; + } } - ret = encoder->InitEncode(&stream_codec, settings); - if (ret < 0) { - // Explicitly destroy the current encoder; because we haven't registered a - // StreamInfo for it yet, Release won't do anything about it. - encoder.reset(); - Release(); - return ret; - } - - std::unique_ptr callback( - new AdapterEncodedImageCallback(this, i)); - encoder->RegisterEncodeCompleteCallback(callback.get()); - streaminfos_.emplace_back(std::move(encoder), std::move(callback), - stream_codec.width, stream_codec.height, - send_stream); - - if (!doing_simulcast) { + if (!doing_simulcast_using_adapter) { // Without simulcast, just pass through the encoder info from the one // active encoder. - encoder_info_ = streaminfos_[0].encoder->GetEncoderInfo(); + encoder_info_ = encoder->GetEncoderInfo(); + encoder->RegisterEncodeCompleteCallback(encoded_complete_callback_); + streaminfos_.emplace_back(std::move(encoder), nullptr, stream_codec.width, + stream_codec.height, send_stream); } else { + std::unique_ptr callback( + new AdapterEncodedImageCallback(this, i)); + encoder->RegisterEncodeCompleteCallback(callback.get()); + streaminfos_.emplace_back(std::move(encoder), std::move(callback), + stream_codec.width, stream_codec.height, + send_stream); + const EncoderInfo encoder_impl_info = streaminfos_[i].encoder->GetEncoderInfo(); @@ -334,7 +381,7 @@ int SimulcastEncoderAdapter::InitEncode( } } - if (doing_simulcast) { + if (doing_simulcast_using_adapter) { encoder_info_.implementation_name += ")"; } @@ -449,6 +496,9 @@ int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback( EncodedImageCallback* callback) { RTC_DCHECK_RUN_ON(&encoder_queue_); encoded_complete_callback_ = callback; + if (streaminfos_.size() == 1) { + streaminfos_[0].encoder->RegisterEncodeCompleteCallback(callback); + } return WEBRTC_VIDEO_CODEC_OK; } @@ -468,6 +518,12 @@ void SimulcastEncoderAdapter::SetRates( codec_.maxFramerate = static_cast(parameters.framerate_fps + 0.5); + if (streaminfos_.size() == 1) { + // Not doing simulcast. + streaminfos_[0].encoder->SetRates(parameters); + return; + } + for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { uint32_t stream_bitrate_kbps = parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000; diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h index 4e0346e645..591839c30d 100644 --- a/media/engine/simulcast_encoder_adapter.h +++ b/media/engine/simulcast_encoder_adapter.h @@ -38,8 +38,15 @@ class VideoEncoderFactory; // interfaces should be called from the encoder task queue. class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { public: - explicit SimulcastEncoderAdapter(VideoEncoderFactory* factory, - const SdpVideoFormat& format); + // TODO(bugs.webrtc.org/11000): Remove when downstream usage is gone. + SimulcastEncoderAdapter(VideoEncoderFactory* primarty_factory, + const SdpVideoFormat& format); + // |primary_factory| produces the first-choice encoders to use. + // |fallback_factory|, if non-null, is used to create fallback encoder that + // will be used if InitEncode() fails for the primary encoder. + SimulcastEncoderAdapter(VideoEncoderFactory* primary_factory, + VideoEncoderFactory* fallback_factory, + const SdpVideoFormat& format); virtual ~SimulcastEncoderAdapter(); // Implements VideoEncoder. @@ -106,7 +113,8 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { void DestroyStoredEncoders(); volatile int inited_; // Accessed atomically. - VideoEncoderFactory* const factory_; + VideoEncoderFactory* const primary_encoder_factory_; + VideoEncoderFactory* const fallback_encoder_factory_; const SdpVideoFormat video_format_; VideoCodec codec_; std::vector streaminfos_; diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index 60fc814f03..48767dc754 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -171,6 +171,9 @@ class MockVideoEncoderFactory : public VideoEncoderFactory { const std::vector& encoders() const; void SetEncoderNames(const std::vector& encoder_names); void set_init_encode_return_value(int32_t value); + void set_supports_simulcast(bool supports_simulcast) { + supports_simulcast_ = supports_simulcast; + } void DestroyVideoEncoder(VideoEncoder* encoder); @@ -178,6 +181,7 @@ class MockVideoEncoderFactory : public VideoEncoderFactory { int32_t init_encode_return_value_ = 0; std::vector encoders_; std::vector encoder_names_; + bool supports_simulcast_ = false; }; class MockVideoEncoder : public VideoEncoder { @@ -226,6 +230,7 @@ class MockVideoEncoder : public VideoEncoder { info.is_hardware_accelerated = is_hardware_accelerated_; info.has_internal_source = has_internal_source_; info.fps_allocation[0] = fps_allocation_; + info.supports_simulcast = supports_simulcast_; return info; } @@ -277,6 +282,12 @@ class MockVideoEncoder : public VideoEncoder { RateControlParameters last_set_rates() const { return last_set_rates_; } + void set_supports_simulcast(bool supports_simulcast) { + supports_simulcast_ = supports_simulcast; + } + + bool supports_simulcast() const { return supports_simulcast_; } + private: MockVideoEncoderFactory* const factory_; bool supports_native_handle_ = false; @@ -288,6 +299,7 @@ class MockVideoEncoder : public VideoEncoder { int32_t init_encode_return_value_ = 0; VideoEncoder::RateControlParameters last_set_rates_; FramerateFractions fps_allocation_; + bool supports_simulcast_ = false; VideoCodec codec_; EncodedImageCallback* callback_; @@ -308,6 +320,7 @@ std::unique_ptr MockVideoEncoderFactory::CreateVideoEncoder( ? "codec_implementation_name" : encoder_names_[encoders_.size()]; encoder->set_implementation_name(encoder_name); + encoder->set_supports_simulcast(supports_simulcast_); encoders_.push_back(encoder.get()); return encoder; } @@ -340,19 +353,26 @@ void MockVideoEncoderFactory::set_init_encode_return_value(int32_t value) { class TestSimulcastEncoderAdapterFakeHelper { public: - TestSimulcastEncoderAdapterFakeHelper() - : factory_(new MockVideoEncoderFactory()) {} + explicit TestSimulcastEncoderAdapterFakeHelper(bool use_fallback_factory) + : primary_factory_(new MockVideoEncoderFactory()), + fallback_factory_(use_fallback_factory ? new MockVideoEncoderFactory() + : nullptr) {} // Can only be called once as the SimulcastEncoderAdapter will take the // ownership of |factory_|. VideoEncoder* CreateMockEncoderAdapter() { - return new SimulcastEncoderAdapter(factory_.get(), SdpVideoFormat("VP8")); + return new SimulcastEncoderAdapter( + primary_factory_.get(), fallback_factory_.get(), SdpVideoFormat("VP8")); } - MockVideoEncoderFactory* factory() { return factory_.get(); } + MockVideoEncoderFactory* factory() { return primary_factory_.get(); } + MockVideoEncoderFactory* fallback_factory() { + return fallback_factory_.get(); + } private: - std::unique_ptr factory_; + std::unique_ptr primary_factory_; + std::unique_ptr fallback_factory_; }; static const int kTestTemporalLayerProfile[3] = {3, 2, 1}; @@ -361,17 +381,26 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, public EncodedImageCallback { public: TestSimulcastEncoderAdapterFake() - : helper_(new TestSimulcastEncoderAdapterFakeHelper()), - adapter_(helper_->CreateMockEncoderAdapter()), - last_encoded_image_width_(-1), + : last_encoded_image_width_(-1), last_encoded_image_height_(-1), - last_encoded_image_simulcast_index_(-1) {} + last_encoded_image_simulcast_index_(-1), + use_fallback_factory_(false) {} + virtual ~TestSimulcastEncoderAdapterFake() { if (adapter_) { adapter_->Release(); } } + void SetUp() override { + helper_ = std::make_unique( + use_fallback_factory_); + adapter_.reset(helper_->CreateMockEncoderAdapter()); + last_encoded_image_width_ = -1; + last_encoded_image_height_ = -1; + last_encoded_image_simulcast_index_ = -1; + } + Result OnEncodedImage(const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) override { @@ -482,6 +511,7 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, int last_encoded_image_height_; int last_encoded_image_simulcast_index_; std::unique_ptr rate_allocator_; + bool use_fallback_factory_; }; TEST_F(TestSimulcastEncoderAdapterFake, InitEncode) { @@ -1218,5 +1248,114 @@ TEST_F(TestSimulcastEncoderAdapterFake, SetRateDistributesBandwithAllocation) { } } +TEST_F(TestSimulcastEncoderAdapterFake, SupportsSimulcast) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + + // Indicate that mock encoders internally support simulcast. + helper_->factory()->set_supports_simulcast(true); + adapter_->RegisterEncodeCompleteCallback(this); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + + // Only one encoder should have been produced. + ASSERT_EQ(1u, helper_->factory()->encoders().size()); + + rtc::scoped_refptr buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + EXPECT_CALL(*helper_->factory()->encoders()[0], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + std::vector frame_types(3, VideoFrameType::kVideoFrameKey); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, SupportsFallback) { + // Enable support for fallback encoder factory and re-setup. + use_fallback_factory_ = true; + SetUp(); + + SetupCodec(); + + // Make sure we have bitrate for all layers. + DataRate max_bitrate = DataRate::Zero(); + for (int i = 0; i < 3; ++i) { + max_bitrate += DataRate::kbps(codec_.simulcastStream[i].maxBitrate); + } + const auto rate_settings = VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(max_bitrate.bps(), 30)), + 30.0, max_bitrate); + adapter_->SetRates(rate_settings); + + std::vector primary_encoders = + helper_->factory()->encoders(); + std::vector fallback_encoders = + helper_->fallback_factory()->encoders(); + + ASSERT_EQ(3u, primary_encoders.size()); + ASSERT_EQ(3u, fallback_encoders.size()); + + // Create frame to test with. + rtc::scoped_refptr buffer(I420Buffer::Create(1280, 720)); + VideoFrame input_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(100) + .set_timestamp_ms(1000) + .set_rotation(kVideoRotation_180) + .build(); + std::vector frame_types(3, VideoFrameType::kVideoFrameKey); + + // All primary encoders used. + for (auto codec : primary_encoders) { + EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + } + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + + // Trigger fallback on first encoder. + primary_encoders[0]->set_init_encode_return_value( + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(rate_settings); + EXPECT_CALL(*fallback_encoders[0], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*primary_encoders[1], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*primary_encoders[2], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + + // Trigger fallback on all encoder. + primary_encoders[1]->set_init_encode_return_value( + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + primary_encoders[2]->set_init_encode_return_value( + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(rate_settings); + EXPECT_CALL(*fallback_encoders[0], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*fallback_encoders[1], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*fallback_encoders[2], Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); + + // Return to primary encoders on all streams. + for (int i = 0; i < 3; ++i) { + primary_encoders[i]->set_init_encode_return_value(WEBRTC_VIDEO_CODEC_OK); + } + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->SetRates(rate_settings); + for (auto codec : primary_encoders) { + EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + } + EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types)); +} + } // namespace test } // namespace webrtc diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/modules/video_coding/codecs/h264/h264_encoder_impl.cc index 5ec1187946..66861e6e74 100644 --- a/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -623,6 +623,7 @@ VideoEncoder::EncoderInfo H264EncoderImpl::GetEncoderInfo() const { VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold); info.is_hardware_accelerated = false; info.has_internal_source = false; + info.supports_simulcast = true; return info; } diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index c8e47d4345..d4f18e08f5 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -1212,6 +1212,7 @@ VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const { rate_control_settings_.LibvpxVp8TrustedRateController(); info.is_hardware_accelerated = false; info.has_internal_source = false; + info.supports_simulcast = true; const bool enable_scaling = encoders_.size() == 1 && vpx_configs_[0].rc_dropframe_thresh > 0 &&