From 084c55a63a2d9bdc71579458406d44f8bab9f454 Mon Sep 17 00:00:00 2001 From: andersc Date: Fri, 1 Sep 2017 10:38:15 -0700 Subject: [PATCH] Prepare for injectable SW decoders Pretty much mirrors the work done on the encoding side in CLs: "Clean up ownership of webrtc::VideoEncoder" https://codereview.webrtc.org/3007643002/ "Let VideoEncoderSoftwareFallbackWrapper own the wrapped encoder" https://codereview.webrtc.org/3007683002/ "WebRtcVideoEngine: Encapsulate logic for unifying internal and external video codecs" https://codereview.webrtc.org/3006713002/ BUG=webrtc:7925 Review-Url: https://codereview.webrtc.org/3009973002 Cr-Commit-Position: refs/heads/master@{#19641} --- webrtc/media/BUILD.gn | 2 + webrtc/media/engine/scopedvideodecoder.cc | 110 ++++++++++++ webrtc/media/engine/scopedvideodecoder.h | 37 ++++ .../videodecodersoftwarefallbackwrapper.cc | 4 +- .../videodecodersoftwarefallbackwrapper.h | 6 +- ...decodersoftwarefallbackwrapper_unittest.cc | 58 ++++--- webrtc/media/engine/webrtcvideoengine.cc | 158 ++++++++++-------- webrtc/media/engine/webrtcvideoengine.h | 35 ++-- 8 files changed, 285 insertions(+), 125 deletions(-) create mode 100644 webrtc/media/engine/scopedvideodecoder.cc create mode 100644 webrtc/media/engine/scopedvideodecoder.h diff --git a/webrtc/media/BUILD.gn b/webrtc/media/BUILD.gn index 0ed8fc2663..9438e6b0d7 100644 --- a/webrtc/media/BUILD.gn +++ b/webrtc/media/BUILD.gn @@ -146,6 +146,8 @@ rtc_static_library("rtc_audio_video") { "engine/nullwebrtcvideoengine.h", "engine/payload_type_mapper.cc", "engine/payload_type_mapper.h", + "engine/scopedvideodecoder.cc", + "engine/scopedvideodecoder.h", "engine/scopedvideoencoder.cc", "engine/scopedvideoencoder.h", "engine/simulcast.cc", diff --git a/webrtc/media/engine/scopedvideodecoder.cc b/webrtc/media/engine/scopedvideodecoder.cc new file mode 100644 index 0000000000..6dfbefa732 --- /dev/null +++ b/webrtc/media/engine/scopedvideodecoder.cc @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/media/engine/scopedvideodecoder.h" + +#include + +#include "webrtc/api/video_codecs/video_decoder.h" + +namespace cricket { + +namespace { + +class ScopedVideoDecoder : public webrtc::VideoDecoder { + public: + ScopedVideoDecoder(WebRtcVideoDecoderFactory* factory, + webrtc::VideoDecoder* decoder); + + int32_t InitDecode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores) override; + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) override; + int32_t Release() override; + int32_t Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + const webrtc::RTPFragmentationHeader* fragmentation, + const webrtc::CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override; + bool PrefersLateDecoding() const override; + const char* ImplementationName() const override; + + ~ScopedVideoDecoder() override; + + private: + WebRtcVideoDecoderFactory* factory_; + webrtc::VideoDecoder* decoder_; +}; + +ScopedVideoDecoder::ScopedVideoDecoder(WebRtcVideoDecoderFactory* factory, + webrtc::VideoDecoder* decoder) + : factory_(factory), decoder_(decoder) {} + +int32_t ScopedVideoDecoder::InitDecode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores) { + return decoder_->InitDecode(codec_settings, number_of_cores); +} + +int32_t ScopedVideoDecoder::RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) { + return decoder_->RegisterDecodeCompleteCallback(callback); +} + +int32_t ScopedVideoDecoder::Release() { + return decoder_->Release(); +} + +int32_t ScopedVideoDecoder::Decode( + const webrtc::EncodedImage& input_image, + bool missing_frames, + const webrtc::RTPFragmentationHeader* fragmentation, + const webrtc::CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) { + return decoder_->Decode(input_image, missing_frames, fragmentation, + codec_specific_info, render_time_ms); +} + +bool ScopedVideoDecoder::PrefersLateDecoding() const { + return decoder_->PrefersLateDecoding(); +} + +const char* ScopedVideoDecoder::ImplementationName() const { + return decoder_->ImplementationName(); +} + +ScopedVideoDecoder::~ScopedVideoDecoder() { + factory_->DestroyVideoDecoder(decoder_); +} + +} // namespace + +std::unique_ptr CreateScopedVideoDecoder( + WebRtcVideoDecoderFactory* factory, + webrtc::VideoCodecType type) { + webrtc::VideoDecoder* decoder = factory->CreateVideoDecoder(type); + if (!decoder) + return nullptr; + return std::unique_ptr( + new ScopedVideoDecoder(factory, decoder)); +} + +std::unique_ptr CreateScopedVideoDecoder( + WebRtcVideoDecoderFactory* factory, + webrtc::VideoCodecType type, + VideoDecoderParams params) { + webrtc::VideoDecoder* decoder = + factory->CreateVideoDecoderWithParams(type, params); + if (!decoder) + return nullptr; + return std::unique_ptr( + new ScopedVideoDecoder(factory, decoder)); +} + +} // namespace cricket diff --git a/webrtc/media/engine/scopedvideodecoder.h b/webrtc/media/engine/scopedvideodecoder.h new file mode 100644 index 0000000000..f921d99024 --- /dev/null +++ b/webrtc/media/engine/scopedvideodecoder.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MEDIA_ENGINE_SCOPEDVIDEODECODER_H_ +#define WEBRTC_MEDIA_ENGINE_SCOPEDVIDEODECODER_H_ + +#include + +#include "webrtc/media/engine/webrtcvideodecoderfactory.h" + +namespace cricket { + +// Helper function that creates a webrtc::VideoDecoder held by an +// std::unique_ptr instead of having to be deleted through +// WebRtcVideoDecoderFactory::DestroyVideoDecoder. The factory passed in must +// outlive the returned encoder. +// TODO(andersc): This helper function will be deleted once +// cricket::WebRtcVideoDecoderFactory is deprecated, see +// https://bugs.chromium.org/p/webrtc/issues/detail?id=7925 for more info. +std::unique_ptr CreateScopedVideoDecoder( + cricket::WebRtcVideoDecoderFactory* factory, + webrtc::VideoCodecType type); +std::unique_ptr CreateScopedVideoDecoder( + cricket::WebRtcVideoDecoderFactory* factory, + webrtc::VideoCodecType type, + VideoDecoderParams params); + +} // namespace cricket + +#endif // WEBRTC_MEDIA_ENGINE_SCOPEDVIDEODECODER_H_ diff --git a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.cc b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.cc index 6a234d54de..0e7632ba14 100644 --- a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.cc +++ b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.cc @@ -22,9 +22,9 @@ namespace webrtc { VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper( VideoCodecType codec_type, - VideoDecoder* decoder) + std::unique_ptr decoder) : codec_type_(codec_type), - decoder_(decoder), + decoder_(std::move(decoder)), decoder_initialized_(false), callback_(nullptr) {} diff --git a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h index 3984e1f1cc..d6c3e0e0a8 100644 --- a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h +++ b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h @@ -21,10 +21,10 @@ namespace webrtc { // Class used to wrap external VideoDecoders to provide a fallback option on // software decoding when a hardware decoder fails to decode a stream due to // hardware restrictions, such as max resolution. -class VideoDecoderSoftwareFallbackWrapper : public webrtc::VideoDecoder { +class VideoDecoderSoftwareFallbackWrapper : public VideoDecoder { public: VideoDecoderSoftwareFallbackWrapper(VideoCodecType codec_type, - VideoDecoder* decoder); + std::unique_ptr decoder); int32_t InitDecode(const VideoCodec* codec_settings, int32_t number_of_cores) override; @@ -47,7 +47,7 @@ class VideoDecoderSoftwareFallbackWrapper : public webrtc::VideoDecoder { bool InitFallbackDecoder(); const VideoCodecType codec_type_; - VideoDecoder* const decoder_; + std::unique_ptr decoder_; bool decoder_initialized_; VideoCodec codec_settings_; diff --git a/webrtc/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc b/webrtc/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc index b4fc7caf83..84a6a59731 100644 --- a/webrtc/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc +++ b/webrtc/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc @@ -19,7 +19,9 @@ namespace webrtc { class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test { protected: VideoDecoderSoftwareFallbackWrapperTest() - : fallback_wrapper_(kVideoCodecVP8, &fake_decoder_) {} + : fake_decoder_(new CountingFakeDecoder()), + fallback_wrapper_(kVideoCodecVP8, + std::unique_ptr(fake_decoder_)) {} class CountingFakeDecoder : public VideoDecoder { public: @@ -61,39 +63,41 @@ class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test { int release_count_ = 0; int reset_count_ = 0; }; - CountingFakeDecoder fake_decoder_; + // |fake_decoder_| is owned and released by |fallback_wrapper_|. + CountingFakeDecoder* fake_decoder_; VideoDecoderSoftwareFallbackWrapper fallback_wrapper_; }; TEST_F(VideoDecoderSoftwareFallbackWrapperTest, InitializesDecoder) { VideoCodec codec = {}; fallback_wrapper_.InitDecode(&codec, 2); - EXPECT_EQ(1, fake_decoder_.init_decode_count_); + EXPECT_EQ(1, fake_decoder_->init_decode_count_); EncodedImage encoded_image; encoded_image._frameType = kVideoFrameKey; fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); - EXPECT_EQ(1, fake_decoder_.init_decode_count_) + EXPECT_EQ(1, fake_decoder_->init_decode_count_) << "Initialized decoder should not be reinitialized."; - EXPECT_EQ(1, fake_decoder_.decode_count_); + EXPECT_EQ(1, fake_decoder_->decode_count_); } TEST_F(VideoDecoderSoftwareFallbackWrapperTest, UsesFallbackDecoderAfterOnInitDecodeFailure) { VideoCodec codec = {}; - fake_decoder_.init_decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + fake_decoder_->init_decode_return_code_ = + WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; fallback_wrapper_.InitDecode(&codec, 2); - EXPECT_EQ(1, fake_decoder_.init_decode_count_); + EXPECT_EQ(1, fake_decoder_->init_decode_count_); EncodedImage encoded_image; encoded_image._frameType = kVideoFrameKey; fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); - EXPECT_EQ(2, fake_decoder_.init_decode_count_) + EXPECT_EQ(2, fake_decoder_->init_decode_count_) << "Should have attempted reinitializing the fallback decoder on " "keyframe."; // Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW // decoder. - EXPECT_EQ(0, fake_decoder_.decode_count_) + EXPECT_EQ(0, fake_decoder_->decode_count_) << "Decoder used even though no InitDecode had succeeded."; } @@ -103,41 +107,41 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest, fallback_wrapper_.InitDecode(&codec, 2); // Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW // decoder. - fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; EncodedImage encoded_image; fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); - EXPECT_EQ(1, fake_decoder_.decode_count_); + EXPECT_EQ(1, fake_decoder_->decode_count_); // Fail -> fake_decoder shouldn't be used anymore. fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); - EXPECT_EQ(1, fake_decoder_.decode_count_) + EXPECT_EQ(1, fake_decoder_->decode_count_) << "Decoder used even though fallback should be active."; // Should be able to recover on a keyframe. encoded_image._frameType = kVideoFrameKey; - fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_OK; fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); - EXPECT_EQ(2, fake_decoder_.decode_count_) + EXPECT_EQ(2, fake_decoder_->decode_count_) << "Wrapper did not try to decode a keyframe using registered decoder."; encoded_image._frameType = kVideoFrameDelta; fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); - EXPECT_EQ(3, fake_decoder_.decode_count_) + EXPECT_EQ(3, fake_decoder_->decode_count_) << "Decoder not used on future delta frames."; } TEST_F(VideoDecoderSoftwareFallbackWrapperTest, DoesNotFallbackOnEveryError) { VideoCodec codec = {}; fallback_wrapper_.InitDecode(&codec, 2); - fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; EncodedImage encoded_image; EXPECT_EQ( - fake_decoder_.decode_return_code_, + fake_decoder_->decode_return_code_, fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1)); - EXPECT_EQ(1, fake_decoder_.decode_count_); + EXPECT_EQ(1, fake_decoder_->decode_count_); fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); - EXPECT_EQ(2, fake_decoder_.decode_count_) + EXPECT_EQ(2, fake_decoder_->decode_count_) << "Decoder should be active even though previous decode failed."; } @@ -145,15 +149,15 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsReleaseCall) { VideoCodec codec = {}; fallback_wrapper_.InitDecode(&codec, 2); fallback_wrapper_.Release(); - EXPECT_EQ(1, fake_decoder_.release_count_); + EXPECT_EQ(1, fake_decoder_->release_count_); - fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; EncodedImage encoded_image; fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); - EXPECT_EQ(1, fake_decoder_.release_count_) + EXPECT_EQ(1, fake_decoder_->release_count_) << "Decoder should not be released during fallback."; fallback_wrapper_.Release(); - EXPECT_EQ(2, fake_decoder_.release_count_); + EXPECT_EQ(2, fake_decoder_->release_count_); } // TODO(pbos): Fake a VP8 frame well enough to actually receive a callback from @@ -177,13 +181,13 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest, VideoCodec codec = {}; fallback_wrapper_.InitDecode(&codec, 2); fallback_wrapper_.RegisterDecodeCompleteCallback(&callback); - EXPECT_EQ(&callback, fake_decoder_.decode_complete_callback_); + EXPECT_EQ(&callback, fake_decoder_->decode_complete_callback_); - fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; EncodedImage encoded_image; fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); fallback_wrapper_.RegisterDecodeCompleteCallback(&callback2); - EXPECT_EQ(&callback2, fake_decoder_.decode_complete_callback_); + EXPECT_EQ(&callback2, fake_decoder_->decode_complete_callback_); } TEST_F(VideoDecoderSoftwareFallbackWrapperTest, @@ -191,7 +195,7 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest, VideoCodec codec = {}; fallback_wrapper_.InitDecode(&codec, 2); - fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; EncodedImage encoded_image; fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); // Hard coded expected value since libvpx is the software implementation name diff --git a/webrtc/media/engine/webrtcvideoengine.cc b/webrtc/media/engine/webrtcvideoengine.cc index 00d138ab4d..dd0a7b029a 100644 --- a/webrtc/media/engine/webrtcvideoengine.cc +++ b/webrtc/media/engine/webrtcvideoengine.cc @@ -24,6 +24,7 @@ #include "webrtc/media/engine/constants.h" #include "webrtc/media/engine/internaldecoderfactory.h" #include "webrtc/media/engine/internalencoderfactory.h" +#include "webrtc/media/engine/scopedvideodecoder.h" #include "webrtc/media/engine/scopedvideoencoder.h" #include "webrtc/media/engine/simulcast.h" #include "webrtc/media/engine/simulcast_encoder_adapter.h" @@ -71,6 +72,17 @@ class EncoderFactoryAdapter { virtual std::unique_ptr clone() const = 0; }; +class DecoderFactoryAdapter { + public: + virtual ~DecoderFactoryAdapter() {} + + virtual std::unique_ptr CreateVideoDecoder( + webrtc::VideoCodecType type, + const VideoDecoderParams& decoder_params) const = 0; + + virtual std::unique_ptr clone() const = 0; +}; + namespace { // Wraps cricket::WebRtcVideoEncoderFactory* into common EncoderFactoryAdapter @@ -104,6 +116,31 @@ class CricketEncoderFactoryAdapter : public EncoderFactoryAdapter { WebRtcVideoEncoderFactory* const external_encoder_factory_; }; +class CricketDecoderFactoryAdapter : public DecoderFactoryAdapter { + public: + explicit CricketDecoderFactoryAdapter( + WebRtcVideoDecoderFactory* external_decoder_factory) + : internal_decoder_factory_(new InternalDecoderFactory()), + external_decoder_factory_(external_decoder_factory) {} + + private: + explicit CricketDecoderFactoryAdapter( + const CricketDecoderFactoryAdapter& other) + : CricketDecoderFactoryAdapter(other.external_decoder_factory_) {} + + std::unique_ptr CreateVideoDecoder( + webrtc::VideoCodecType type, + const VideoDecoderParams& decoder_params) const override; + + std::unique_ptr clone() const override { + return std::unique_ptr( + new CricketDecoderFactoryAdapter(*this)); + } + + const std::unique_ptr internal_decoder_factory_; + WebRtcVideoDecoderFactory* const external_decoder_factory_; +}; + // If this field trial is enabled, we will enable sending FlexFEC and disable // sending ULPFEC whenever the former has been negotiated in the SDPs. bool IsFlexfecFieldTrialEnabled() { @@ -367,7 +404,8 @@ void DefaultUnsignalledSsrcHandler::SetDefaultSink( WebRtcVideoEngine::WebRtcVideoEngine() : initialized_(false), - external_decoder_factory_(NULL), + decoder_factory_(new CricketDecoderFactoryAdapter( + nullptr /* external_decoder_factory */)), encoder_factory_(new CricketEncoderFactoryAdapter( nullptr /* external_encoder_factory */)) { LOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine()"; @@ -389,7 +427,7 @@ WebRtcVideoChannel* WebRtcVideoEngine::CreateChannel( RTC_DCHECK(initialized_); LOG(LS_INFO) << "CreateChannel. Options: " << options.ToString(); return new WebRtcVideoChannel(call, config, options, *encoder_factory_, - external_decoder_factory_); + *decoder_factory_); } std::vector WebRtcVideoEngine::codecs() const { @@ -425,7 +463,7 @@ RtpCapabilities WebRtcVideoEngine::GetCapabilities() const { void WebRtcVideoEngine::SetExternalDecoderFactory( WebRtcVideoDecoderFactory* decoder_factory) { RTC_DCHECK(!initialized_); - external_decoder_factory_ = decoder_factory; + decoder_factory_.reset(new CricketDecoderFactoryAdapter(decoder_factory)); } void WebRtcVideoEngine::SetExternalEncoderFactory( @@ -503,13 +541,13 @@ WebRtcVideoChannel::WebRtcVideoChannel( const MediaConfig& config, const VideoOptions& options, const EncoderFactoryAdapter& encoder_factory, - WebRtcVideoDecoderFactory* external_decoder_factory) + const DecoderFactoryAdapter& decoder_factory) : VideoMediaChannel(config), call_(call), unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), video_config_(config.video), encoder_factory_(encoder_factory.clone()), - external_decoder_factory_(external_decoder_factory), + decoder_factory_(decoder_factory.clone()), default_send_options_(options), last_stats_log_ms_(-1) { RTC_DCHECK(thread_checker_.CalledOnValidThread()); @@ -1125,7 +1163,7 @@ bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp, config.sync_group = sp.sync_label; receive_streams_[ssrc] = new WebRtcVideoReceiveStream( - call_, sp, std::move(config), external_decoder_factory_, default_stream, + call_, sp, std::move(config), *decoder_factory_, default_stream, recv_codecs_, flexfec_config); return true; @@ -2063,7 +2101,7 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( webrtc::Call* call, const StreamParams& sp, webrtc::VideoReceiveStream::Config config, - WebRtcVideoDecoderFactory* external_decoder_factory, + const DecoderFactoryAdapter& decoder_factory, bool default_stream, const std::vector& recv_codecs, const webrtc::FlexfecReceiveStream::Config& flexfec_config) @@ -2074,12 +2112,13 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( config_(std::move(config)), flexfec_config_(flexfec_config), flexfec_stream_(nullptr), - external_decoder_factory_(external_decoder_factory), + decoder_factory_(decoder_factory.clone()), sink_(NULL), first_frame_timestamp_(-1), estimated_remote_start_ntp_time_ms_(0) { config_.renderer = this; - std::vector old_decoders; + std::map> + old_decoders; ConfigureCodecs(recv_codecs, &old_decoders); ConfigureFlexfecCodec(flexfec_config.payload_type); MaybeRecreateWebRtcFlexfecStream(); @@ -2087,28 +2126,13 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( RTC_DCHECK(old_decoders.empty()); } -WebRtcVideoChannel::WebRtcVideoReceiveStream::AllocatedDecoder:: - AllocatedDecoder(webrtc::VideoDecoder* decoder, - webrtc::VideoCodecType type, - bool external) - : decoder(decoder), - external_decoder(nullptr), - type(type), - external(external) { - if (external) { - external_decoder = decoder; - this->decoder = - new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder); - } -} - WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { if (flexfec_stream_) { MaybeDissociateFlexfecFromVideo(); call_->DestroyFlexfecReceiveStream(flexfec_stream_); } call_->DestroyVideoReceiveStream(stream_); - ClearDecoders(&allocated_decoders_); + allocated_decoders_.clear(); } const std::vector& @@ -2129,53 +2153,59 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetFirstPrimarySsrc() const { } } -WebRtcVideoChannel::WebRtcVideoReceiveStream::AllocatedDecoder -WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder( - std::vector* old_decoders, - const VideoCodec& codec) { - webrtc::VideoCodecType type = webrtc::PayloadStringToCodecType(codec.name); - - for (size_t i = 0; i < old_decoders->size(); ++i) { - if ((*old_decoders)[i].type == type) { - AllocatedDecoder decoder = (*old_decoders)[i]; - (*old_decoders)[i] = old_decoders->back(); - old_decoders->pop_back(); - return decoder; +std::unique_ptr +CricketDecoderFactoryAdapter::CreateVideoDecoder( + webrtc::VideoCodecType type, + const VideoDecoderParams& decoder_params) const { + if (external_decoder_factory_ != nullptr) { + std::unique_ptr external_decoder = + CreateScopedVideoDecoder(external_decoder_factory_, type, + decoder_params); + if (external_decoder) { + std::unique_ptr internal_decoder( + new webrtc::VideoDecoderSoftwareFallbackWrapper( + type, std::move(external_decoder))); + return internal_decoder; } } - if (external_decoder_factory_ != NULL) { - webrtc::VideoDecoder* decoder = - external_decoder_factory_->CreateVideoDecoderWithParams( - type, {stream_params_.id}); - if (decoder != NULL) { - return AllocatedDecoder(decoder, type, true /* is_external */); - } - } - - InternalDecoderFactory internal_decoder_factory; - return AllocatedDecoder(internal_decoder_factory.CreateVideoDecoderWithParams( - type, {stream_params_.id}), - type, false /* is_external */); + std::unique_ptr internal_decoder( + internal_decoder_factory_->CreateVideoDecoderWithParams(type, + decoder_params)); + return internal_decoder; } void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( const std::vector& recv_codecs, - std::vector* old_decoders) { - *old_decoders = allocated_decoders_; + std::map>* + old_decoders) { + *old_decoders = std::move(allocated_decoders_); allocated_decoders_.clear(); config_.decoders.clear(); for (size_t i = 0; i < recv_codecs.size(); ++i) { - AllocatedDecoder allocated_decoder = - CreateOrReuseVideoDecoder(old_decoders, recv_codecs[i].codec); - allocated_decoders_.push_back(allocated_decoder); + webrtc::VideoCodecType type = + webrtc::PayloadStringToCodecType(recv_codecs[i].codec.name); + std::unique_ptr new_decoder; + + auto it = old_decoders->find(type); + if (it != old_decoders->end()) { + new_decoder = std::move(it->second); + old_decoders->erase(it); + } + + if (!new_decoder) { + new_decoder = + decoder_factory_->CreateVideoDecoder(type, {stream_params_.id}); + } webrtc::VideoReceiveStream::Decoder decoder; - decoder.decoder = allocated_decoder.decoder; + decoder.decoder = new_decoder.get(); decoder.payload_type = recv_codecs[i].codec.id; decoder.payload_name = recv_codecs[i].codec.name; decoder.codec_params = recv_codecs[i].codec.params; config_.decoders.push_back(decoder); + + allocated_decoders_.insert(std::make_pair(type, std::move(new_decoder))); } config_.rtp.rtx_associated_payload_types.clear(); @@ -2253,7 +2283,8 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( const ChangedRecvParameters& params) { bool video_needs_recreation = false; bool flexfec_needs_recreation = false; - std::vector old_decoders; + std::map> + old_decoders; if (params.codec_settings) { ConfigureCodecs(*params.codec_settings, &old_decoders); video_needs_recreation = true; @@ -2277,7 +2308,6 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( LOG(LS_INFO) << "RecreateWebRtcVideoStream (recv) because of SetRecvParameters"; RecreateWebRtcVideoStream(); - ClearDecoders(&old_decoders); } } @@ -2322,18 +2352,6 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream:: } } -void WebRtcVideoChannel::WebRtcVideoReceiveStream::ClearDecoders( - std::vector* allocated_decoders) { - for (size_t i = 0; i < allocated_decoders->size(); ++i) { - if ((*allocated_decoders)[i].external) { - external_decoder_factory_->DestroyVideoDecoder( - (*allocated_decoders)[i].external_decoder); - } - delete (*allocated_decoders)[i].decoder; - } - allocated_decoders->clear(); -} - void WebRtcVideoChannel::WebRtcVideoReceiveStream::OnFrame( const webrtc::VideoFrame& frame) { rtc::CritScope crit(&sink_lock_); diff --git a/webrtc/media/engine/webrtcvideoengine.h b/webrtc/media/engine/webrtcvideoengine.h index 3224d0027f..8f7a790441 100644 --- a/webrtc/media/engine/webrtcvideoengine.h +++ b/webrtc/media/engine/webrtcvideoengine.h @@ -47,6 +47,7 @@ class Thread; namespace cricket { +class DecoderFactoryAdapter; class EncoderFactoryAdapter; class VideoCapturer; class VideoProcessor; @@ -121,7 +122,7 @@ class WebRtcVideoEngine { private: bool initialized_; - WebRtcVideoDecoderFactory* external_decoder_factory_; + std::unique_ptr decoder_factory_; std::unique_ptr encoder_factory_; }; @@ -131,7 +132,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { const MediaConfig& config, const VideoOptions& options, const EncoderFactoryAdapter& encoder_factory, - WebRtcVideoDecoderFactory* external_decoder_factory); + const DecoderFactoryAdapter& decoder_factory); ~WebRtcVideoChannel() override; // VideoMediaChannel implementation @@ -369,7 +370,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { webrtc::Call* call, const StreamParams& sp, webrtc::VideoReceiveStream::Config config, - WebRtcVideoDecoderFactory* external_decoder_factory, + const DecoderFactoryAdapter& decoder_factory, bool default_stream, const std::vector& recv_codecs, const webrtc::FlexfecReceiveStream::Config& flexfec_config); @@ -394,30 +395,17 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); private: - struct AllocatedDecoder { - AllocatedDecoder(webrtc::VideoDecoder* decoder, - webrtc::VideoCodecType type, - bool external); - webrtc::VideoDecoder* decoder; - // Decoder wrapped into a fallback decoder to permit software fallback. - webrtc::VideoDecoder* external_decoder; - webrtc::VideoCodecType type; - bool external; - }; - void RecreateWebRtcVideoStream(); void MaybeRecreateWebRtcFlexfecStream(); void MaybeAssociateFlexfecWithVideo(); void MaybeDissociateFlexfecFromVideo(); - void ConfigureCodecs(const std::vector& recv_codecs, - std::vector* old_codecs); + void ConfigureCodecs( + const std::vector& recv_codecs, + std::map>* + old_codecs); void ConfigureFlexfecCodec(int flexfec_payload_type); - AllocatedDecoder CreateOrReuseVideoDecoder( - std::vector* old_decoder, - const VideoCodec& codec); - void ClearDecoders(std::vector* allocated_decoders); std::string GetCodecNameFromPayloadType(int payload_type); @@ -433,8 +421,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { webrtc::FlexfecReceiveStream::Config flexfec_config_; webrtc::FlexfecReceiveStream* flexfec_stream_; - WebRtcVideoDecoderFactory* const external_decoder_factory_; - std::vector allocated_decoders_; + std::unique_ptr decoder_factory_; + std::map> + allocated_decoders_; rtc::CriticalSection sink_lock_; rtc::VideoSinkInterface* sink_ GUARDED_BY(sink_lock_); @@ -497,7 +486,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { rtc::Optional> send_rtp_extensions_; std::unique_ptr encoder_factory_; - WebRtcVideoDecoderFactory* const external_decoder_factory_; + std::unique_ptr decoder_factory_; std::vector recv_codecs_; std::vector recv_rtp_extensions_; // See reason for keeping track of the FlexFEC payload type separately in