diff --git a/media/engine/convert_legacy_video_factory.cc b/media/engine/convert_legacy_video_factory.cc index d40f8699dd..6495e277c4 100644 --- a/media/engine/convert_legacy_video_factory.cc +++ b/media/engine/convert_legacy_video_factory.cc @@ -15,7 +15,6 @@ #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" -#include "media/base/h264_profile_level_id.h" #include "media/engine/internaldecoderfactory.h" #include "media/engine/internalencoderfactory.h" #include "media/engine/scopedvideodecoder.h" @@ -32,34 +31,6 @@ namespace cricket { namespace { -bool IsSameFormat(const webrtc::SdpVideoFormat& format1, - const webrtc::SdpVideoFormat& format2) { - // If different names (case insensitive), then not same formats. - if (!CodecNamesEq(format1.name, format2.name)) - return false; - // For every format besides H264, comparing names is enough. - if (!CodecNamesEq(format1.name.c_str(), kH264CodecName)) - return true; - // Compare H264 profiles. - const rtc::Optional profile_level_id = - webrtc::H264::ParseSdpProfileLevelId(format1.parameters); - const rtc::Optional other_profile_level_id = - webrtc::H264::ParseSdpProfileLevelId(format2.parameters); - // Compare H264 profiles, but not levels. - return profile_level_id && other_profile_level_id && - profile_level_id->profile == other_profile_level_id->profile; -} - -bool IsFormatSupported( - const std::vector& supported_formats, - const webrtc::SdpVideoFormat& format) { - for (const webrtc::SdpVideoFormat& supported_format : supported_formats) { - if (IsSameFormat(format, supported_format)) - return true; - } - return false; -} - class EncoderAdapter : public webrtc::VideoEncoderFactory { public: explicit EncoderAdapter( @@ -167,17 +138,11 @@ class DecoderAdapter : public webrtc::VideoDecoderFactory { public: explicit DecoderAdapter( std::unique_ptr external_decoder_factory) - : external_decoder_factory_(std::move(external_decoder_factory)) {} + : internal_decoder_factory_(new InternalDecoderFactory()), + external_decoder_factory_(std::move(external_decoder_factory)) {} std::unique_ptr CreateVideoDecoder( const webrtc::SdpVideoFormat& format) override { - std::unique_ptr internal_decoder; - webrtc::InternalDecoderFactory internal_decoder_factory; - if (IsFormatSupported(internal_decoder_factory.GetSupportedFormats(), - format)) { - internal_decoder = internal_decoder_factory.CreateVideoDecoder(format); - } - const VideoCodec codec(format); const VideoDecoderParams params = {}; if (external_decoder_factory_ != nullptr) { @@ -185,16 +150,16 @@ class DecoderAdapter : public webrtc::VideoDecoderFactory { CreateScopedVideoDecoder(external_decoder_factory_.get(), codec, params); if (external_decoder) { - if (!internal_decoder) - return external_decoder; - // Both external and internal decoder available - create fallback - // wrapper. - return std::unique_ptr( + webrtc::VideoCodecType type = + webrtc::PayloadStringToCodecType(codec.name); + std::unique_ptr internal_decoder( new webrtc::VideoDecoderSoftwareFallbackWrapper( - std::move(internal_decoder), std::move(external_decoder))); + type, std::move(external_decoder))); + return internal_decoder; } } - + std::unique_ptr internal_decoder( + internal_decoder_factory_->CreateVideoDecoderWithParams(codec, params)); return internal_decoder; } @@ -205,6 +170,7 @@ class DecoderAdapter : public webrtc::VideoDecoderFactory { } private: + const std::unique_ptr internal_decoder_factory_; const std::unique_ptr external_decoder_factory_; }; diff --git a/media/engine/internaldecoderfactory.cc b/media/engine/internaldecoderfactory.cc index 5e7aac18e5..5e69602a5d 100644 --- a/media/engine/internaldecoderfactory.cc +++ b/media/engine/internaldecoderfactory.cc @@ -10,41 +10,81 @@ #include "media/engine/internaldecoderfactory.h" -#include "media/base/mediaconstants.h" +#include + #include "modules/video_coding/codecs/h264/include/h264.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" -#include "rtc_base/checks.h" #include "rtc_base/logging.h" -namespace webrtc { +namespace cricket { -std::vector InternalDecoderFactory::GetSupportedFormats() - const { - std::vector formats; - formats.push_back(SdpVideoFormat(cricket::kVp8CodecName)); - if (VP9Decoder::IsSupported()) - formats.push_back(SdpVideoFormat(cricket::kVp9CodecName)); - for (const SdpVideoFormat& h264_format : SupportedH264Codecs()) - formats.push_back(h264_format); - return formats; -} +namespace { -std::unique_ptr InternalDecoderFactory::CreateVideoDecoder( - const SdpVideoFormat& format) { - if (cricket::CodecNamesEq(format.name, cricket::kVp8CodecName)) - return std::unique_ptr(VP8Decoder::Create()); - - if (cricket::CodecNamesEq(format.name, cricket::kVp9CodecName)) { - RTC_DCHECK(VP9Decoder::IsSupported()); - return std::unique_ptr(VP9Decoder::Create()); +// Video decoder class to be used for unknown codecs. Doesn't support decoding +// but logs messages to LS_ERROR. +class NullVideoDecoder : public webrtc::VideoDecoder { + public: + int32_t InitDecode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores) override { + LOG(LS_ERROR) << "Can't initialize NullVideoDecoder."; + return WEBRTC_VIDEO_CODEC_OK; } - if (cricket::CodecNamesEq(format.name, cricket::kH264CodecName)) - return std::unique_ptr(H264Decoder::Create()); + int32_t Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + const webrtc::RTPFragmentationHeader* fragmentation, + const webrtc::CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override { + LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding."; + return WEBRTC_VIDEO_CODEC_OK; + } - LOG(LS_ERROR) << "Trying to create decoder for unsupported format"; - return nullptr; + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) override { + LOG(LS_ERROR) + << "Can't register decode complete callback on NullVideoDecoder."; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } + + const char* ImplementationName() const override { return "NullVideoDecoder"; } +}; + +} // anonymous namespace + +InternalDecoderFactory::InternalDecoderFactory() {} + +InternalDecoderFactory::~InternalDecoderFactory() {} + +// WebRtcVideoDecoderFactory implementation. +webrtc::VideoDecoder* InternalDecoderFactory::CreateVideoDecoder( + webrtc::VideoCodecType type) { + switch (type) { + case webrtc::kVideoCodecH264: + if (webrtc::H264Decoder::IsSupported()) + return webrtc::H264Decoder::Create(); + // This could happen in a software-fallback for a codec type only + // supported externally (e.g. H.264 on iOS or Android) or in current usage + // in WebRtcVideoEngine if the external decoder fails to be created. + LOG(LS_ERROR) << "Unable to create an H.264 decoder fallback. " + << "Decoding of this stream will be broken."; + return new NullVideoDecoder(); + case webrtc::kVideoCodecVP8: + return webrtc::VP8Decoder::Create(); + case webrtc::kVideoCodecVP9: + RTC_DCHECK(webrtc::VP9Decoder::IsSupported()); + return webrtc::VP9Decoder::Create(); + default: + LOG(LS_ERROR) << "Creating NullVideoDecoder for unsupported codec."; + return new NullVideoDecoder(); + } } -} // namespace webrtc +void InternalDecoderFactory::DestroyVideoDecoder( + webrtc::VideoDecoder* decoder) { + delete decoder; +} + +} // namespace cricket diff --git a/media/engine/internaldecoderfactory.h b/media/engine/internaldecoderfactory.h index 7420129600..283d103f7b 100644 --- a/media/engine/internaldecoderfactory.h +++ b/media/engine/internaldecoderfactory.h @@ -11,20 +11,24 @@ #ifndef MEDIA_ENGINE_INTERNALDECODERFACTORY_H_ #define MEDIA_ENGINE_INTERNALDECODERFACTORY_H_ -#include #include -#include "api/video_codecs/video_decoder_factory.h" +#include "media/engine/webrtcvideodecoderfactory.h" -namespace webrtc { +namespace cricket { -class InternalDecoderFactory : public VideoDecoderFactory { +class InternalDecoderFactory : public WebRtcVideoDecoderFactory { public: - std::vector GetSupportedFormats() const override; - std::unique_ptr CreateVideoDecoder( - const SdpVideoFormat& format) override; + InternalDecoderFactory(); + virtual ~InternalDecoderFactory(); + + // WebRtcVideoDecoderFactory implementation. + webrtc::VideoDecoder* CreateVideoDecoder( + webrtc::VideoCodecType type) override; + + void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override; }; -} // namespace webrtc +} // namespace cricket #endif // MEDIA_ENGINE_INTERNALDECODERFACTORY_H_ diff --git a/media/engine/internaldecoderfactory_unittest.cc b/media/engine/internaldecoderfactory_unittest.cc index 147293dcf7..cbf39dfd03 100644 --- a/media/engine/internaldecoderfactory_unittest.cc +++ b/media/engine/internaldecoderfactory_unittest.cc @@ -10,18 +10,12 @@ #include "media/engine/internaldecoderfactory.h" -#include "api/video_codecs/sdp_video_format.h" -#include "api/video_codecs/video_decoder.h" -#include "media/base/mediaconstants.h" #include "test/gtest.h" -namespace webrtc { - TEST(InternalDecoderFactory, TestVP8) { - InternalDecoderFactory factory; - std::unique_ptr decoder = - factory.CreateVideoDecoder(SdpVideoFormat(cricket::kVp8CodecName)); + cricket::InternalDecoderFactory factory; + webrtc::VideoDecoder* decoder = + factory.CreateVideoDecoder(webrtc::kVideoCodecVP8); EXPECT_TRUE(decoder); + factory.DestroyVideoDecoder(decoder); } - -} // namespace webrtc diff --git a/media/engine/videodecodersoftwarefallbackwrapper.cc b/media/engine/videodecodersoftwarefallbackwrapper.cc index 5b1601659a..9d6274d1a5 100644 --- a/media/engine/videodecodersoftwarefallbackwrapper.cc +++ b/media/engine/videodecodersoftwarefallbackwrapper.cc @@ -22,49 +22,52 @@ namespace webrtc { VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper( - std::unique_ptr sw_fallback_decoder, - std::unique_ptr hw_decoder) - : use_hw_decoder_(true), - hw_decoder_(std::move(hw_decoder)), - hw_decoder_initialized_(false), - fallback_decoder_(std::move(sw_fallback_decoder)), - fallback_implementation_name_( - std::string(fallback_decoder_->ImplementationName()) + - " (fallback from: " + hw_decoder_->ImplementationName() + ")"), + VideoCodecType codec_type, + std::unique_ptr decoder) + : codec_type_(codec_type), + decoder_(std::move(decoder)), + decoder_initialized_(false), callback_(nullptr) {} int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode( const VideoCodec* codec_settings, int32_t number_of_cores) { - // Always try to use the HW decoder in this state. - use_hw_decoder_ = true; + RTC_DCHECK(!fallback_decoder_) << "Fallback decoder should never be " + "initialized here, it should've been " + "released."; codec_settings_ = *codec_settings; number_of_cores_ = number_of_cores; - int32_t ret = hw_decoder_->InitDecode(codec_settings, number_of_cores); + int32_t ret = decoder_->InitDecode(codec_settings, number_of_cores); if (ret == WEBRTC_VIDEO_CODEC_OK) { - hw_decoder_initialized_ = true; + decoder_initialized_ = true; return ret; } - hw_decoder_initialized_ = false; + decoder_initialized_ = false; // Try to initialize fallback decoder. if (InitFallbackDecoder()) return WEBRTC_VIDEO_CODEC_OK; - return ret; } bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() { + RTC_CHECK(codec_type_ != kVideoCodecUnknown) + << "Decoder requesting fallback to codec not supported in software."; LOG(LS_WARNING) << "Decoder falling back to software decoding."; + cricket::InternalDecoderFactory internal_decoder_factory; + fallback_decoder_.reset( + internal_decoder_factory.CreateVideoDecoder(codec_type_)); if (fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_) != WEBRTC_VIDEO_CODEC_OK) { LOG(LS_ERROR) << "Failed to initialize software-decoder fallback."; - use_hw_decoder_ = true; + fallback_decoder_.reset(); return false; } if (callback_) fallback_decoder_->RegisterDecodeCompleteCallback(callback_); - use_hw_decoder_ = false; + fallback_implementation_name_ = + std::string(fallback_decoder_->ImplementationName()) + + " (fallback from: " + decoder_->ImplementationName() + ")"; return true; } @@ -77,30 +80,31 @@ int32_t VideoDecoderSoftwareFallbackWrapper::Decode( TRACE_EVENT0("webrtc", "VideoDecoderSoftwareFallbackWrapper::Decode"); // Try initializing and decoding with the provided decoder on every keyframe // or when there's no fallback decoder. This is the normal case. - if (use_hw_decoder_ || input_image._frameType == kVideoFrameKey) { + if (!fallback_decoder_ || input_image._frameType == kVideoFrameKey) { int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; // Try reinitializing the decoder if it had failed before. - if (!hw_decoder_initialized_) { - hw_decoder_initialized_ = - hw_decoder_->InitDecode(&codec_settings_, number_of_cores_) == + if (!decoder_initialized_) { + decoder_initialized_ = + decoder_->InitDecode(&codec_settings_, number_of_cores_) == WEBRTC_VIDEO_CODEC_OK; } - if (hw_decoder_initialized_) { - ret = hw_decoder_->Decode(input_image, missing_frames, fragmentation, + if (decoder_initialized_) { + ret = decoder_->Decode(input_image, missing_frames, fragmentation, codec_specific_info, render_time_ms); } if (ret == WEBRTC_VIDEO_CODEC_OK) { - if (!use_hw_decoder_) { + if (fallback_decoder_) { // Decode OK -> stop using fallback decoder. LOG(LS_WARNING) << "Decode OK, no longer using the software fallback decoder."; - use_hw_decoder_ = true; + fallback_decoder_->Release(); + fallback_decoder_.reset(); return WEBRTC_VIDEO_CODEC_OK; } } if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) return ret; - if (use_hw_decoder_) { + if (!fallback_decoder_) { // Try to initialize fallback decoder. if (!InitFallbackDecoder()) return ret; @@ -113,29 +117,32 @@ int32_t VideoDecoderSoftwareFallbackWrapper::Decode( int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback( DecodedImageCallback* callback) { callback_ = callback; - int32_t ret = hw_decoder_->RegisterDecodeCompleteCallback(callback); - if (!use_hw_decoder_) + int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback); + if (fallback_decoder_) return fallback_decoder_->RegisterDecodeCompleteCallback(callback); return ret; } int32_t VideoDecoderSoftwareFallbackWrapper::Release() { - if (!use_hw_decoder_) { + if (fallback_decoder_) { LOG(LS_INFO) << "Releasing software fallback decoder."; fallback_decoder_->Release(); + fallback_decoder_.reset(); } - hw_decoder_initialized_ = false; - return hw_decoder_->Release(); + decoder_initialized_ = false; + return decoder_->Release(); } bool VideoDecoderSoftwareFallbackWrapper::PrefersLateDecoding() const { - return use_hw_decoder_ ? hw_decoder_->PrefersLateDecoding() - : fallback_decoder_->PrefersLateDecoding(); + if (fallback_decoder_) + return fallback_decoder_->PrefersLateDecoding(); + return decoder_->PrefersLateDecoding(); } const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const { - return use_hw_decoder_ ? hw_decoder_->ImplementationName() - : fallback_implementation_name_.c_str(); + if (fallback_decoder_) + return fallback_implementation_name_.c_str(); + return decoder_->ImplementationName(); } } // namespace webrtc diff --git a/media/engine/videodecodersoftwarefallbackwrapper.h b/media/engine/videodecodersoftwarefallbackwrapper.h index 97953ddc82..84a86ca59c 100644 --- a/media/engine/videodecodersoftwarefallbackwrapper.h +++ b/media/engine/videodecodersoftwarefallbackwrapper.h @@ -23,9 +23,8 @@ namespace webrtc { // hardware restrictions, such as max resolution. class VideoDecoderSoftwareFallbackWrapper : public VideoDecoder { public: - VideoDecoderSoftwareFallbackWrapper( - std::unique_ptr sw_fallback_decoder, - std::unique_ptr hw_decoder); + VideoDecoderSoftwareFallbackWrapper(VideoCodecType codec_type, + std::unique_ptr decoder); int32_t InitDecode(const VideoCodec* codec_settings, int32_t number_of_cores) override; @@ -47,15 +46,14 @@ class VideoDecoderSoftwareFallbackWrapper : public VideoDecoder { private: bool InitFallbackDecoder(); - // Determines if we are trying to use the HW or SW decoder. - bool use_hw_decoder_; - std::unique_ptr hw_decoder_; - bool hw_decoder_initialized_; + const VideoCodecType codec_type_; + std::unique_ptr decoder_; + bool decoder_initialized_; VideoCodec codec_settings_; int32_t number_of_cores_; - const std::unique_ptr fallback_decoder_; - const std::string fallback_implementation_name_; + std::string fallback_implementation_name_; + std::unique_ptr fallback_decoder_; DecodedImageCallback* callback_; }; diff --git a/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc b/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc index 367527eff7..dfd388be13 100644 --- a/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc +++ b/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc @@ -10,7 +10,6 @@ #include "media/engine/videodecodersoftwarefallbackwrapper.h" #include "api/video_codecs/video_decoder.h" -#include "modules/video_coding/codecs/vp8/include/vp8.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/checks.h" #include "test/gtest.h" @@ -21,7 +20,7 @@ class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test { protected: VideoDecoderSoftwareFallbackWrapperTest() : fake_decoder_(new CountingFakeDecoder()), - fallback_wrapper_(std::unique_ptr(VP8Decoder::Create()), + fallback_wrapper_(kVideoCodecVP8, std::unique_ptr(fake_decoder_)) {} class CountingFakeDecoder : public VideoDecoder { diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 33bb4c703d..84a77b7185 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -80,23 +80,6 @@ bool RunEncodeInRealTime(const TestConfig& config) { #endif } -// An internal decoder factory in the old WebRtcVideoDecoderFactory format. -// TODO(magjed): Update these tests to use new webrtc::VideoDecoderFactory -// instead. -class LegacyInternalDecoderFactory : public cricket::WebRtcVideoDecoderFactory { - public: - // WebRtcVideoDecoderFactory implementation. - VideoDecoder* CreateVideoDecoderWithParams( - const cricket::VideoCodec& codec, - cricket::VideoDecoderParams params) override { - return InternalDecoderFactory() - .CreateVideoDecoder(SdpVideoFormat(codec.name, codec.params)) - .release(); - } - - void DestroyVideoDecoder(VideoDecoder* decoder) override { delete decoder; } -}; - } // namespace void VideoProcessorIntegrationTest::H264KeyframeChecker::CheckEncodedFrame( @@ -328,7 +311,7 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { RTC_NOTREACHED() << "Only support HW decoder on Android and iOS."; #endif } else { - decoder_factory.reset(new LegacyInternalDecoderFactory()); + decoder_factory.reset(new cricket::InternalDecoderFactory()); } cricket::VideoCodec codec; @@ -386,9 +369,7 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { } if (config_.sw_fallback_decoder) { decoder_ = rtc::MakeUnique( - InternalDecoderFactory().CreateVideoDecoder( - SdpVideoFormat(codec.name, codec.params)), - std::move(decoder_)); + config_.codec_settings.codecType, std::move(decoder_)); } EXPECT_TRUE(encoder_) << "Encoder not successfully created.";