admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update internal video decoder factory to new interface

Browse Source 
We want to move away from cricket::WebRtcVideoDecoderFactory and this CL
updates the internal factory. Also, VideoDecoderSoftwareFallbackWrapper
is updated to take a VideoDecoder as argument instead of a factory so it
can be used with external SW decoders.

Bug: webrtc:7925
Change-Id: Ie6dc6c24f8610a2129620c6e2b42e3cebb2ddef7
Reviewed-on: https://webrtc-review.googlesource.com/7301
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Reviewed-by: Anders Carlsson <andersc@webrtc.org>
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20597}
This commit is contained in:
Magnus Jedvert 2017-11-06 17:41:54 +01:00 committed by Commit Bot
parent 2416f894f5
commit b2fc9b1b10
8 changed files with 157 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
media/engine/convert_legacy_video_factory.cc
View File

@ -15,6 +15,7 @@
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "media/base/h264_profile_level_id.h"
#include "media/engine/internaldecoderfactory.h"
#include "media/engine/internalencoderfactory.h"
#include "media/engine/scopedvideodecoder.h"
@ -31,6 +32,34 @@ namespace cricket {
namespace {
bool IsSameFormat(const webrtc::SdpVideoFormat& format1,
const webrtc::SdpVideoFormat& format2) {
// If different names (case insensitive), then not same formats.
if (!CodecNamesEq(format1.name, format2.name))
return false;
// For every format besides H264, comparing names is enough.
if (!CodecNamesEq(format1.name.c_str(), kH264CodecName))
return true;
// Compare H264 profiles.
const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id =
webrtc::H264::ParseSdpProfileLevelId(format1.parameters);
const rtc::Optional<webrtc::H264::ProfileLevelId> other_profile_level_id =
webrtc::H264::ParseSdpProfileLevelId(format2.parameters);
// Compare H264 profiles, but not levels.
return profile_level_id && other_profile_level_id &&
profile_level_id->profile == other_profile_level_id->profile;
}
bool IsFormatSupported(
const std::vector<webrtc::SdpVideoFormat>& supported_formats,
const webrtc::SdpVideoFormat& format) {
for (const webrtc::SdpVideoFormat& supported_format : supported_formats) {
if (IsSameFormat(format, supported_format))
return true;
}
return false;
}
class EncoderAdapter : public webrtc::VideoEncoderFactory {
public:
explicit EncoderAdapter(
@ -138,11 +167,17 @@ class DecoderAdapter : public webrtc::VideoDecoderFactory {
public:
explicit DecoderAdapter(
std::unique_ptr<WebRtcVideoDecoderFactory> external_decoder_factory)
: internal_decoder_factory_(new InternalDecoderFactory()),
external_decoder_factory_(std::move(external_decoder_factory)) {}
: external_decoder_factory_(std::move(external_decoder_factory)) {}
std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(
const webrtc::SdpVideoFormat& format) override {
std::unique_ptr<webrtc::VideoDecoder> internal_decoder;
webrtc::InternalDecoderFactory internal_decoder_factory;
if (IsFormatSupported(internal_decoder_factory.GetSupportedFormats(),
format)) {
internal_decoder = internal_decoder_factory.CreateVideoDecoder(format);
}
const VideoCodec codec(format);
const VideoDecoderParams params = {};
if (external_decoder_factory_ != nullptr) {
@ -150,16 +185,16 @@ class DecoderAdapter : public webrtc::VideoDecoderFactory {
CreateScopedVideoDecoder(external_decoder_factory_.get(), codec,
params);
if (external_decoder) {
webrtc::VideoCodecType type =
webrtc::PayloadStringToCodecType(codec.name);
std::unique_ptr<webrtc::VideoDecoder> internal_decoder(
if (!internal_decoder)
return external_decoder;
// Both external and internal decoder available - create fallback
// wrapper.
return std::unique_ptr<webrtc::VideoDecoder>(
new webrtc::VideoDecoderSoftwareFallbackWrapper(
type, std::move(external_decoder)));
return internal_decoder;
std::move(internal_decoder), std::move(external_decoder)));
}
}
std::unique_ptr<webrtc::VideoDecoder> internal_decoder(
internal_decoder_factory_->CreateVideoDecoderWithParams(codec, params));
return internal_decoder;
}
@ -170,7 +205,6 @@ class DecoderAdapter : public webrtc::VideoDecoderFactory {
}
private:
const std::unique_ptr<WebRtcVideoDecoderFactory> internal_decoder_factory_;
const std::unique_ptr<WebRtcVideoDecoderFactory> external_decoder_factory_;
};

96
media/engine/internaldecoderfactory.cc
View File

@ -10,81 +10,41 @@
#include "media/engine/internaldecoderfactory.h"
#include <utility>
#include "media/base/mediaconstants.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace cricket {
namespace webrtc {
namespace {
// Video decoder class to be used for unknown codecs. Doesn't support decoding
// but logs messages to LS_ERROR.
class NullVideoDecoder : public webrtc::VideoDecoder {
public:
int32_t InitDecode(const webrtc::VideoCodec* codec_settings,
int32_t number_of_cores) override {
LOG(LS_ERROR) << "Can't initialize NullVideoDecoder.";
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t Decode(const webrtc::EncodedImage& input_image,
bool missing_frames,
const webrtc::RTPFragmentationHeader* fragmentation,
const webrtc::CodecSpecificInfo* codec_specific_info,
int64_t render_time_ms) override {
LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding.";
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t RegisterDecodeCompleteCallback(
webrtc::DecodedImageCallback* callback) override {
LOG(LS_ERROR)
<< "Can't register decode complete callback on NullVideoDecoder.";
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; }
const char* ImplementationName() const override { return "NullVideoDecoder"; }
};
} // anonymous namespace
InternalDecoderFactory::InternalDecoderFactory() {}
InternalDecoderFactory::~InternalDecoderFactory() {}
// WebRtcVideoDecoderFactory implementation.
webrtc::VideoDecoder* InternalDecoderFactory::CreateVideoDecoder(
webrtc::VideoCodecType type) {
switch (type) {
case webrtc::kVideoCodecH264:
if (webrtc::H264Decoder::IsSupported())
return webrtc::H264Decoder::Create();
// This could happen in a software-fallback for a codec type only
// supported externally (e.g. H.264 on iOS or Android) or in current usage
// in WebRtcVideoEngine if the external decoder fails to be created.
LOG(LS_ERROR) << "Unable to create an H.264 decoder fallback. "
<< "Decoding of this stream will be broken.";
return new NullVideoDecoder();
case webrtc::kVideoCodecVP8:
return webrtc::VP8Decoder::Create();
case webrtc::kVideoCodecVP9:
RTC_DCHECK(webrtc::VP9Decoder::IsSupported());
return webrtc::VP9Decoder::Create();
default:
LOG(LS_ERROR) << "Creating NullVideoDecoder for unsupported codec.";
return new NullVideoDecoder();
}
std::vector<SdpVideoFormat> InternalDecoderFactory::GetSupportedFormats()
const {
std::vector<SdpVideoFormat> formats;
formats.push_back(SdpVideoFormat(cricket::kVp8CodecName));
if (VP9Decoder::IsSupported())
formats.push_back(SdpVideoFormat(cricket::kVp9CodecName));
for (const SdpVideoFormat& h264_format : SupportedH264Codecs())
formats.push_back(h264_format);
return formats;
}
void InternalDecoderFactory::DestroyVideoDecoder(
webrtc::VideoDecoder* decoder) {
delete decoder;
std::unique_ptr<VideoDecoder> InternalDecoderFactory::CreateVideoDecoder(
const SdpVideoFormat& format) {
if (cricket::CodecNamesEq(format.name, cricket::kVp8CodecName))
return std::unique_ptr<VideoDecoder>(VP8Decoder::Create());
if (cricket::CodecNamesEq(format.name, cricket::kVp9CodecName)) {
RTC_DCHECK(VP9Decoder::IsSupported());
return std::unique_ptr<VideoDecoder>(VP9Decoder::Create());
}
if (cricket::CodecNamesEq(format.name, cricket::kH264CodecName))
return std::unique_ptr<VideoDecoder>(H264Decoder::Create());
LOG(LS_ERROR) << "Trying to create decoder for unsupported format";
return nullptr;
}
} // namespace cricket
} // namespace webrtc

20
media/engine/internaldecoderfactory.h
View File

@ -11,24 +11,20 @@
#ifndef MEDIA_ENGINE_INTERNALDECODERFACTORY_H_
#define MEDIA_ENGINE_INTERNALDECODERFACTORY_H_
#include <memory>
#include <vector>
#include "media/engine/webrtcvideodecoderfactory.h"
#include "api/video_codecs/video_decoder_factory.h"
namespace cricket {
namespace webrtc {
class InternalDecoderFactory : public WebRtcVideoDecoderFactory {
class InternalDecoderFactory : public VideoDecoderFactory {
public:
InternalDecoderFactory();
virtual ~InternalDecoderFactory();
// WebRtcVideoDecoderFactory implementation.
webrtc::VideoDecoder* CreateVideoDecoder(
webrtc::VideoCodecType type) override;
void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override;
std::vector<SdpVideoFormat> GetSupportedFormats() const override;
std::unique_ptr<VideoDecoder> CreateVideoDecoder(
const SdpVideoFormat& format) override;
};
} // namespace cricket
} // namespace webrtc
#endif // MEDIA_ENGINE_INTERNALDECODERFACTORY_H_

14
media/engine/internaldecoderfactory_unittest.cc
View File

@ -10,12 +10,18 @@
#include "media/engine/internaldecoderfactory.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "media/base/mediaconstants.h"
#include "test/gtest.h"
namespace webrtc {
TEST(InternalDecoderFactory, TestVP8) {
cricket::InternalDecoderFactory factory;
webrtc::VideoDecoder* decoder =
factory.CreateVideoDecoder(webrtc::kVideoCodecVP8);
InternalDecoderFactory factory;
std::unique_ptr<VideoDecoder> decoder =
factory.CreateVideoDecoder(SdpVideoFormat(cricket::kVp8CodecName));
EXPECT_TRUE(decoder);
factory.DestroyVideoDecoder(decoder);
}
} // namespace webrtc

77
media/engine/videodecodersoftwarefallbackwrapper.cc
View File

@ -22,52 +22,49 @@
namespace webrtc {
VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
VideoCodecType codec_type,
std::unique_ptr<VideoDecoder> decoder)
: codec_type_(codec_type),
decoder_(std::move(decoder)),
decoder_initialized_(false),
std::unique_ptr<VideoDecoder> sw_fallback_decoder,
std::unique_ptr<VideoDecoder> hw_decoder)
: use_hw_decoder_(true),
hw_decoder_(std::move(hw_decoder)),
hw_decoder_initialized_(false),
fallback_decoder_(std::move(sw_fallback_decoder)),
fallback_implementation_name_(
std::string(fallback_decoder_->ImplementationName()) +
" (fallback from: " + hw_decoder_->ImplementationName() + ")"),
callback_(nullptr) {}
int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode(
const VideoCodec* codec_settings,
int32_t number_of_cores) {
RTC_DCHECK(!fallback_decoder_) << "Fallback decoder should never be "
"initialized here, it should've been "
"released.";
// Always try to use the HW decoder in this state.
use_hw_decoder_ = true;
codec_settings_ = *codec_settings;
number_of_cores_ = number_of_cores;
int32_t ret = decoder_->InitDecode(codec_settings, number_of_cores);
int32_t ret = hw_decoder_->InitDecode(codec_settings, number_of_cores);
if (ret == WEBRTC_VIDEO_CODEC_OK) {
decoder_initialized_ = true;
hw_decoder_initialized_ = true;
return ret;
}
decoder_initialized_ = false;
hw_decoder_initialized_ = false;
// Try to initialize fallback decoder.
if (InitFallbackDecoder())
return WEBRTC_VIDEO_CODEC_OK;
return ret;
}
bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() {
RTC_CHECK(codec_type_ != kVideoCodecUnknown)
<< "Decoder requesting fallback to codec not supported in software.";
LOG(LS_WARNING) << "Decoder falling back to software decoding.";
cricket::InternalDecoderFactory internal_decoder_factory;
fallback_decoder_.reset(
internal_decoder_factory.CreateVideoDecoder(codec_type_));
if (fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_) !=
WEBRTC_VIDEO_CODEC_OK) {
LOG(LS_ERROR) << "Failed to initialize software-decoder fallback.";
fallback_decoder_.reset();
use_hw_decoder_ = true;
return false;
}
if (callback_)
fallback_decoder_->RegisterDecodeCompleteCallback(callback_);
fallback_implementation_name_ =
std::string(fallback_decoder_->ImplementationName()) +
" (fallback from: " + decoder_->ImplementationName() + ")";
use_hw_decoder_ = false;
return true;
}
@ -80,31 +77,30 @@ int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
TRACE_EVENT0("webrtc", "VideoDecoderSoftwareFallbackWrapper::Decode");
// Try initializing and decoding with the provided decoder on every keyframe
// or when there's no fallback decoder. This is the normal case.
if (!fallback_decoder_ || input_image._frameType == kVideoFrameKey) {
if (use_hw_decoder_ || input_image._frameType == kVideoFrameKey) {
int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
// Try reinitializing the decoder if it had failed before.
if (!decoder_initialized_) {
decoder_initialized_ =
decoder_->InitDecode(&codec_settings_, number_of_cores_) ==
if (!hw_decoder_initialized_) {
hw_decoder_initialized_ =
hw_decoder_->InitDecode(&codec_settings_, number_of_cores_) ==
WEBRTC_VIDEO_CODEC_OK;
}
if (decoder_initialized_) {
ret = decoder_->Decode(input_image, missing_frames, fragmentation,
if (hw_decoder_initialized_) {
ret = hw_decoder_->Decode(input_image, missing_frames, fragmentation,
codec_specific_info, render_time_ms);
}
if (ret == WEBRTC_VIDEO_CODEC_OK) {
if (fallback_decoder_) {
if (!use_hw_decoder_) {
// Decode OK -> stop using fallback decoder.
LOG(LS_WARNING)
<< "Decode OK, no longer using the software fallback decoder.";
fallback_decoder_->Release();
fallback_decoder_.reset();
use_hw_decoder_ = true;
return WEBRTC_VIDEO_CODEC_OK;
}
}
if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)
return ret;
if (!fallback_decoder_) {
if (use_hw_decoder_) {
// Try to initialize fallback decoder.
if (!InitFallbackDecoder())
return ret;
@ -117,32 +113,29 @@ int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
callback_ = callback;
int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
if (fallback_decoder_)
int32_t ret = hw_decoder_->RegisterDecodeCompleteCallback(callback);
if (!use_hw_decoder_)
return fallback_decoder_->RegisterDecodeCompleteCallback(callback);
return ret;
}
int32_t VideoDecoderSoftwareFallbackWrapper::Release() {
if (fallback_decoder_) {
if (!use_hw_decoder_) {
LOG(LS_INFO) << "Releasing software fallback decoder.";
fallback_decoder_->Release();
fallback_decoder_.reset();
}
decoder_initialized_ = false;
return decoder_->Release();
hw_decoder_initialized_ = false;
return hw_decoder_->Release();
}
bool VideoDecoderSoftwareFallbackWrapper::PrefersLateDecoding() const {
if (fallback_decoder_)
return fallback_decoder_->PrefersLateDecoding();
return decoder_->PrefersLateDecoding();
return use_hw_decoder_ ? hw_decoder_->PrefersLateDecoding()
: fallback_decoder_->PrefersLateDecoding();
}
const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const {
if (fallback_decoder_)
return fallback_implementation_name_.c_str();
return decoder_->ImplementationName();
return use_hw_decoder_ ? hw_decoder_->ImplementationName()
: fallback_implementation_name_.c_str();
}
} // namespace webrtc

16
media/engine/videodecodersoftwarefallbackwrapper.h
View File

@ -23,8 +23,9 @@ namespace webrtc {
// hardware restrictions, such as max resolution.
class VideoDecoderSoftwareFallbackWrapper : public VideoDecoder {
public:
VideoDecoderSoftwareFallbackWrapper(VideoCodecType codec_type,
std::unique_ptr<VideoDecoder> decoder);
VideoDecoderSoftwareFallbackWrapper(
std::unique_ptr<VideoDecoder> sw_fallback_decoder,
std::unique_ptr<VideoDecoder> hw_decoder);
int32_t InitDecode(const VideoCodec* codec_settings,
int32_t number_of_cores) override;
@ -46,14 +47,15 @@ class VideoDecoderSoftwareFallbackWrapper : public VideoDecoder {
private:
bool InitFallbackDecoder();
const VideoCodecType codec_type_;
std::unique_ptr<VideoDecoder> decoder_;
bool decoder_initialized_;
// Determines if we are trying to use the HW or SW decoder.
bool use_hw_decoder_;
std::unique_ptr<VideoDecoder> hw_decoder_;
bool hw_decoder_initialized_;
VideoCodec codec_settings_;
int32_t number_of_cores_;
std::string fallback_implementation_name_;
std::unique_ptr<VideoDecoder> fallback_decoder_;
const std::unique_ptr<VideoDecoder> fallback_decoder_;
const std::string fallback_implementation_name_;
DecodedImageCallback* callback_;
};

3
media/engine/videodecodersoftwarefallbackwrapper_unittest.cc
View File

@ -10,6 +10,7 @@
#include "media/engine/videodecodersoftwarefallbackwrapper.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "test/gtest.h"
@ -20,7 +21,7 @@ class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test {
protected:
VideoDecoderSoftwareFallbackWrapperTest()
: fake_decoder_(new CountingFakeDecoder()),
fallback_wrapper_(kVideoCodecVP8,
fallback_wrapper_(std::unique_ptr<VideoDecoder>(VP8Decoder::Create()),
std::unique_ptr<VideoDecoder>(fake_decoder_)) {}
class CountingFakeDecoder : public VideoDecoder {

23
modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
View File

@ -80,6 +80,23 @@ bool RunEncodeInRealTime(const TestConfig& config) {
#endif
}
// An internal decoder factory in the old WebRtcVideoDecoderFactory format.
// TODO(magjed): Update these tests to use new webrtc::VideoDecoderFactory
// instead.
class LegacyInternalDecoderFactory : public cricket::WebRtcVideoDecoderFactory {
public:
// WebRtcVideoDecoderFactory implementation.
VideoDecoder* CreateVideoDecoderWithParams(
const cricket::VideoCodec& codec,
cricket::VideoDecoderParams params) override {
return InternalDecoderFactory()
.CreateVideoDecoder(SdpVideoFormat(codec.name, codec.params))
.release();
}
void DestroyVideoDecoder(VideoDecoder* decoder) override { delete decoder; }
};
} // namespace
void VideoProcessorIntegrationTest::H264KeyframeChecker::CheckEncodedFrame(
@ -311,7 +328,7 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() {
RTC_NOTREACHED() << "Only support HW decoder on Android and iOS.";
#endif
} else {
decoder_factory.reset(new cricket::InternalDecoderFactory());
decoder_factory.reset(new LegacyInternalDecoderFactory());
}
cricket::VideoCodec codec;
@ -369,7 +386,9 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() {
}
if (config_.sw_fallback_decoder) {
decoder_ = rtc::MakeUnique<VideoDecoderSoftwareFallbackWrapper>(
config_.codec_settings.codecType, std::move(decoder_));
InternalDecoderFactory().CreateVideoDecoder(
SdpVideoFormat(codec.name, codec.params)),
std::move(decoder_));
}
EXPECT_TRUE(encoder_) << "Encoder not successfully created.";