admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make decoder software fallback sticky.

Browse Source 
In practice, we never want to go back to HW decoding after deciding to
start decoding with SW codecs. This allows simplifying the code in the
fallback wrapper and makes it easier to implement HW codecs.

HW decoder is also now released when software fallback is activated
because it will not be used again. This could free up some resources.

Bug: b/73498933
Change-Id: Ibea4e32fce0c605179b649c8ac2744031799f3ee
Reviewed-on: https://webrtc-review.googlesource.com/55263
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Alex Glaznev <glaznev@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22168}
This commit is contained in:
Sami Kalliomäki 2018-02-23 09:05:49 +01:00 committed by Commit Bot
parent a2e3ab1fca
commit 99f52f83e4
4 changed files with 128 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
media/BUILD.gn
View File

@ -244,6 +244,7 @@ rtc_static_library("rtc_software_fallback_wrappers") {
"../modules/video_coding:video_codec_interface",
"../rtc_base:checks",
"../rtc_base:rtc_base_approved",
"../rtc_base/system:fallthrough",
"../system_wrappers:field_trial_api",
]
}

148
media/engine/videodecodersoftwarefallbackwrapper.cc
View File

@ -16,6 +16,7 @@
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/system/fallthrough.h"
#include "rtc_base/trace_event.h"
namespace webrtc {
@ -23,9 +24,8 @@ namespace webrtc {
VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
std::unique_ptr<VideoDecoder> sw_fallback_decoder,
std::unique_ptr<VideoDecoder> hw_decoder)
: use_hw_decoder_(true),
: decoder_type_(DecoderType::kNone),
hw_decoder_(std::move(hw_decoder)),
hw_decoder_initialized_(false),
fallback_decoder_(std::move(sw_fallback_decoder)),
fallback_implementation_name_(
std::string(fallback_decoder_->ImplementationName()) +
@ -35,35 +35,53 @@ VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode(
const VideoCodec* codec_settings,
int32_t number_of_cores) {
// Always try to use the HW decoder in this state.
use_hw_decoder_ = true;
codec_settings_ = *codec_settings;
number_of_cores_ = number_of_cores;
int32_t ret = hw_decoder_->InitDecode(codec_settings, number_of_cores);
if (ret == WEBRTC_VIDEO_CODEC_OK) {
hw_decoder_initialized_ = true;
return ret;
}
hw_decoder_initialized_ = false;
// Try to initialize fallback decoder.
if (InitFallbackDecoder())
int32_t status = InitHwDecoder();
if (status == WEBRTC_VIDEO_CODEC_OK) {
return WEBRTC_VIDEO_CODEC_OK;
}
return ret;
RTC_DCHECK(decoder_type_ == DecoderType::kNone);
if (InitFallbackDecoder()) {
return WEBRTC_VIDEO_CODEC_OK;
}
return status;
}
int32_t VideoDecoderSoftwareFallbackWrapper::InitHwDecoder() {
RTC_DCHECK(decoder_type_ == DecoderType::kNone);
int32_t status = hw_decoder_->InitDecode(&codec_settings_, number_of_cores_);
if (status != WEBRTC_VIDEO_CODEC_OK) {
return status;
}
decoder_type_ = DecoderType::kHardware;
if (callback_)
hw_decoder_->RegisterDecodeCompleteCallback(callback_);
return status;
}
bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() {
RTC_DCHECK(decoder_type_ == DecoderType::kNone ||
decoder_type_ == DecoderType::kHardware);
RTC_LOG(LS_WARNING) << "Decoder falling back to software decoding.";
if (fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_) !=
WEBRTC_VIDEO_CODEC_OK) {
int32_t status =
fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_);
if (status != WEBRTC_VIDEO_CODEC_OK) {
RTC_LOG(LS_ERROR) << "Failed to initialize software-decoder fallback.";
use_hw_decoder_ = true;
return false;
}
if (decoder_type_ == DecoderType::kHardware) {
hw_decoder_->Release();
}
decoder_type_ = DecoderType::kFallback;
if (callback_)
fallback_decoder_->RegisterDecodeCompleteCallback(callback_);
use_hw_decoder_ = false;
return true;
}
@ -73,68 +91,78 @@ int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
const RTPFragmentationHeader* fragmentation,
const CodecSpecificInfo* codec_specific_info,
int64_t render_time_ms) {
TRACE_EVENT0("webrtc", "VideoDecoderSoftwareFallbackWrapper::Decode");
// Try initializing and decoding with the provided decoder on every keyframe
// or when there's no fallback decoder. This is the normal case.
if (use_hw_decoder_ || input_image._frameType == kVideoFrameKey) {
int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
// Try reinitializing the decoder if it had failed before.
if (!hw_decoder_initialized_) {
hw_decoder_initialized_ =
hw_decoder_->InitDecode(&codec_settings_, number_of_cores_) ==
WEBRTC_VIDEO_CODEC_OK;
}
if (hw_decoder_initialized_) {
TRACE_EVENT0("webrtc", "VideoDecoderSoftwareFallbackWrapper::Decode");
switch (decoder_type_) {
case DecoderType::kNone:
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
case DecoderType::kHardware: {
int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
ret = hw_decoder_->Decode(input_image, missing_frames, fragmentation,
codec_specific_info, render_time_ms);
}
if (ret == WEBRTC_VIDEO_CODEC_OK) {
if (!use_hw_decoder_) {
// Decode OK -> stop using fallback decoder.
RTC_LOG(LS_WARNING)
<< "Decode OK, no longer using the software fallback decoder.";
use_hw_decoder_ = true;
return WEBRTC_VIDEO_CODEC_OK;
}
}
if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)
return ret;
if (use_hw_decoder_) {
// Try to initialize fallback decoder.
if (!InitFallbackDecoder())
codec_specific_info, render_time_ms);
if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) {
return ret;
}
// HW decoder returned WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE or
// initialization failed, fallback to software.
if (!InitFallbackDecoder()) {
return ret;
}
// Fallback decoder initialized, fall-through.
RTC_FALLTHROUGH();
}
case DecoderType::kFallback:
return fallback_decoder_->Decode(input_image, missing_frames,
fragmentation, codec_specific_info,
render_time_ms);
default:
RTC_NOTREACHED();
return WEBRTC_VIDEO_CODEC_ERROR;
}
return fallback_decoder_->Decode(input_image, missing_frames, fragmentation,
codec_specific_info, render_time_ms);
}
int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
callback_ = callback;
int32_t ret = hw_decoder_->RegisterDecodeCompleteCallback(callback);
if (!use_hw_decoder_)
return fallback_decoder_->RegisterDecodeCompleteCallback(callback);
return ret;
return active_decoder().RegisterDecodeCompleteCallback(callback);
}
int32_t VideoDecoderSoftwareFallbackWrapper::Release() {
if (!use_hw_decoder_) {
RTC_LOG(LS_INFO) << "Releasing software fallback decoder.";
fallback_decoder_->Release();
int32_t status;
switch (decoder_type_) {
case DecoderType::kHardware:
status = hw_decoder_->Release();
break;
case DecoderType::kFallback:
RTC_LOG(LS_INFO) << "Releasing software fallback decoder.";
status = fallback_decoder_->Release();
break;
case DecoderType::kNone:
status = WEBRTC_VIDEO_CODEC_OK;
break;
default:
RTC_NOTREACHED();
status = WEBRTC_VIDEO_CODEC_ERROR;
}
hw_decoder_initialized_ = false;
return hw_decoder_->Release();
decoder_type_ = DecoderType::kNone;
return status;
}
bool VideoDecoderSoftwareFallbackWrapper::PrefersLateDecoding() const {
return use_hw_decoder_ ? hw_decoder_->PrefersLateDecoding()
: fallback_decoder_->PrefersLateDecoding();
return active_decoder().PrefersLateDecoding();
}
const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const {
return use_hw_decoder_ ? hw_decoder_->ImplementationName()
: fallback_implementation_name_.c_str();
return decoder_type_ == DecoderType::kFallback
? fallback_implementation_name_.c_str()
: hw_decoder_->ImplementationName();
}
VideoDecoder& VideoDecoderSoftwareFallbackWrapper::active_decoder() const {
return decoder_type_ == DecoderType::kFallback ? *fallback_decoder_
: *hw_decoder_;
}
} // namespace webrtc

10
media/engine/videodecodersoftwarefallbackwrapper.h
View File

@ -46,11 +46,17 @@ class VideoDecoderSoftwareFallbackWrapper : public VideoDecoder {
private:
bool InitFallbackDecoder();
int32_t InitHwDecoder();
VideoDecoder& active_decoder() const;
// Determines if we are trying to use the HW or SW decoder.
bool use_hw_decoder_;
enum class DecoderType {
kNone,
kHardware,
kFallback,
} decoder_type_;
std::unique_ptr<VideoDecoder> hw_decoder_;
bool hw_decoder_initialized_;
VideoCodec codec_settings_;
int32_t number_of_cores_;

59
media/engine/videodecodersoftwarefallbackwrapper_unittest.cc
View File

@ -93,8 +93,8 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
EncodedImage encoded_image;
encoded_image._frameType = kVideoFrameKey;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
EXPECT_EQ(2, fake_decoder_->init_decode_count_)
<< "Should have attempted reinitializing the fallback decoder on "
EXPECT_EQ(1, fake_decoder_->init_decode_count_)
<< "Should not have attempted reinitializing the fallback decoder on "
"keyframe.";
// Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW
// decoder.
@ -102,33 +102,23 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
<< "Decoder used even though no InitDecode had succeeded.";
}
TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
CanRecoverFromSoftwareFallback) {
TEST_F(VideoDecoderSoftwareFallbackWrapperTest, IsSoftwareFallbackSticky) {
VideoCodec codec = {};
fallback_wrapper_.InitDecode(&codec, 2);
// Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW
// decoder.
fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
EncodedImage encoded_image;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
EXPECT_EQ(1, fake_decoder_->decode_count_);
// Fail -> fake_decoder shouldn't be used anymore.
// Software fallback should be sticky, fake_decoder_ shouldn't be used.
encoded_image._frameType = kVideoFrameKey;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
EXPECT_EQ(1, fake_decoder_->decode_count_)
<< "Decoder used even though fallback should be active.";
<< "Decoder shouldn't be used after failure.";
// Should be able to recover on a keyframe.
encoded_image._frameType = kVideoFrameKey;
fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
EXPECT_EQ(2, fake_decoder_->decode_count_)
<< "Wrapper did not try to decode a keyframe using registered decoder.";
encoded_image._frameType = kVideoFrameDelta;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
EXPECT_EQ(3, fake_decoder_->decode_count_)
<< "Decoder not used on future delta frames.";
// fake_decoder_ should have only been initialized once during the test.
EXPECT_EQ(1, fake_decoder_->init_decode_count_);
}
TEST_F(VideoDecoderSoftwareFallbackWrapperTest, DoesNotFallbackOnEveryError) {
@ -146,17 +136,36 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest, DoesNotFallbackOnEveryError) {
<< "Decoder should be active even though previous decode failed.";
}
TEST_F(VideoDecoderSoftwareFallbackWrapperTest, UsesHwDecoderAfterReinit) {
VideoCodec codec = {};
fallback_wrapper_.InitDecode(&codec, 2);
fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
EncodedImage encoded_image;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
EXPECT_EQ(1, fake_decoder_->decode_count_);
fallback_wrapper_.Release();
fallback_wrapper_.InitDecode(&codec, 2);
fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
EXPECT_EQ(2, fake_decoder_->decode_count_)
<< "Should not be using fallback after reinit.";
}
TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsReleaseCall) {
VideoCodec codec = {};
fallback_wrapper_.InitDecode(&codec, 2);
fallback_wrapper_.Release();
EXPECT_EQ(1, fake_decoder_->release_count_);
fallback_wrapper_.InitDecode(&codec, 2);
fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
EncodedImage encoded_image;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
EXPECT_EQ(1, fake_decoder_->release_count_)
<< "Decoder should not be released during fallback.";
EXPECT_EQ(2, fake_decoder_->release_count_)
<< "Decoder should be released during fallback.";
fallback_wrapper_.Release();
EXPECT_EQ(2, fake_decoder_->release_count_);
}
@ -177,18 +186,12 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
rtc::Optional<uint8_t> qp) override {
RTC_NOTREACHED();
}
} callback, callback2;
} callback;
VideoCodec codec = {};
fallback_wrapper_.InitDecode(&codec, 2);
fallback_wrapper_.RegisterDecodeCompleteCallback(&callback);
EXPECT_EQ(&callback, fake_decoder_->decode_complete_callback_);
fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
EncodedImage encoded_image;
fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
fallback_wrapper_.RegisterDecodeCompleteCallback(&callback2);
EXPECT_EQ(&callback2, fake_decoder_->decode_complete_callback_);
}
TEST_F(VideoDecoderSoftwareFallbackWrapperTest,