admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Delay creation of decoders until they are needed

Browse Source 
Before this CL, WebRTC created a decoder for each negotiated codec
profile. This quickly consumed all available HW decoder resources
on some platforms. This CL adds a field trial,
WebRTC-PreStreamDecoders, that makes it possible to set how many
decoders that should be created up front, from 0 to ALL. If the
field trial is set to 1, we only create a decoder for the
preferred codec. The other decoders are only created when they are
needed (i.e., if we receive the corresponding payload type).

Bug: webrtc:12462
Change-Id: I087571b540f6796d32d34923f9c7f8e89b0959c5
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/208284
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33300}
This commit is contained in:
Johannes Kron 2021-02-18 23:37:22 +01:00 committed by Commit Bot
parent c9b9930c97
commit 16359f65c4
7 changed files with 211 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
modules/video_coding/decoder_database.cc
View File

@ -56,7 +56,6 @@ bool VCMDecoderDataBase::DeregisterExternalDecoder(uint8_t payload_type) {
// Release it if it was registered and in use.
ptr_decoder_.reset();
}
DeregisterReceiveCodec(payload_type);
delete it->second;
dec_external_map_.erase(it);
return true;
@ -73,6 +72,12 @@ void VCMDecoderDataBase::RegisterExternalDecoder(VideoDecoder* external_decoder,
dec_external_map_[payload_type] = ext_decoder;
}
bool VCMDecoderDataBase::IsExternalDecoderRegistered(
uint8_t payload_type) const {
return payload_type == current_payload_type_ ||
FindExternalDecoderItem(payload_type);
}
bool VCMDecoderDataBase::RegisterReceiveCodec(uint8_t payload_type,
const VideoCodec* receive_codec,
int number_of_cores) {

1
modules/video_coding/decoder_database.h
View File

@ -44,6 +44,7 @@ class VCMDecoderDataBase {
bool DeregisterExternalDecoder(uint8_t payload_type);
void RegisterExternalDecoder(VideoDecoder* external_decoder,
uint8_t payload_type);
bool IsExternalDecoderRegistered(uint8_t payload_type) const;
bool RegisterReceiveCodec(uint8_t payload_type,
const VideoCodec* receive_codec,

39
modules/video_coding/video_receiver2.cc
View File

@ -33,18 +33,18 @@ VideoReceiver2::VideoReceiver2(Clock* clock, VCMTiming* timing)
timing_(timing),
decodedFrameCallback_(timing_, clock_),
codecDataBase_() {
decoder_thread_checker_.Detach();
decoder_sequence_checker_.Detach();
}
VideoReceiver2::~VideoReceiver2() {
RTC_DCHECK_RUN_ON(&construction_thread_checker_);
RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
}
// Register a receive callback. Will be called whenever there is a new frame
// ready for rendering.
int32_t VideoReceiver2::RegisterReceiveCallback(
VCMReceiveCallback* receiveCallback) {
RTC_DCHECK_RUN_ON(&construction_thread_checker_);
RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
RTC_DCHECK(!IsDecoderThreadRunning());
// This value is set before the decoder thread starts and unset after
// the decoder thread has been stopped.
@ -52,20 +52,35 @@ int32_t VideoReceiver2::RegisterReceiveCallback(
return VCM_OK;
}
// Register an externally defined decoder object.
// Register an externally defined decoder object. This may be called on either
// the construction sequence or the decoder sequence to allow for lazy creation
// of video decoders. If called on the decoder sequence |externalDecoder| cannot
// be a nullptr. It's the responsibility of the caller to make sure that the
// access from the two sequences are mutually exclusive.
void VideoReceiver2::RegisterExternalDecoder(VideoDecoder* externalDecoder,
uint8_t payloadType) {
RTC_DCHECK_RUN_ON(&construction_thread_checker_);
RTC_DCHECK(!IsDecoderThreadRunning());
if (IsDecoderThreadRunning()) {
RTC_DCHECK_RUN_ON(&decoder_sequence_checker_);
// Don't allow deregistering decoders on the decoder thread.
RTC_DCHECK(externalDecoder != nullptr);
} else {
RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
}
if (externalDecoder == nullptr) {
RTC_CHECK(codecDataBase_.DeregisterExternalDecoder(payloadType));
codecDataBase_.DeregisterExternalDecoder(payloadType);
return;
}
codecDataBase_.RegisterExternalDecoder(externalDecoder, payloadType);
}
bool VideoReceiver2::IsExternalDecoderRegistered(uint8_t payloadType) const {
RTC_DCHECK_RUN_ON(&decoder_sequence_checker_);
return codecDataBase_.IsExternalDecoderRegistered(payloadType);
}
void VideoReceiver2::DecoderThreadStarting() {
RTC_DCHECK_RUN_ON(&construction_thread_checker_);
RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
RTC_DCHECK(!IsDecoderThreadRunning());
#if RTC_DCHECK_IS_ON
decoder_thread_is_running_ = true;
@ -73,17 +88,17 @@ void VideoReceiver2::DecoderThreadStarting() {
}
void VideoReceiver2::DecoderThreadStopped() {
RTC_DCHECK_RUN_ON(&construction_thread_checker_);
RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
RTC_DCHECK(IsDecoderThreadRunning());
#if RTC_DCHECK_IS_ON
decoder_thread_is_running_ = false;
decoder_thread_checker_.Detach();
decoder_sequence_checker_.Detach();
#endif
}
// Must be called from inside the receive side critical section.
int32_t VideoReceiver2::Decode(const VCMEncodedFrame* frame) {
RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
RTC_DCHECK_RUN_ON(&decoder_sequence_checker_);
TRACE_EVENT0("webrtc", "VideoReceiver2::Decode");
// Change decoder if payload type has changed
VCMGenericDecoder* decoder =
@ -98,7 +113,7 @@ int32_t VideoReceiver2::Decode(const VCMEncodedFrame* frame) {
int32_t VideoReceiver2::RegisterReceiveCodec(uint8_t payload_type,
const VideoCodec* receiveCodec,
int32_t numberOfCores) {
RTC_DCHECK_RUN_ON(&construction_thread_checker_);
RTC_DCHECK_RUN_ON(&construction_sequence_checker_);
RTC_DCHECK(!IsDecoderThreadRunning());
if (receiveCodec == nullptr) {
return VCM_PARAMETER_ERROR;

5
modules/video_coding/video_receiver2.h
View File

@ -36,6 +36,7 @@ class VideoReceiver2 {
void RegisterExternalDecoder(VideoDecoder* externalDecoder,
uint8_t payloadType);
bool IsExternalDecoderRegistered(uint8_t payloadType) const;
int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback);
int32_t Decode(const webrtc::VCMEncodedFrame* frame);
@ -54,8 +55,8 @@ class VideoReceiver2 {
// In builds where DCHECKs aren't enabled, it will return true.
bool IsDecoderThreadRunning();
SequenceChecker construction_thread_checker_;
SequenceChecker decoder_thread_checker_;
SequenceChecker construction_sequence_checker_;
SequenceChecker decoder_sequence_checker_;
Clock* const clock_;
VCMTiming* timing_;
VCMDecodedFrameCallback decodedFrameCallback_;

92
video/video_receive_stream2.cc
View File

@ -66,6 +66,8 @@ constexpr int kMaxBaseMinimumDelayMs = 10000;
constexpr int kMaxWaitForFrameMs = 3000;
constexpr int kDefaultMaximumPreStreamDecoders = 100;
// Concrete instance of RecordableEncodedFrame wrapping needed content
// from video_coding::EncodedFrame.
class WebRtcRecordableEncodedFrame : public RecordableEncodedFrame {
@ -234,6 +236,7 @@ VideoReceiveStream2::VideoReceiveStream2(
low_latency_renderer_enabled_("enabled", true),
low_latency_renderer_include_predecode_buffer_("include_predecode_buffer",
true),
maximum_pre_stream_decoders_("max", kDefaultMaximumPreStreamDecoders),
decode_queue_(task_queue_factory_->CreateTaskQueue(
"DecodingQueue",
TaskQueueFactory::Priority::HIGH)) {
@ -278,6 +281,11 @@ VideoReceiveStream2::VideoReceiveStream2(
ParseFieldTrial({&low_latency_renderer_enabled_,
&low_latency_renderer_include_predecode_buffer_},
field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
ParseFieldTrial(
{
&maximum_pre_stream_decoders_,
},
field_trial::FindFullName("WebRTC-PreStreamDecoders"));
}
VideoReceiveStream2::~VideoReceiveStream2() {
@ -325,41 +333,16 @@ void VideoReceiveStream2::Start() {
renderer = this;
}
int decoders_count = 0;
for (const Decoder& decoder : config_.decoders) {
std::unique_ptr<VideoDecoder> video_decoder =
config_.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format,
config_.stream_id);
// If we still have no valid decoder, we have to create a "Null" decoder
// that ignores all calls. The reason we can get into this state is that the
// old decoder factory interface doesn't have a way to query supported
// codecs.
if (!video_decoder) {
video_decoder = std::make_unique<NullVideoDecoder>();
// Create up to maximum_pre_stream_decoders_ up front, wait the the other
// decoders until they are requested (i.e., we receive the corresponding
// payload).
if (decoders_count < maximum_pre_stream_decoders_) {
CreateAndRegisterExternalDecoder(decoder);
++decoders_count;
}
std::string decoded_output_file =
field_trial::FindFullName("WebRTC-DecoderDataDumpDirectory");
// Because '/' can't be used inside a field trial parameter, we use ';'
// instead.
// This is only relevant to WebRTC-DecoderDataDumpDirectory
// field trial. ';' is chosen arbitrary. Even though it's a legal character
// in some file systems, we can sacrifice ability to use it in the path to
// dumped video, since it's developers-only feature for debugging.
absl::c_replace(decoded_output_file, ';', '/');
if (!decoded_output_file.empty()) {
char filename_buffer[256];
rtc::SimpleStringBuilder ssb(filename_buffer);
ssb << decoded_output_file << "/webrtc_receive_stream_"
<< this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros()
<< ".ivf";
video_decoder = CreateFrameDumpingDecoderWrapper(
std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str()));
}
video_decoders_.push_back(std::move(video_decoder));
video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(),
decoder.payload_type);
VideoCodec codec = CreateDecoderVideoCodec(decoder);
const bool raw_payload =
@ -429,6 +412,41 @@ void VideoReceiveStream2::Stop() {
transport_adapter_.Disable();
}
void VideoReceiveStream2::CreateAndRegisterExternalDecoder(
const Decoder& decoder) {
std::unique_ptr<VideoDecoder> video_decoder =
config_.decoder_factory->CreateVideoDecoder(decoder.video_format);
// If we still have no valid decoder, we have to create a "Null" decoder
// that ignores all calls. The reason we can get into this state is that the
// old decoder factory interface doesn't have a way to query supported
// codecs.
if (!video_decoder) {
video_decoder = std::make_unique<NullVideoDecoder>();
}
std::string decoded_output_file =
field_trial::FindFullName("WebRTC-DecoderDataDumpDirectory");
// Because '/' can't be used inside a field trial parameter, we use ';'
// instead.
// This is only relevant to WebRTC-DecoderDataDumpDirectory
// field trial. ';' is chosen arbitrary. Even though it's a legal character
// in some file systems, we can sacrifice ability to use it in the path to
// dumped video, since it's developers-only feature for debugging.
absl::c_replace(decoded_output_file, ';', '/');
if (!decoded_output_file.empty()) {
char filename_buffer[256];
rtc::SimpleStringBuilder ssb(filename_buffer);
ssb << decoded_output_file << "/webrtc_receive_stream_"
<< this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros() << ".ivf";
video_decoder = CreateFrameDumpingDecoderWrapper(
std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str()));
}
video_decoders_.push_back(std::move(video_decoder));
video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(),
decoder.payload_type);
}
VideoReceiveStream::Stats VideoReceiveStream2::GetStats() const {
RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
VideoReceiveStream2::Stats stats = stats_proxy_.GetStats();
@ -661,6 +679,16 @@ void VideoReceiveStream2::HandleEncodedFrame(
const bool keyframe_request_is_due =
now_ms >= (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_);
if (!video_receiver_.IsExternalDecoderRegistered(frame->PayloadType())) {
// Look for the decoder with this payload type.
for (const Decoder& decoder : config_.decoders) {
if (decoder.payload_type == frame->PayloadType()) {
CreateAndRegisterExternalDecoder(decoder);
break;
}
}
}
int decode_result = video_receiver_.Decode(frame.get());
if (decode_result == WEBRTC_VIDEO_CODEC_OK ||
decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) {

6
video/video_receive_stream2.h
View File

@ -155,6 +155,7 @@ class VideoReceiveStream2 : public webrtc::VideoReceiveStream,
void GenerateKeyFrame() override;
private:
void CreateAndRegisterExternalDecoder(const Decoder& decoder);
int64_t GetMaxWaitMs() const RTC_RUN_ON(decode_queue_);
void StartNextDecode() RTC_RUN_ON(decode_queue_);
void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame)
@ -265,6 +266,11 @@ class VideoReceiveStream2 : public webrtc::VideoReceiveStream,
// queue.
FieldTrialParameter<bool> low_latency_renderer_include_predecode_buffer_;
// Set by the field trial WebRTC-PreStreamDecoders. The parameter |max|
// determines the maximum number of decoders that are created up front before
// any video frame has been received.
FieldTrialParameter<int> maximum_pre_stream_decoders_;
// Defined last so they are destroyed before all other members.
rtc::TaskQueue decode_queue_;

108
video/video_receive_stream2_unittest.cc
View File

@ -76,6 +76,14 @@ class MockVideoDecoder : public VideoDecoder {
const char* ImplementationName() const { return "MockVideoDecoder"; }
};
class MockVideoDecoderFactory : public VideoDecoderFactory {
public:
MOCK_CONST_METHOD0(GetSupportedFormats, std::vector<SdpVideoFormat>());
MOCK_METHOD1(CreateVideoDecoder,
std::unique_ptr<VideoDecoder>(const SdpVideoFormat& format));
};
class FrameObjectFake : public video_coding::EncodedFrame {
public:
void SetPayloadType(uint8_t payload_type) { _payloadType = payload_type; }
@ -111,6 +119,7 @@ class VideoReceiveStream2Test : public ::testing::Test {
h264_decoder.video_format = SdpVideoFormat("H264");
h264_decoder.video_format.parameters.insert(
{"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
config_.decoders.clear();
config_.decoders.push_back(h264_decoder);
clock_ = Clock::GetRealTimeClock();
@ -593,4 +602,103 @@ TEST_F(VideoReceiveStream2TestWithSimulatedClock,
loop_.Run();
}
class VideoReceiveStream2TestWithLazyDecoderCreation : public ::testing::Test {
public:
VideoReceiveStream2TestWithLazyDecoderCreation()
: process_thread_(ProcessThread::Create("TestThread")),
task_queue_factory_(CreateDefaultTaskQueueFactory()),
config_(&mock_transport_),
call_stats_(Clock::GetRealTimeClock(), loop_.task_queue()) {}
void SetUp() {
webrtc::test::ScopedFieldTrials field_trials(
"WebRTC-PreStreamDecoders/max:0/");
constexpr int kDefaultNumCpuCores = 2;
config_.rtp.remote_ssrc = 1111;
config_.rtp.local_ssrc = 2222;
config_.renderer = &fake_renderer_;
config_.decoder_factory = &mock_h264_decoder_factory_;
VideoReceiveStream::Decoder h264_decoder;
h264_decoder.payload_type = 99;
h264_decoder.video_format = SdpVideoFormat("H264");
h264_decoder.video_format.parameters.insert(
{"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
config_.decoders.clear();
config_.decoders.push_back(h264_decoder);
clock_ = Clock::GetRealTimeClock();
timing_ = new VCMTiming(clock_);
video_receive_stream_ =
std::make_unique<webrtc::internal::VideoReceiveStream2>(
task_queue_factory_.get(), loop_.task_queue(),
&rtp_stream_receiver_controller_, kDefaultNumCpuCores,
&packet_router_, config_.Copy(), process_thread_.get(),
&call_stats_, clock_, timing_);
}
protected:
test::RunLoop loop_;
std::unique_ptr<ProcessThread> process_thread_;
const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
VideoReceiveStream::Config config_;
internal::CallStats call_stats_;
MockVideoDecoder mock_h264_video_decoder_;
MockVideoDecoderFactory mock_h264_decoder_factory_;
cricket::FakeVideoRenderer fake_renderer_;
MockTransport mock_transport_;
PacketRouter packet_router_;
RtpStreamReceiverController rtp_stream_receiver_controller_;
std::unique_ptr<webrtc::internal::VideoReceiveStream2> video_receive_stream_;
Clock* clock_;
VCMTiming* timing_;
};
TEST_F(VideoReceiveStream2TestWithLazyDecoderCreation, LazyDecoderCreation) {
constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
RtpPacketToSend rtppacket(nullptr);
uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
memcpy(payload, idr_nalu, sizeof(idr_nalu));
rtppacket.SetMarker(true);
rtppacket.SetSsrc(1111);
rtppacket.SetPayloadType(99);
rtppacket.SetSequenceNumber(1);
rtppacket.SetTimestamp(0);
// No decoder is created here.
EXPECT_CALL(mock_h264_decoder_factory_, CreateVideoDecoder(_)).Times(0);
video_receive_stream_->Start();
EXPECT_CALL(mock_h264_decoder_factory_, CreateVideoDecoder(_))
.WillOnce(Invoke([this](const SdpVideoFormat& format) {
test::VideoDecoderProxyFactory h264_decoder_factory(
&mock_h264_video_decoder_);
return h264_decoder_factory.CreateVideoDecoder(format);
}));
rtc::Event init_decode_event_;
EXPECT_CALL(mock_h264_video_decoder_, InitDecode(_, _))
.WillOnce(Invoke([&init_decode_event_](const VideoCodec* config,
int32_t number_of_cores) {
init_decode_event_.Set();
return 0;
}));
EXPECT_CALL(mock_h264_video_decoder_, RegisterDecodeCompleteCallback(_));
EXPECT_CALL(mock_h264_video_decoder_, Decode(_, false, _));
RtpPacketReceived parsed_packet;
ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
EXPECT_CALL(mock_h264_video_decoder_, Release());
// Make sure the decoder thread had a chance to run.
init_decode_event_.Wait(kDefaultTimeOutMs);
}
TEST_F(VideoReceiveStream2TestWithLazyDecoderCreation,
DeregisterDecoderThatsNotCreated) {
// No decoder is created here.
EXPECT_CALL(mock_h264_decoder_factory_, CreateVideoDecoder(_)).Times(0);
video_receive_stream_->Start();
video_receive_stream_->Stop();
}
} // namespace webrtc