From 844876d050246ea3dead9d73c2d56fae9dd6f59d Mon Sep 17 00:00:00 2001 From: philipel Date: Thu, 5 Apr 2018 11:02:54 +0200 Subject: [PATCH] VideoStreamDecoderImpl implementation, part 3. This CL implements the functions related to decoding. Bug: webrtc:8909 Change-Id: Iefa3c1565a9b9ae93f14992b4a1cca141b7c5193 Reviewed-on: https://webrtc-review.googlesource.com/66403 Reviewed-by: Stefan Holmer Commit-Queue: Philip Eliasson Cr-Commit-Position: refs/heads/master@{#22747} --- video/video_stream_decoder_impl.cc | 102 ++++++++++++++++++++++++++++- video/video_stream_decoder_impl.h | 20 ++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/video/video_stream_decoder_impl.cc b/video/video_stream_decoder_impl.cc index 3a9751e67f..7b4a1bab49 100644 --- a/video/video_stream_decoder_impl.cc +++ b/video/video_stream_decoder_impl.cc @@ -11,6 +11,7 @@ #include "video/video_stream_decoder_impl.h" #include "rtc_base/logging.h" +#include "rtc_base/numerics/mod_ops.h" #include "rtc_base/ptr_util.h" namespace webrtc { @@ -23,15 +24,24 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl( decoder_factory_(decoder_factory), decoder_settings_(std::move(decoder_settings)), bookkeeping_queue_("video_stream_decoder_bookkeeping_queue"), + decode_thread_(&DecodeLoop, + this, + "video_stream_decoder_decode_thread", + rtc::kHighestPriority), jitter_estimator_(Clock::GetRealTimeClock()), timing_(Clock::GetRealTimeClock()), frame_buffer_(Clock::GetRealTimeClock(), &jitter_estimator_, &timing_, - nullptr) {} + nullptr), + next_start_time_index_(0) { + decode_start_time_.fill({-1, 0}); + decode_thread_.Start(); +} VideoStreamDecoderImpl::~VideoStreamDecoderImpl() { frame_buffer_.Stop(); + decode_thread_.Stop(); } void VideoStreamDecoderImpl::OnFrame( @@ -111,4 +121,94 @@ VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) { return decoder_.get(); } +// static +void VideoStreamDecoderImpl::DecodeLoop(void* ptr) { + // TODO(philipel): Remove this and use rtc::Event::kForever when it's + // supported by the |frame_buffer_|. + static constexpr int kForever = 100000000; + + int max_wait_time_ms = kForever; + bool keyframe_required = true; + auto* vs_decoder = static_cast(ptr); + while (true) { + DecodeResult decode_result = + vs_decoder->DecodeNextFrame(max_wait_time_ms, keyframe_required); + + switch (decode_result) { + case kOk: { + max_wait_time_ms = kForever; + keyframe_required = false; + break; + } + case kDecodeFailure: { + max_wait_time_ms = 0; + keyframe_required = true; + break; + } + case kNoFrame: { + max_wait_time_ms = kForever; + // If we end up here it means that we got a decoding error and there is + // no keyframe available in the |frame_buffer_|. + vs_decoder->bookkeeping_queue_.PostTask([vs_decoder]() { + RTC_DCHECK_RUN_ON(&vs_decoder->bookkeeping_queue_); + vs_decoder->callbacks_->OnNonDecodableState(); + }); + break; + } + case kNoDecoder: { + max_wait_time_ms = kForever; + break; + } + case kShutdown: { + return; + } + } + } +} + +VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeNextFrame( + int max_wait_time_ms, + bool keyframe_required) { + std::unique_ptr frame; + video_coding::FrameBuffer::ReturnReason res = + frame_buffer_.NextFrame(max_wait_time_ms, &frame, keyframe_required); + + if (res == video_coding::FrameBuffer::ReturnReason::kStopped) + return kShutdown; + + if (frame) { + VideoDecoder* decoder = GetDecoder(frame->PayloadType()); + if (!decoder) { + RTC_LOG(LS_WARNING) << "Failed to get decoder, dropping frame (" + << frame->id.picture_id << ":" + << frame->id.spatial_layer << ")."; + return kNoDecoder; + } + + int64_t decode_start_time_ms = rtc::TimeMillis(); + uint32_t frame_timestamp = frame->timestamp; + bookkeeping_queue_.PostTask( + [this, decode_start_time_ms, frame_timestamp]() { + RTC_DCHECK_RUN_ON(&bookkeeping_queue_); + // Saving decode start time this way wont work if we decode spatial + // layers sequentially. + decode_start_time_[next_start_time_index_] = {frame_timestamp, + decode_start_time_ms}; + next_start_time_index_ = + Add(next_start_time_index_, 1); + }); + + int32_t decode_result = + decoder->Decode(frame->EncodedImage(), + false, // missing_frame + nullptr, // rtp fragmentation header + nullptr, // codec specific info + frame->RenderTimeMs()); + + return decode_result == WEBRTC_VIDEO_CODEC_OK ? kOk : kDecodeFailure; + } + + return kNoFrame; +} + } // namespace webrtc diff --git a/video/video_stream_decoder_impl.h b/video/video_stream_decoder_impl.h index 55751764ef..42c1b946d2 100644 --- a/video/video_stream_decoder_impl.h +++ b/video/video_stream_decoder_impl.h @@ -20,6 +20,7 @@ #include "modules/video_coding/frame_buffer2.h" #include "modules/video_coding/jitter_estimator.h" #include "modules/video_coding/timing.h" +#include "rtc_base/platform_thread.h" #include "rtc_base/task_queue.h" #include "rtc_base/thread_checker.h" #include "system_wrappers/include/clock.h" @@ -39,7 +40,17 @@ class VideoStreamDecoderImpl : public VideoStreamDecoder, void OnFrame(std::unique_ptr frame) override; private: + enum DecodeResult { + kOk, + kDecodeFailure, + kNoFrame, + kNoDecoder, + kShutdown, + }; + VideoDecoder* GetDecoder(int payload_type); + static void DecodeLoop(void* ptr); + DecodeResult DecodeNextFrame(int max_wait_time_ms, bool keyframe_required); // Implements DecodedImageCallback interface int32_t Decoded(VideoFrame& decodedImage) override; @@ -59,12 +70,21 @@ class VideoStreamDecoderImpl : public VideoStreamDecoder, // - Synchronize with whatever thread that makes the Decoded callback. rtc::TaskQueue bookkeeping_queue_; + rtc::PlatformThread decode_thread_; VCMJitterEstimator jitter_estimator_; VCMTiming timing_; video_coding::FrameBuffer frame_buffer_; video_coding::VideoLayerFrameId last_continuous_id_; rtc::Optional current_payload_type_; std::unique_ptr decoder_; + + // Keep track of the |decode_start_time_| of the last |kDecodeTimeMemory| + // number of frames. The |decode_start_time_| array contain + // --> pairs. + static constexpr int kDecodeTimeMemory = 8; + std::array, kDecodeTimeMemory> decode_start_time_ + RTC_GUARDED_BY(bookkeeping_queue_); + int next_start_time_index_ RTC_GUARDED_BY(bookkeeping_queue_); }; } // namespace webrtc