From 9718711deed8d932503842df39bb161d578b1d64 Mon Sep 17 00:00:00 2001 From: philipel Date: Fri, 23 Mar 2018 10:43:21 +0100 Subject: [PATCH] VideoStreamDecoderImpl implementation, part 1. In this CL the OnFrame function is implemented. Bug: webrtc:8909 Change-Id: I68488a033e86eadd0b16d091faad14e9cda7cc36 Reviewed-on: https://webrtc-review.googlesource.com/64121 Reviewed-by: Niels Moller Reviewed-by: Karl Wiberg Commit-Queue: Philip Eliasson Cr-Commit-Position: refs/heads/master@{#22583} --- api/video/video_stream_decoder.h | 7 ++-- modules/video_coding/frame_buffer2.cc | 2 +- modules/video_coding/frame_buffer2.h | 1 + video/BUILD.gn | 3 ++ video/video_stream_decoder_impl.cc | 48 +++++++++++++++++++++++++-- video/video_stream_decoder_impl.h | 28 +++++++++++----- 6 files changed, 72 insertions(+), 17 deletions(-) diff --git a/api/video/video_stream_decoder.h b/api/video/video_stream_decoder.h index 0f04827f0e..1c4c5ff2a2 100644 --- a/api/video/video_stream_decoder.h +++ b/api/video/video_stream_decoder.h @@ -21,10 +21,6 @@ #include "api/video_codecs/video_decoder_factory.h" namespace webrtc { -// TODO(philipel): #include instead of forward declare when the relevant CL has -// landed. -class FrameKey; - // NOTE: This class is still under development and may change without notice. class VideoStreamDecoder { public: @@ -36,7 +32,8 @@ class VideoStreamDecoder { virtual void OnNonDecodableState() = 0; // Called with the last continuous frame. - virtual void OnContinuousUntil(const FrameKey& key) = 0; + virtual void OnContinuousUntil( + const video_coding::VideoLayerFrameId& key) = 0; // Called with the decoded frame. virtual void OnDecodedFrame(VideoFrame decodedImage, diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index 3a3a72751f..86f6e42c46 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -588,7 +588,7 @@ void FrameBuffer::UpdateJitterDelay() { void FrameBuffer::UpdateTimingFrameInfo() { TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo"); rtc::Optional info = timing_->GetTimingFrameInfo(); - if (info) + if (info && stats_callback_) stats_callback_->OnTimingFrameInfoUpdated(*info); } diff --git a/modules/video_coding/frame_buffer2.h b/modules/video_coding/frame_buffer2.h index c05fb8a369..1197e82dec 100644 --- a/modules/video_coding/frame_buffer2.h +++ b/modules/video_coding/frame_buffer2.h @@ -47,6 +47,7 @@ class FrameBuffer { // Insert a frame into the frame buffer. Returns the picture id // of the last continuous frame or -1 if there is no continuous frame. + // TODO(philipel): Return a VideoLayerFrameId and not only the picture id. int64_t InsertFrame(std::unique_ptr frame); // Get the next frame for decoding. Will return at latest after diff --git a/video/BUILD.gn b/video/BUILD.gn index 50deea8b75..8a996a3c56 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -116,7 +116,10 @@ rtc_source_set("video_stream_decoder_impl") { "../api:video_frame_api", "../api:video_stream_decoder", "../api/video_codecs:video_codecs_api", + "../modules/video_coding:video_coding", "../rtc_base:rtc_base_approved", + "../rtc_base:rtc_task_queue_api", + "../system_wrappers:system_wrappers", ] if (!build_with_chromium && is_clang) { diff --git a/video/video_stream_decoder_impl.cc b/video/video_stream_decoder_impl.cc index 5eca7d210d..6d262b63c3 100644 --- a/video/video_stream_decoder_impl.cc +++ b/video/video_stream_decoder_impl.cc @@ -10,19 +10,61 @@ #include "video/video_stream_decoder_impl.h" +#include "rtc_base/logging.h" #include "rtc_base/ptr_util.h" namespace webrtc { + VideoStreamDecoderImpl::VideoStreamDecoderImpl( VideoStreamDecoder::Callbacks* callbacks, VideoDecoderFactory* decoder_factory, std::map> decoder_settings) : callbacks_(callbacks), decoder_factory_(decoder_factory), - decoder_settings_(std::move(decoder_settings)) {} + decoder_settings_(std::move(decoder_settings)), + bookkeeping_queue_("video_stream_decoder_bookkeeping_queue"), + jitter_estimator_(Clock::GetRealTimeClock()), + timing_(Clock::GetRealTimeClock()), + frame_buffer_(Clock::GetRealTimeClock(), + &jitter_estimator_, + &timing_, + nullptr) {} -VideoStreamDecoderImpl::~VideoStreamDecoderImpl() {} +VideoStreamDecoderImpl::~VideoStreamDecoderImpl() { + frame_buffer_.Stop(); +} void VideoStreamDecoderImpl::OnFrame( - std::unique_ptr frame) {} + std::unique_ptr frame) { + if (!bookkeeping_queue_.IsCurrent()) { + struct OnFrameTask : rtc::QueuedTask { + OnFrameTask(std::unique_ptr frame, + VideoStreamDecoderImpl* video_stream_decoder) + : frame_(std::move(frame)), + video_stream_decoder_(video_stream_decoder) {} + + bool Run() { + video_stream_decoder_->OnFrame(std::move(frame_)); + return true; + } + + std::unique_ptr frame_; + VideoStreamDecoderImpl* video_stream_decoder_; + }; + + bookkeeping_queue_.PostTask( + rtc::MakeUnique(std::move(frame), this)); + return; + } + + RTC_DCHECK_RUN_ON(&bookkeeping_queue_); + + uint64_t continuous_pid = frame_buffer_.InsertFrame(std::move(frame)); + video_coding::VideoLayerFrameId continuous_id(continuous_pid, 0); + if (last_continuous_id_ < continuous_id) { + last_continuous_id_ = continuous_id; + callbacks_->OnContinuousUntil(last_continuous_id_); + } +} + } // namespace webrtc diff --git a/video/video_stream_decoder_impl.h b/video/video_stream_decoder_impl.h index f44913ba05..97de050104 100644 --- a/video/video_stream_decoder_impl.h +++ b/video/video_stream_decoder_impl.h @@ -11,17 +11,17 @@ #ifndef VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ #define VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ -#include #include #include #include -#include "api/optional.h" -#include "api/video/encoded_frame.h" -#include "api/video/video_frame.h" #include "api/video/video_stream_decoder.h" -#include "api/video_codecs/sdp_video_format.h" -#include "api/video_codecs/video_decoder_factory.h" +#include "modules/video_coding/frame_buffer2.h" +#include "modules/video_coding/jitter_estimator.h" +#include "modules/video_coding/timing.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/thread_checker.h" +#include "system_wrappers/include/clock.h" namespace webrtc { @@ -45,9 +45,21 @@ class VideoStreamDecoderImpl : public VideoStreamDecoder, rtc::Optional decode_time_ms, rtc::Optional qp) override; - VideoStreamDecoder::Callbacks* callbacks_; - VideoDecoderFactory* decoder_factory_; + VideoStreamDecoder::Callbacks* const callbacks_ + RTC_PT_GUARDED_BY(bookkeeping_queue_); + VideoDecoderFactory* const decoder_factory_; std::map> decoder_settings_; + + // The |bookkeeping_queue_| is used to: + // - Make |callbacks_|. + // - Insert/extract frames from the |frame_buffer_| + // - Synchronize with whatever thread that makes the Decoded callback. + rtc::TaskQueue bookkeeping_queue_; + + VCMJitterEstimator jitter_estimator_; + VCMTiming timing_; + video_coding::FrameBuffer frame_buffer_; + video_coding::VideoLayerFrameId last_continuous_id_; }; } // namespace webrtc