From 3dc4780d8e58e1de597876b385324132c50fc8ec Mon Sep 17 00:00:00 2001 From: philipel Date: Thu, 10 Sep 2020 15:30:02 +0200 Subject: [PATCH] Added VideoContentType to OnDecodedFrame callback. Also added a FrameInfo struct to propagate various meta information along side it. Bug: webrtc:9106 Change-Id: I1feb9f94c662c367f7c6e0a50d33705fdd5346bb Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/183880 Reviewed-by: Ilya Nikolaevskiy Reviewed-by: Niels Moller Commit-Queue: Philip Eliasson Cr-Commit-Position: refs/heads/master@{#32079} --- api/video/BUILD.gn | 1 + api/video/video_stream_decoder.h | 17 +++++++-- video/video_stream_decoder_impl.cc | 55 +++++++++++++++--------------- video/video_stream_decoder_impl.h | 16 ++++----- 4 files changed, 51 insertions(+), 38 deletions(-) diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 6de518cd19..4d1d3e3264 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -222,6 +222,7 @@ rtc_source_set("video_stream_decoder") { deps = [ ":encoded_frame", ":video_frame", + ":video_rtp_headers", "../task_queue", "../units:time_delta", "../video_codecs:video_codecs_api", diff --git a/api/video/video_stream_decoder.h b/api/video/video_stream_decoder.h index 8f27fa4dbe..48e6b195ff 100644 --- a/api/video/video_stream_decoder.h +++ b/api/video/video_stream_decoder.h @@ -17,6 +17,7 @@ #include "api/units/time_delta.h" #include "api/video/encoded_frame.h" +#include "api/video/video_content_type.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder_factory.h" @@ -29,6 +30,12 @@ class VideoStreamDecoderInterface { public: virtual ~Callbacks() = default; + struct FrameInfo { + absl::optional decode_time_ms; + absl::optional qp; + VideoContentType content_type; + }; + // Called when the VideoStreamDecoder enters a non-decodable state. virtual void OnNonDecodableState() = 0; @@ -37,9 +44,15 @@ class VideoStreamDecoderInterface { const video_coding::VideoLayerFrameId& key) = 0; // Called with the decoded frame. - virtual void OnDecodedFrame(VideoFrame decodedImage, + virtual void OnDecodedFrame(VideoFrame frame, absl::optional decode_time_ms, - absl::optional qp) = 0; + absl::optional qp) {} + + // TODO(philipel): Make pure virtual and remove old function above when + // it is no longer used. + virtual void OnDecodedFrame(VideoFrame frame, const FrameInfo& meta_info) { + OnDecodedFrame(std::move(frame), meta_info.decode_time_ms, meta_info.qp); + } }; virtual ~VideoStreamDecoderInterface() = default; diff --git a/video/video_stream_decoder_impl.cc b/video/video_stream_decoder_impl.cc index 02ba45e253..e7fe7b0b86 100644 --- a/video/video_stream_decoder_impl.cc +++ b/video/video_stream_decoder_impl.cc @@ -26,7 +26,7 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl( std::map> decoder_settings) : timing_(Clock::GetRealTimeClock()), decode_callbacks_(this), - next_frame_timestamps_index_(0), + next_frame_info_index_(0), callbacks_(callbacks), keyframe_required_(true), decoder_factory_(decoder_factory), @@ -39,7 +39,6 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl( decode_queue_(task_queue_factory->CreateTaskQueue( "video_stream_decoder_decode_queue", TaskQueueFactory::Priority::NORMAL)) { - frame_timestamps_.fill({-1, -1, -1}); bookkeeping_queue_.PostTask([this]() { RTC_DCHECK_RUN_ON(&bookkeeping_queue_); StartNextDecode(); @@ -125,16 +124,15 @@ VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) { return decoder_.get(); } -void VideoStreamDecoderImpl::SaveFrameTimestamps( +void VideoStreamDecoderImpl::SaveFrameInfo( const video_coding::EncodedFrame& frame) { - FrameTimestamps* frame_timestamps = - &frame_timestamps_[next_frame_timestamps_index_]; - frame_timestamps->timestamp = frame.Timestamp(); - frame_timestamps->decode_start_time_ms = rtc::TimeMillis(); - frame_timestamps->render_time_us = frame.RenderTimeMs() * 1000; + FrameInfo* frame_info = &frame_info_[next_frame_info_index_]; + frame_info->timestamp = frame.Timestamp(); + frame_info->decode_start_time_ms = rtc::TimeMillis(); + frame_info->render_time_us = frame.RenderTimeMs() * 1000; + frame_info->content_type = frame.EncodedImage().content_type_; - next_frame_timestamps_index_ = - Add(next_frame_timestamps_index_, 1); + next_frame_info_index_ = Add(next_frame_info_index_, 1); } void VideoStreamDecoderImpl::StartNextDecode() { @@ -155,7 +153,7 @@ void VideoStreamDecoderImpl::OnNextFrameCallback( switch (result) { case video_coding::FrameBuffer::kFrameFound: { RTC_DCHECK(frame); - SaveFrameTimestamps(*frame); + SaveFrameInfo(*frame); MutexLock lock(&shut_down_mutex_); if (shut_down_) { @@ -230,14 +228,14 @@ VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeFrame( } } -VideoStreamDecoderImpl::FrameTimestamps* -VideoStreamDecoderImpl::GetFrameTimestamps(int64_t timestamp) { - int start_time_index = next_frame_timestamps_index_; - for (int i = 0; i < kFrameTimestampsMemory; ++i) { - start_time_index = Subtract(start_time_index, 1); +VideoStreamDecoderImpl::FrameInfo* VideoStreamDecoderImpl::GetFrameInfo( + int64_t timestamp) { + int start_time_index = next_frame_info_index_; + for (int i = 0; i < kFrameInfoMemory; ++i) { + start_time_index = Subtract(start_time_index, 1); - if (frame_timestamps_[start_time_index].timestamp == timestamp) - return &frame_timestamps_[start_time_index]; + if (frame_info_[start_time_index].timestamp == timestamp) + return &frame_info_[start_time_index]; } return nullptr; @@ -253,26 +251,27 @@ void VideoStreamDecoderImpl::OnDecodedFrameCallback( decode_time_ms, qp]() { RTC_DCHECK_RUN_ON(&bookkeeping_queue_); - FrameTimestamps* frame_timestamps = - GetFrameTimestamps(decoded_image.timestamp()); - if (!frame_timestamps) { + FrameInfo* frame_info = GetFrameInfo(decoded_image.timestamp()); + if (!frame_info) { RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp" << decoded_image.timestamp(); return; } - absl::optional casted_qp; + Callbacks::FrameInfo callback_info; + callback_info.content_type = frame_info->content_type; + if (qp) - casted_qp.emplace(*qp); + callback_info.qp.emplace(*qp); - absl::optional casted_decode_time_ms(decode_time_ms.value_or( - decode_stop_time_ms - frame_timestamps->decode_start_time_ms)); + callback_info.decode_time_ms = decode_time_ms.value_or( + decode_stop_time_ms - frame_info->decode_start_time_ms); - timing_.StopDecodeTimer(*casted_decode_time_ms, decode_stop_time_ms); + timing_.StopDecodeTimer(*callback_info.decode_time_ms, decode_stop_time_ms); VideoFrame copy = decoded_image; - copy.set_timestamp_us(frame_timestamps->render_time_us); - callbacks_->OnDecodedFrame(copy, casted_decode_time_ms, casted_qp); + copy.set_timestamp_us(frame_info->render_time_us); + callbacks_->OnDecodedFrame(copy, callback_info); }); } diff --git a/video/video_stream_decoder_impl.h b/video/video_stream_decoder_impl.h index 2f33e9d349..69a8195054 100644 --- a/video/video_stream_decoder_impl.h +++ b/video/video_stream_decoder_impl.h @@ -62,16 +62,16 @@ class VideoStreamDecoderImpl : public VideoStreamDecoderInterface { kDecodeFailure, }; - struct FrameTimestamps { - int64_t timestamp; + struct FrameInfo { + int64_t timestamp = -1; int64_t decode_start_time_ms; int64_t render_time_us; + VideoContentType content_type; }; - void SaveFrameTimestamps(const video_coding::EncodedFrame& frame) - RTC_RUN_ON(bookkeeping_queue_); - FrameTimestamps* GetFrameTimestamps(int64_t timestamp) + void SaveFrameInfo(const video_coding::EncodedFrame& frame) RTC_RUN_ON(bookkeeping_queue_); + FrameInfo* GetFrameInfo(int64_t timestamp) RTC_RUN_ON(bookkeeping_queue_); void StartNextDecode() RTC_RUN_ON(bookkeeping_queue_); void OnNextFrameCallback(std::unique_ptr frame, video_coding::FrameBuffer::ReturnReason res) @@ -90,10 +90,10 @@ class VideoStreamDecoderImpl : public VideoStreamDecoderInterface { // Some decoders are pipelined so it is not sufficient to save frame info // for the last frame only. - static constexpr int kFrameTimestampsMemory = 8; - std::array frame_timestamps_ + static constexpr int kFrameInfoMemory = 8; + std::array frame_info_ RTC_GUARDED_BY(bookkeeping_queue_); - int next_frame_timestamps_index_ RTC_GUARDED_BY(bookkeeping_queue_); + int next_frame_info_index_ RTC_GUARDED_BY(bookkeeping_queue_); VideoStreamDecoderInterface::Callbacks* const callbacks_ RTC_PT_GUARDED_BY(bookkeeping_queue_); video_coding::VideoLayerFrameId last_continuous_id_