From 24f9a8b398ceb104b5c6247bb85368ce98041124 Mon Sep 17 00:00:00 2001 From: Rasmus Brandt Date: Mon, 8 May 2023 16:32:48 +0200 Subject: [PATCH] Document VCMTiming::VideoDelayTimings better. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reorder and rename members. * Add comments. * Define struct first in the class, as per style guide. * Update direct callers. Bug: webrtc:15085 Change-Id: I37d26cae1953dacbba7d0507da48e3829ab84ba5 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/304403 Commit-Queue: Rasmus Brandt Reviewed-by: Åsa Persson Cr-Commit-Position: refs/heads/main@{#40011} --- .../deprecated/receiver_unittest.cc | 2 +- .../include/video_coding_defines.h | 2 +- modules/video_coding/timing/timing.cc | 27 ++++++------ modules/video_coding/timing/timing.h | 44 ++++++++++++------- video/receive_statistics_proxy2.cc | 4 +- video/receive_statistics_proxy2.h | 2 +- video/video_stream_buffer_controller.cc | 4 +- 7 files changed, 48 insertions(+), 37 deletions(-) diff --git a/modules/video_coding/deprecated/receiver_unittest.cc b/modules/video_coding/deprecated/receiver_unittest.cc index 9b7f020a27..8845a47c3a 100644 --- a/modules/video_coding/deprecated/receiver_unittest.cc +++ b/modules/video_coding/deprecated/receiver_unittest.cc @@ -453,7 +453,7 @@ TEST_F(VCMReceiverTimingTest, FrameForDecodingPreferLateDecoding) { auto timings = timing_.GetTimings(); TimeDelta render_delay = timings.render_delay; - TimeDelta max_decode = timings.max_decode_duration; + TimeDelta max_decode = timings.estimated_max_decode_time; // Construct test samples. // render_timestamps are the timestamps stored in the Frame; diff --git a/modules/video_coding/include/video_coding_defines.h b/modules/video_coding/include/video_coding_defines.h index 8f70e0298d..4026c8b339 100644 --- a/modules/video_coding/include/video_coding_defines.h +++ b/modules/video_coding/include/video_coding_defines.h @@ -76,7 +76,7 @@ class VCMReceiveStatisticsCallback { virtual void OnDroppedFrames(uint32_t frames_dropped) = 0; - virtual void OnFrameBufferTimingsUpdated(int max_decode_ms, + virtual void OnFrameBufferTimingsUpdated(int estimated_max_decode_time_ms, int current_delay_ms, int target_delay_ms, int jitter_buffer_ms, diff --git a/modules/video_coding/timing/timing.cc b/modules/video_coding/timing/timing.cc index 5c3fe69553..1035d6f01e 100644 --- a/modules/video_coding/timing/timing.cc +++ b/modules/video_coding/timing/timing.cc @@ -156,8 +156,8 @@ void VCMTiming::UpdateCurrentDelay(Timestamp render_time, Timestamp actual_decode_time) { MutexLock lock(&mutex_); TimeDelta target_delay = TargetDelayInternal(); - TimeDelta delayed = - (actual_decode_time - render_time) + RequiredDecodeTime() + render_delay_; + TimeDelta delayed = (actual_decode_time - render_time) + + EstimatedMaxDecodeTime() + render_delay_; // Only consider `delayed` as negative by more than a few microseconds. if (delayed.ms() < 0) { @@ -211,7 +211,7 @@ Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp, return estimated_complete_time + actual_delay; } -TimeDelta VCMTiming::RequiredDecodeTime() const { +TimeDelta VCMTiming::EstimatedMaxDecodeTime() const { const int decode_time_ms = decode_time_filter_->RequiredDecodeTimeMs(); RTC_DCHECK_GE(decode_time_ms, 0); return TimeDelta::Millis(decode_time_ms); @@ -239,7 +239,7 @@ TimeDelta VCMTiming::MaxWaitingTime(Timestamp render_time, : earliest_next_decode_start_time - now; return max_wait_time; } - return render_time - now - RequiredDecodeTime() - render_delay_; + return render_time - now - EstimatedMaxDecodeTime() - render_delay_; } TimeDelta VCMTiming::TargetVideoDelay() const { @@ -249,7 +249,7 @@ TimeDelta VCMTiming::TargetVideoDelay() const { TimeDelta VCMTiming::TargetDelayInternal() const { return std::max(min_playout_delay_, - jitter_delay_ + RequiredDecodeTime() + render_delay_); + jitter_delay_ + EstimatedMaxDecodeTime() + render_delay_); } VideoFrame::RenderParameters VCMTiming::RenderParameters() const { @@ -269,14 +269,15 @@ bool VCMTiming::UseLowLatencyRendering() const { VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const { MutexLock lock(&mutex_); - return VideoDelayTimings{.max_decode_duration = RequiredDecodeTime(), - .current_delay = current_delay_, - .target_delay = TargetDelayInternal(), - .jitter_buffer_delay = jitter_delay_, - .min_playout_delay = min_playout_delay_, - .max_playout_delay = max_playout_delay_, - .render_delay = render_delay_, - .num_decoded_frames = num_decoded_frames_}; + return VideoDelayTimings{ + .num_decoded_frames = num_decoded_frames_, + .jitter_delay = jitter_delay_, + .estimated_max_decode_time = EstimatedMaxDecodeTime(), + .render_delay = render_delay_, + .min_playout_delay = min_playout_delay_, + .max_playout_delay = max_playout_delay_, + .target_delay = TargetDelayInternal(), + .current_delay = current_delay_}; } void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) { diff --git a/modules/video_coding/timing/timing.h b/modules/video_coding/timing/timing.h index 161c8d6a64..dbac40df58 100644 --- a/modules/video_coding/timing/timing.h +++ b/modules/video_coding/timing/timing.h @@ -29,8 +29,30 @@ namespace webrtc { class VCMTiming { public: - static constexpr auto kDefaultRenderDelay = TimeDelta::Millis(10); - static constexpr auto kDelayMaxChangeMsPerS = 100; + struct VideoDelayTimings { + size_t num_decoded_frames; + // Delay added to smooth out frame delay variation ("jitter") caused by + // the network. + TimeDelta jitter_delay; + // Estimated time needed to decode a video frame. Obtained as the 95th + // percentile decode time over a recent time window. + TimeDelta estimated_max_decode_time; + // Estimated time needed to render a frame. Set to a constant. + TimeDelta render_delay; + // Minimum total delay used when determining render time for a frame. + // Obtained from API, `playout-delay` RTP header extension, or A/V sync. + TimeDelta min_playout_delay; + // Maximum total delay used when determining render time for a frame. + // Obtained from `playout-delay` RTP header extension. + TimeDelta max_playout_delay; + // Target delay. Obtained from all the elements above. + TimeDelta target_delay; + // Current delay. Obtained by smoothing out the target delay. + TimeDelta current_delay; + }; + + static constexpr TimeDelta kDefaultRenderDelay = TimeDelta::Millis(10); + static constexpr int kDelayMaxChangeMsPerS = 100; VCMTiming(Clock* clock, const FieldTrialsView& field_trials); virtual ~VCMTiming() = default; @@ -92,18 +114,7 @@ class VCMTiming { // render delay. TimeDelta TargetVideoDelay() const; - // Return current timing information. Returns true if the first frame has been - // decoded, false otherwise. - struct VideoDelayTimings { - TimeDelta max_decode_duration; - TimeDelta current_delay; - TimeDelta target_delay; - TimeDelta jitter_buffer_delay; - TimeDelta min_playout_delay; - TimeDelta max_playout_delay; - TimeDelta render_delay; - size_t num_decoded_frames; - }; + // Return current timing information. VideoDelayTimings GetTimings() const; void SetTimingFrameInfo(const TimingFrameInfo& info); @@ -117,14 +128,13 @@ class VCMTiming { // Updates the last time a frame was scheduled for decoding. void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled); - protected: - TimeDelta RequiredDecodeTime() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + private: + TimeDelta EstimatedMaxDecodeTime() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - private: mutable Mutex mutex_; Clock* const clock_; const std::unique_ptr ts_extrapolator_ diff --git a/video/receive_statistics_proxy2.cc b/video/receive_statistics_proxy2.cc index f5e6307be5..67798b4453 100644 --- a/video/receive_statistics_proxy2.cc +++ b/video/receive_statistics_proxy2.cc @@ -663,14 +663,14 @@ void ReceiveStatisticsProxy::OnDecoderInfo( } void ReceiveStatisticsProxy::OnFrameBufferTimingsUpdated( - int max_decode_ms, + int estimated_max_decode_time_ms, int current_delay_ms, int target_delay_ms, int jitter_buffer_ms, int min_playout_delay_ms, int render_delay_ms) { RTC_DCHECK_RUN_ON(&main_thread_); - stats_.max_decode_ms = max_decode_ms; + stats_.max_decode_ms = estimated_max_decode_time_ms; stats_.current_delay_ms = current_delay_ms; stats_.target_delay_ms = target_delay_ms; stats_.jitter_buffer_ms = jitter_buffer_ms; diff --git a/video/receive_statistics_proxy2.h b/video/receive_statistics_proxy2.h index 8613417d14..6e7445aeb2 100644 --- a/video/receive_statistics_proxy2.h +++ b/video/receive_statistics_proxy2.h @@ -90,7 +90,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, size_t size_bytes, VideoContentType content_type) override; void OnDroppedFrames(uint32_t frames_dropped) override; - void OnFrameBufferTimingsUpdated(int max_decode_ms, + void OnFrameBufferTimingsUpdated(int estimated_max_decode_time_ms, int current_delay_ms, int target_delay_ms, int jitter_buffer_ms, diff --git a/video/video_stream_buffer_controller.cc b/video/video_stream_buffer_controller.cc index c95fb7d64d..7b1619cd04 100644 --- a/video/video_stream_buffer_controller.cc +++ b/video/video_stream_buffer_controller.cc @@ -319,8 +319,8 @@ void VideoStreamBufferController::UpdateJitterDelay() { auto timings = timing_->GetTimings(); if (timings.num_decoded_frames) { stats_proxy_->OnFrameBufferTimingsUpdated( - timings.max_decode_duration.ms(), timings.current_delay.ms(), - timings.target_delay.ms(), timings.jitter_buffer_delay.ms(), + timings.estimated_max_decode_time.ms(), timings.current_delay.ms(), + timings.target_delay.ms(), timings.jitter_delay.ms(), timings.min_playout_delay.ms(), timings.render_delay.ms()); } }