From 985905d42df03d9e3cdeaa24447eeeced22c5557 Mon Sep 17 00:00:00 2001 From: Johannes Kron Date: Tue, 29 Jun 2021 11:37:06 +0200 Subject: [PATCH] Add fieldtrial to enable minimum pacing of video frames If the RTP header extension playout-delay is used and set to min=0, max>=0, frames are scheduled to be decoded as soon as possible. There's a risk that too many frames are sent to the decoder at once, which may cause problems further down in the video pipeline. This CL adds the fieldtrial WebRTC-ZeroPlayoutDelay with the parameter min_pacing that determines the minimum pacing interval between two frames scheduled for decoding. Bug: None Change-Id: I471f7718761cfce9789b3aa8adea3e8a16ecb2fd Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/223742 Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Johannes Kron Cr-Commit-Position: refs/heads/master@{#34387} --- .../video_coding/frame_buffer2_unittest.cc | 3 +- modules/video_coding/timing.cc | 26 ++++-- modules/video_coding/timing.h | 12 ++- modules/video_coding/timing_unittest.cc | 90 ++++++++++++++++++- 4 files changed, 119 insertions(+), 12 deletions(-) diff --git a/modules/video_coding/frame_buffer2_unittest.cc b/modules/video_coding/frame_buffer2_unittest.cc index 68acf813ae..d37efda57b 100644 --- a/modules/video_coding/frame_buffer2_unittest.cc +++ b/modules/video_coding/frame_buffer2_unittest.cc @@ -55,8 +55,7 @@ class VCMTimingFake : public VCMTiming { return last_ms_; } - int64_t MaxWaitingTime(int64_t render_time_ms, - int64_t now_ms) const override { + int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) override { return render_time_ms - now_ms - kDecodeTime; } diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc index eddac4f5de..7ad5edffb7 100644 --- a/modules/video_coding/timing.cc +++ b/modules/video_coding/timing.cc @@ -34,9 +34,13 @@ VCMTiming::VCMTiming(Clock* clock) prev_frame_timestamp_(0), timing_frame_info_(), num_decoded_frames_(0), - low_latency_renderer_enabled_("enabled", true) { + low_latency_renderer_enabled_("enabled", true), + zero_playout_delay_min_pacing_("min_pacing", TimeDelta::Millis(0)), + earliest_next_decode_start_time_(0) { ParseFieldTrial({&low_latency_renderer_enabled_}, field_trial::FindFullName("WebRTC-LowLatencyRenderer")); + ParseFieldTrial({&zero_playout_delay_min_pacing_}, + field_trial::FindFullName("WebRTC-ZeroPlayoutDelay")); } void VCMTiming::Reset() { @@ -199,14 +203,22 @@ int VCMTiming::RequiredDecodeTimeMs() const { return decode_time_ms; } -int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, - int64_t now_ms) const { +int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) { MutexLock lock(&mutex_); - const int64_t max_wait_time_ms = - render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_; - - return max_wait_time_ms; + if (render_time_ms == 0 && zero_playout_delay_min_pacing_->us() > 0) { + // |render_time_ms| == 0 indicates that the frame should be decoded and + // rendered as soon as possible. However, the decoder can be choked if too + // many frames are sent at ones. Therefore, limit the interframe delay to + // |zero_playout_delay_min_pacing_|. + int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time_ + ? 0 + : earliest_next_decode_start_time_ - now_ms; + earliest_next_decode_start_time_ = + now_ms + max_wait_time_ms + zero_playout_delay_min_pacing_->ms(); + return max_wait_time_ms; + } + return render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_; } int VCMTiming::TargetVideoDelay() const { diff --git a/modules/video_coding/timing.h b/modules/video_coding/timing.h index 736b5e9ae4..1583082a58 100644 --- a/modules/video_coding/timing.h +++ b/modules/video_coding/timing.h @@ -14,6 +14,7 @@ #include #include "absl/types/optional.h" +#include "api/units/time_delta.h" #include "api/video/video_timing.h" #include "modules/video_coding/codec_timer.h" #include "rtc_base/experiments/field_trial_parser.h" @@ -82,7 +83,7 @@ class VCMTiming { // Returns the maximum time in ms that we can wait for a frame to become // complete before we must pass it to the decoder. - virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) const; + virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms); // Returns the current target delay which is required delay + decode time + // render delay. @@ -139,6 +140,15 @@ class VCMTiming { FieldTrialParameter low_latency_renderer_enabled_ RTC_GUARDED_BY(mutex_); absl::optional max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); + // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing + // determines the minimum delay between frames scheduled for decoding that is + // used when min playout delay=0 and max playout delay>=0. + FieldTrialParameter zero_playout_delay_min_pacing_ + RTC_GUARDED_BY(mutex_); + // An estimate of when the last frame is scheduled to be sent to the decoder. + // Used only when the RTP header extension playout delay is set to min=0 ms + // which is indicated by a render time set to 0. + int64_t earliest_next_decode_start_time_ RTC_GUARDED_BY(mutex_); }; } // namespace webrtc diff --git a/modules/video_coding/timing_unittest.cc b/modules/video_coding/timing_unittest.cc index ee86605fb6..be6ac52d92 100644 --- a/modules/video_coding/timing_unittest.cc +++ b/modules/video_coding/timing_unittest.cc @@ -11,6 +11,7 @@ #include "modules/video_coding/timing.h" #include "system_wrappers/include/clock.h" +#include "test/field_trial.h" #include "test/gtest.h" namespace webrtc { @@ -18,7 +19,7 @@ namespace { const int kFps = 25; } // namespace -TEST(ReceiverTiming, Tests) { +TEST(ReceiverTimingTest, JitterDelay) { SimulatedClock clock(0); VCMTiming timing(&clock); timing.Reset(); @@ -110,7 +111,7 @@ TEST(ReceiverTiming, Tests) { timing.UpdateCurrentDelay(timestamp); } -TEST(ReceiverTiming, WrapAround) { +TEST(ReceiverTimingTest, TimestampWrapAround) { SimulatedClock clock(0); VCMTiming timing(&clock); // Provoke a wrap-around. The fifth frame will have wrapped at 25 fps. @@ -127,4 +128,89 @@ TEST(ReceiverTiming, WrapAround) { } } +TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) { + // This is the default path when the RTP playout delay header extension is set + // to min==0. + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0; + constexpr int64_t kZeroRenderTimeMs = 0; + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock); + timing.Reset(); + for (int i = 0; i < 10; ++i) { + clock.AdvanceTimeMilliseconds(kTimeDeltaMs); + int64_t now_ms = clock.TimeInMilliseconds(); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0); + } + // Another frame submitted at the same time also returns a negative max + // waiting time. + int64_t now_ms = clock.TimeInMilliseconds(); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0); + // MaxWaitingTime should be less than zero even if there's a burst of frames. + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0); +} + +TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) { + // The minimum pacing is enabled by a field trial and active if the RTP + // playout delay header extension is set to min==0. + constexpr int64_t kMinPacingMs = 3; + test::ScopedFieldTrials override_field_trials( + "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0; + constexpr int64_t kZeroRenderTimeMs = 0; + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock); + timing.Reset(); + // MaxWaitingTime() returns zero for evenly spaced video frames. + for (int i = 0; i < 10; ++i) { + clock.AdvanceTimeMilliseconds(kTimeDeltaMs); + int64_t now_ms = clock.TimeInMilliseconds(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0); + } + // Another frame submitted at the same time is paced according to the field + // trial setting. + int64_t now_ms = clock.TimeInMilliseconds(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); + // If there's a burst of frames, the min pacing interval is summed. + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 2 * kMinPacingMs); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 3 * kMinPacingMs); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 4 * kMinPacingMs); + // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime. + constexpr int64_t kTwoMs = 2; + clock.AdvanceTimeMilliseconds(kTwoMs); + now_ms = clock.TimeInMilliseconds(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), + 5 * kMinPacingMs - kTwoMs); +} + +TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) { + // The minimum pacing is enabled by a field trial but should not have any + // effect if render_time_ms is greater than 0; + test::ScopedFieldTrials override_field_trials( + "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0; + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock); + timing.Reset(); + clock.AdvanceTimeMilliseconds(kTimeDeltaMs); + int64_t now_ms = clock.TimeInMilliseconds(); + int64_t render_time_ms = now_ms + 30; + // Estimate the internal processing delay from the first frame. + int64_t estimated_processing_delay = + (render_time_ms - now_ms) - timing.MaxWaitingTime(render_time_ms, now_ms); + EXPECT_GT(estimated_processing_delay, 0); + + // Any other frame submitted at the same time should be scheduled according to + // its render time. + for (int i = 0; i < 5; ++i) { + render_time_ms += kTimeDeltaMs; + EXPECT_EQ(timing.MaxWaitingTime(render_time_ms, now_ms), + render_time_ms - now_ms - estimated_processing_delay); + } +} + } // namespace webrtc