diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index 903b9fb623..ba77b72d4f 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -110,6 +110,8 @@ void FrameBuffer::StartWaitForNextFrameOnQueue() { if (!frames_to_decode_.empty()) { // We have frames, deliver! frame = absl::WrapUnique(GetNextFrame()); + timing_->SetLastDecodeScheduledTimestamp( + clock_->TimeInMilliseconds()); } else if (clock_->TimeInMilliseconds() < latest_return_time_ms_) { // If there's no frames to decode and there is still time left, it // means that the frame buffer was cleared between creation and diff --git a/modules/video_coding/frame_buffer2_unittest.cc b/modules/video_coding/frame_buffer2_unittest.cc index d37efda57b..68acf813ae 100644 --- a/modules/video_coding/frame_buffer2_unittest.cc +++ b/modules/video_coding/frame_buffer2_unittest.cc @@ -55,7 +55,8 @@ class VCMTimingFake : public VCMTiming { return last_ms_; } - int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) override { + int64_t MaxWaitingTime(int64_t render_time_ms, + int64_t now_ms) const override { return render_time_ms - now_ms - kDecodeTime; } diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc index 748013cf47..dc25efe0c7 100644 --- a/modules/video_coding/timing.cc +++ b/modules/video_coding/timing.cc @@ -35,7 +35,7 @@ VCMTiming::VCMTiming(Clock* clock) num_decoded_frames_(0), low_latency_renderer_enabled_("enabled", true), zero_playout_delay_min_pacing_("min_pacing", TimeDelta::Millis(0)), - earliest_next_decode_start_time_(0) { + last_decode_scheduled_ts_(0) { ParseFieldTrial({&low_latency_renderer_enabled_}, field_trial::FindFullName("WebRTC-LowLatencyRenderer")); ParseFieldTrial({&zero_playout_delay_min_pacing_}, @@ -171,6 +171,12 @@ int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp, return RenderTimeMsInternal(frame_timestamp, now_ms); } +void VCMTiming::SetLastDecodeScheduledTimestamp( + int64_t last_decode_scheduled_ts) { + MutexLock lock(&mutex_); + last_decode_scheduled_ts_ = last_decode_scheduled_ts; +} + int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp, int64_t now_ms) const { constexpr int kLowLatencyRendererMaxPlayoutDelayMs = 500; @@ -202,7 +208,8 @@ int VCMTiming::RequiredDecodeTimeMs() const { return decode_time_ms; } -int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) { +int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, + int64_t now_ms) const { MutexLock lock(&mutex_); if (render_time_ms == 0 && zero_playout_delay_min_pacing_->us() > 0) { @@ -210,11 +217,11 @@ int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) { // rendered as soon as possible. However, the decoder can be choked if too // many frames are sent at ones. Therefore, limit the interframe delay to // |zero_playout_delay_min_pacing_|. - int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time_ + int64_t earliest_next_decode_start_time = + last_decode_scheduled_ts_ + zero_playout_delay_min_pacing_->ms(); + int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time ? 0 - : earliest_next_decode_start_time_ - now_ms; - earliest_next_decode_start_time_ = - now_ms + max_wait_time_ms + zero_playout_delay_min_pacing_->ms(); + : earliest_next_decode_start_time - now_ms; return max_wait_time_ms; } return render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_; diff --git a/modules/video_coding/timing.h b/modules/video_coding/timing.h index 1583082a58..2f3fdfe057 100644 --- a/modules/video_coding/timing.h +++ b/modules/video_coding/timing.h @@ -83,7 +83,7 @@ class VCMTiming { // Returns the maximum time in ms that we can wait for a frame to become // complete before we must pass it to the decoder. - virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms); + virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) const; // Returns the current target delay which is required delay + decode time + // render delay. @@ -105,6 +105,9 @@ class VCMTiming { absl::optional max_composition_delay_in_frames); absl::optional MaxCompositionDelayInFrames() const; + // Updates the last time a frame was scheduled for decoding. + void SetLastDecodeScheduledTimestamp(int64_t last_decode_scheduled_ts); + enum { kDefaultRenderDelayMs = 10 }; enum { kDelayMaxChangeMsPerS = 100 }; @@ -145,10 +148,10 @@ class VCMTiming { // used when min playout delay=0 and max playout delay>=0. FieldTrialParameter zero_playout_delay_min_pacing_ RTC_GUARDED_BY(mutex_); - // An estimate of when the last frame is scheduled to be sent to the decoder. + // Timestamp at which the last frame was scheduled to be sent to the decoder. // Used only when the RTP header extension playout delay is set to min=0 ms // which is indicated by a render time set to 0. - int64_t earliest_next_decode_start_time_ RTC_GUARDED_BY(mutex_); + int64_t last_decode_scheduled_ts_ RTC_GUARDED_BY(mutex_); }; } // namespace webrtc diff --git a/modules/video_coding/timing_unittest.cc b/modules/video_coding/timing_unittest.cc index be6ac52d92..988e55faab 100644 --- a/modules/video_coding/timing_unittest.cc +++ b/modules/video_coding/timing_unittest.cc @@ -169,21 +169,26 @@ TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) { clock.AdvanceTimeMilliseconds(kTimeDeltaMs); int64_t now_ms = clock.TimeInMilliseconds(); EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0); + timing.SetLastDecodeScheduledTimestamp(now_ms); } // Another frame submitted at the same time is paced according to the field // trial setting. int64_t now_ms = clock.TimeInMilliseconds(); EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); - // If there's a burst of frames, the min pacing interval is summed. - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 2 * kMinPacingMs); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 3 * kMinPacingMs); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 4 * kMinPacingMs); + // If there's a burst of frames, the wait time is calculated based on next + // decode time. + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime. constexpr int64_t kTwoMs = 2; clock.AdvanceTimeMilliseconds(kTwoMs); now_ms = clock.TimeInMilliseconds(); EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), - 5 * kMinPacingMs - kTwoMs); + kMinPacingMs - kTwoMs); + // A frame is decoded at the current time, the wait time should be restored to + // pacing delay. + timing.SetLastDecodeScheduledTimestamp(now_ms); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); } TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) {