From 29dd8d864be366b148bbcb0ca14972c5187174f9 Mon Sep 17 00:00:00 2001 From: Markus Handell Date: Wed, 15 Dec 2021 12:19:15 +0100 Subject: [PATCH] ZeroHertzAdapterMode: delay & repeat frames. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change introduces a delay in the frame cadence forwarded to the VideoStreamEncoder. In case the delivery of frames into the VideoSinkInterface stops, ZeroHertzAdapterMode will repeat a previously received frame until new frames appear. go/rtc-0hz-present Bug: chromium:1255737 Change-Id: I689ac63a41a09951715ea2c26f491e7c4ad0d11d Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/240060 Reviewed-by: Erik Språng Commit-Queue: Markus Handell Cr-Commit-Position: refs/heads/main@{#35542} --- video/BUILD.gn | 1 + video/frame_cadence_adapter.cc | 139 +++++++++++++++++++- video/frame_cadence_adapter_unittest.cc | 161 +++++++++++++++++++++++- 3 files changed, 294 insertions(+), 7 deletions(-) diff --git a/video/BUILD.gn b/video/BUILD.gn index c1dfd62e14..f4d9668861 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -266,6 +266,7 @@ rtc_library("frame_cadence_adapter") { deps = [ "../api:sequence_checker", "../api/task_queue", + "../api/units:time_delta", "../api/video:video_frame", "../rtc_base:logging", "../rtc_base:macromagic", diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc index c46790992c..3b961c8107 100644 --- a/video/frame_cadence_adapter.cc +++ b/video/frame_cadence_adapter.cc @@ -11,11 +11,14 @@ #include "video/frame_cadence_adapter.h" #include +#include #include #include #include "api/sequence_checker.h" #include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" +#include "api/video/video_frame.h" #include "rtc_base/logging.h" #include "rtc_base/race_checker.h" #include "rtc_base/rate_statistics.h" @@ -23,6 +26,8 @@ #include "rtc_base/system/no_unique_address.h" #include "rtc_base/task_utils/pending_task_safety_flag.h" #include "rtc_base/task_utils/to_queued_task.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/time_utils.h" #include "system_wrappers/include/clock.h" #include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/metrics.h" @@ -86,7 +91,9 @@ class PassthroughAdapterMode : public AdapterMode { // Implements a frame cadence adapter supporting zero-hertz input. class ZeroHertzAdapterMode : public AdapterMode { public: - ZeroHertzAdapterMode(FrameCadenceAdapterInterface::Callback* callback, + ZeroHertzAdapterMode(TaskQueueBase* queue, + Clock* clock, + FrameCadenceAdapterInterface::Callback* callback, double max_fps); // Adapter overrides. @@ -97,11 +104,37 @@ class ZeroHertzAdapterMode : public AdapterMode { void UpdateFrameRate() override {} private: + // Processes incoming frames on a delayed cadence. + void ProcessOnDelayedCadence() RTC_RUN_ON(sequence_checker_); + // Repeats a frame in the abscence of incoming frames. Slows down when QP + // convergence is attained, and stops the cadence terminally when new frames + // have arrived. `scheduled_delay` specifies the delay by which to modify the + // repeate frame's timestamps when it's sent. + void ProcessRepeatedFrameOnDelayedCadence(int frame_id, + TimeDelta scheduled_delay) + RTC_RUN_ON(sequence_checker_); + // Sends a frame, updating the timestamp to the current time. + void SendFrameNow(const VideoFrame& frame); + + TaskQueueBase* const queue_; + Clock* const clock_; FrameCadenceAdapterInterface::Callback* const callback_; // The configured max_fps. // TODO(crbug.com/1255737): support max_fps updates. const double max_fps_; + // How much the incoming frame sequence is delayed by. + const TimeDelta frame_delay_ = TimeDelta::Seconds(1) / max_fps_; + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; + // A queue of incoming frames and repeated frames. + std::deque queued_frames_ RTC_GUARDED_BY(sequence_checker_); + // The current frame ID to use when starting to repeat frames. This is used + // for cancelling deferred repeated frame processing happening. + int current_frame_id_ RTC_GUARDED_BY(sequence_checker_) = 0; + // True when we are repeating frames. + bool is_repeating_ RTC_GUARDED_BY(sequence_checker_) = false; + + ScopedTaskSafety safety_; }; class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { @@ -175,9 +208,11 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { }; ZeroHertzAdapterMode::ZeroHertzAdapterMode( + TaskQueueBase* queue, + Clock* clock, FrameCadenceAdapterInterface::Callback* callback, double max_fps) - : callback_(callback), max_fps_(max_fps) { + : queue_(queue), clock_(clock), callback_(callback), max_fps_(max_fps) { sequence_checker_.Detach(); } @@ -185,8 +220,25 @@ void ZeroHertzAdapterMode::OnFrame(Timestamp post_time, int frames_scheduled_for_processing, const VideoFrame& frame) { RTC_DCHECK_RUN_ON(&sequence_checker_); - // TODO(crbug.com/1255737): fill with meaningful implementation. - callback_->OnFrame(post_time, frames_scheduled_for_processing, frame); + + // Remove stored repeating frame if needed. + if (is_repeating_) { + RTC_DCHECK(queued_frames_.size() == 1); + RTC_LOG(LS_VERBOSE) << __func__ << " this " << this + << " cancel repeat and restart with original"; + queued_frames_.pop_front(); + } + + // Store the frame in the queue and schedule deferred processing. + queued_frames_.push_back(frame); + current_frame_id_++; + is_repeating_ = false; + queue_->PostDelayedTask(ToQueuedTask(safety_, + [this] { + RTC_DCHECK_RUN_ON(&sequence_checker_); + ProcessOnDelayedCadence(); + }), + frame_delay_.ms()); } absl::optional ZeroHertzAdapterMode::GetInputFrameRateFps() { @@ -194,6 +246,83 @@ absl::optional ZeroHertzAdapterMode::GetInputFrameRateFps() { return max_fps_; } +// RTC_RUN_ON(&sequence_checker_) +void ZeroHertzAdapterMode::ProcessOnDelayedCadence() { + RTC_DCHECK(!queued_frames_.empty()); + + SendFrameNow(queued_frames_.front()); + + // If there were two or more frames stored, we do not have to schedule repeats + // of the front frame. + if (queued_frames_.size() > 1) { + queued_frames_.pop_front(); + return; + } + + // There's only one frame to send. Schedule a repeat sequence, which is + // cancelled by |current_frame_id_| getting incremented should new frames + // arrive. + is_repeating_ = true; + int frame_id = current_frame_id_; + queue_->PostDelayedTask(ToQueuedTask(safety_, + [this, frame_id] { + RTC_DCHECK_RUN_ON(&sequence_checker_); + ProcessRepeatedFrameOnDelayedCadence( + frame_id, frame_delay_); + }), + frame_delay_.ms()); +} + +// RTC_RUN_ON(&sequence_checker_) +void ZeroHertzAdapterMode::ProcessRepeatedFrameOnDelayedCadence( + int frame_id, + TimeDelta scheduled_delay) { + RTC_DCHECK(!queued_frames_.empty()); + + // Cancel this invocation if new frames turned up. + if (frame_id != current_frame_id_) + return; + + VideoFrame& frame = queued_frames_.front(); + + // Since this is a repeated frame, nothing changed compared to before. + VideoFrame::UpdateRect empty_update_rect; + empty_update_rect.MakeEmptyUpdate(); + frame.set_update_rect(empty_update_rect); + + // Adjust timestamps of the frame of the repeat, accounting for the delay in + // scheduling this method. + // NOTE: No need to update the RTP timestamp as the VideoStreamEncoder + // overwrites it based on its chosen NTP timestamp source. + if (frame.timestamp_us() > 0) + frame.set_timestamp_us(frame.timestamp_us() + scheduled_delay.us()); + if (frame.ntp_time_ms()) + frame.set_ntp_time_ms(frame.ntp_time_ms() + scheduled_delay.ms()); + SendFrameNow(frame); + + // TODO(crbug.com/1255737): Wire in a QP convergence signal here and adjust + // the delay on QP convergence to some lowest rate being a compromise between + // RTP receiver keyframe-requesting timeout (3s), backend limitations and some + // worst case RTT. + int delay_ms = frame_delay_.ms(); + + // Schedule another repeat depending on if QP converged. + queue_->PostDelayedTask(ToQueuedTask(safety_, + [this, frame_id] { + RTC_DCHECK_RUN_ON(&sequence_checker_); + ProcessRepeatedFrameOnDelayedCadence( + frame_id, frame_delay_); + }), + delay_ms); +} + +void ZeroHertzAdapterMode::SendFrameNow(const VideoFrame& frame) { + // TODO(crbug.com/1255737): figure out if frames_scheduled_for_processing + // makes sense to compute in this implementation. + callback_->OnFrame(/*post_time=*/clock_->CurrentTime(), + /*frames_scheduled_for_processing=*/1, frame); +} + FrameCadenceAdapterImpl::FrameCadenceAdapterImpl(Clock* clock, TaskQueueBase* queue) : clock_(clock), @@ -284,7 +413,7 @@ void FrameCadenceAdapterImpl::MaybeReconfigureAdapters( bool is_zero_hertz_enabled = IsZeroHertzScreenshareEnabled(); if (is_zero_hertz_enabled) { if (!was_zero_hertz_enabled) { - zero_hertz_adapter_.emplace(callback_, + zero_hertz_adapter_.emplace(queue_, clock_, callback_, source_constraints_->max_fps.value()); } current_adapter_mode_ = &zero_hertz_adapter_.value(); diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc index dae0b864a8..946fe0c949 100644 --- a/video/frame_cadence_adapter_unittest.cc +++ b/video/frame_cadence_adapter_unittest.cc @@ -18,7 +18,9 @@ #include "api/video/video_frame.h" #include "rtc_base/rate_statistics.h" #include "rtc_base/ref_counted_object.h" +#include "rtc_base/time_utils.h" #include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/ntp_time.h" #include "test/field_trial.h" #include "test/gmock.h" #include "test/gtest.h" @@ -29,10 +31,9 @@ namespace { using ::testing::_; using ::testing::ElementsAre; +using ::testing::Invoke; using ::testing::Mock; using ::testing::Pair; -using ::testing::Ref; -using ::testing::UnorderedElementsAre; VideoFrame CreateFrame() { return VideoFrame::Builder() @@ -41,6 +42,16 @@ VideoFrame CreateFrame() { .build(); } +VideoFrame CreateFrameWithTimestamps( + GlobalSimulatedTimeController* time_controller) { + return VideoFrame::Builder() + .set_video_frame_buffer( + rtc::make_ref_counted(/*width=*/16, /*height=*/16)) + .set_ntp_time_ms(time_controller->GetClock()->CurrentNtpInMilliseconds()) + .set_timestamp_us(time_controller->GetClock()->CurrentTime().us()) + .build(); +} + std::unique_ptr CreateAdapter(Clock* clock) { return FrameCadenceAdapterInterface::Create(clock, TaskQueueBase::Current()); } @@ -186,6 +197,152 @@ TEST(FrameCadenceAdapterTest, adapter->GetInputFrameRateFps()); } +TEST(FrameCadenceAdapterTest, ForwardsFramesDelayed) { + ZeroHertzFieldTrialEnabler enabler; + MockCallback callback; + GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); + auto adapter = CreateAdapter(time_controller.GetClock()); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); + constexpr int kNumFrames = 3; + NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime(); + auto frame = CreateFrameWithTimestamps(&time_controller); + int64_t original_timestamp_us = frame.timestamp_us(); + for (int index = 0; index != kNumFrames; ++index) { + EXPECT_CALL(callback, OnFrame).Times(0); + adapter->OnFrame(frame); + EXPECT_CALL(callback, OnFrame) + .WillOnce(Invoke([&](Timestamp post_time, int, + const VideoFrame& frame) { + EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); + EXPECT_EQ(frame.timestamp_us(), + original_timestamp_us + index * rtc::kNumMicrosecsPerSec); + EXPECT_EQ(frame.ntp_time_ms(), original_ntp_time.ToMs() + + index * rtc::kNumMillisecsPerSec); + })); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); + frame = CreateFrameWithTimestamps(&time_controller); + } +} + +TEST(FrameCadenceAdapterTest, RepeatsFramesDelayed) { + // Logic in the frame cadence adapter avoids modifying frame NTP and render + // timestamps if these timestamps looks unset, which is the case when the + // clock is initialized running from 0. For this reason we choose the + // `time_controller` initialization constant to something arbitrary which is + // not 0. + ZeroHertzFieldTrialEnabler enabler; + MockCallback callback; + GlobalSimulatedTimeController time_controller(Timestamp::Millis(47892223)); + auto adapter = CreateAdapter(time_controller.GetClock()); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); + NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime(); + + // Send one frame, expect 2 subsequent repeats. + auto frame = CreateFrameWithTimestamps(&time_controller); + int64_t original_timestamp_us = frame.timestamp_us(); + adapter->OnFrame(frame); + + EXPECT_CALL(callback, OnFrame) + .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); + EXPECT_EQ(frame.timestamp_us(), original_timestamp_us); + EXPECT_EQ(frame.ntp_time_ms(), original_ntp_time.ToMs()); + })); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); + Mock::VerifyAndClearExpectations(&callback); + + EXPECT_CALL(callback, OnFrame) + .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); + EXPECT_EQ(frame.timestamp_us(), + original_timestamp_us + rtc::kNumMicrosecsPerSec); + EXPECT_EQ(frame.ntp_time_ms(), + original_ntp_time.ToMs() + rtc::kNumMillisecsPerSec); + })); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); + Mock::VerifyAndClearExpectations(&callback); + + EXPECT_CALL(callback, OnFrame) + .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); + EXPECT_EQ(frame.timestamp_us(), + original_timestamp_us + 2 * rtc::kNumMicrosecsPerSec); + EXPECT_EQ(frame.ntp_time_ms(), + original_ntp_time.ToMs() + 2 * rtc::kNumMillisecsPerSec); + })); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); +} + +TEST(FrameCadenceAdapterTest, + RepeatsFramesWithoutTimestampsWithUnsetTimestamps) { + // Logic in the frame cadence adapter avoids modifying frame NTP and render + // timestamps if these timestamps looks unset, which is the case when the + // clock is initialized running from 0. In this test we deliberately don't set + // it to zero, but select unset timestamps in the frames (via CreateFrame()) + // and verify that the timestamp modifying logic doesn't depend on the current + // time. + ZeroHertzFieldTrialEnabler enabler; + MockCallback callback; + GlobalSimulatedTimeController time_controller(Timestamp::Millis(4711)); + auto adapter = CreateAdapter(time_controller.GetClock()); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); + + // Send one frame, expect a repeat. + adapter->OnFrame(CreateFrame()); + EXPECT_CALL(callback, OnFrame) + .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); + EXPECT_EQ(frame.timestamp_us(), 0); + EXPECT_EQ(frame.ntp_time_ms(), 0); + })); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); + Mock::VerifyAndClearExpectations(&callback); + EXPECT_CALL(callback, OnFrame) + .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); + EXPECT_EQ(frame.timestamp_us(), 0); + EXPECT_EQ(frame.ntp_time_ms(), 0); + })); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); +} + +TEST(FrameCadenceAdapterTest, StopsRepeatingFramesDelayed) { + // At 1s, the initially scheduled frame appears. + // At 2s, the repeated initial frame appears. + // At 2.5s, we schedule another new frame. + // At 3.5s, we receive this frame. + ZeroHertzFieldTrialEnabler enabler; + MockCallback callback; + GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); + auto adapter = CreateAdapter(time_controller.GetClock()); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); + NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime(); + + // Send one frame, expect 1 subsequent repeat. + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); + EXPECT_CALL(callback, OnFrame).Times(2); + time_controller.AdvanceTime(TimeDelta::Seconds(2.5)); + Mock::VerifyAndClearExpectations(&callback); + + // Send the new frame at 2.5s, which should appear after 3.5s. + adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); + EXPECT_CALL(callback, OnFrame) + .WillOnce(Invoke([&](Timestamp, int, const VideoFrame& frame) { + EXPECT_EQ(frame.timestamp_us(), 5 * rtc::kNumMicrosecsPerSec / 2); + EXPECT_EQ(frame.ntp_time_ms(), + original_ntp_time.ToMs() + 5u * rtc::kNumMillisecsPerSec / 2); + })); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); +} + class FrameCadenceAdapterMetricsTest : public ::testing::Test { public: FrameCadenceAdapterMetricsTest() : time_controller_(Timestamp::Millis(1)) {