admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ZeroHertzAdapterMode: delay & repeat frames.

Browse Source 
This change introduces a delay in the frame cadence forwarded to
the VideoStreamEncoder. In case the delivery of frames into the
VideoSinkInterface stops, ZeroHertzAdapterMode will repeat a
previously received frame until new frames appear.

go/rtc-0hz-present

Bug: chromium:1255737
Change-Id: I689ac63a41a09951715ea2c26f491e7c4ad0d11d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/240060
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Markus Handell <handellm@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35542}
This commit is contained in:
Markus Handell 2021-12-15 12:19:15 +01:00 committed by WebRTC LUCI CQ
parent 1fe08e1abe
commit 29dd8d864b
3 changed files with 294 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
video/BUILD.gn
View File

@ -266,6 +266,7 @@ rtc_library("frame_cadence_adapter") {
deps = [
"../api:sequence_checker",
"../api/task_queue",
"../api/units:time_delta",
"../api/video:video_frame",
"../rtc_base:logging",
"../rtc_base:macromagic",

139
video/frame_cadence_adapter.cc
View File

@ -11,11 +11,14 @@
#include "video/frame_cadence_adapter.h"
#include <atomic>
#include <deque>
#include <memory>
#include <utility>
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
#include "api/units/time_delta.h"
#include "api/video/video_frame.h"
#include "rtc_base/logging.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/rate_statistics.h"
@ -23,6 +26,8 @@
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/task_utils/pending_task_safety_flag.h"
#include "rtc_base/task_utils/to_queued_task.h"
#include "rtc_base/thread_annotations.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/clock.h"
#include "system_wrappers/include/field_trial.h"
#include "system_wrappers/include/metrics.h"
@ -86,7 +91,9 @@ class PassthroughAdapterMode : public AdapterMode {
// Implements a frame cadence adapter supporting zero-hertz input.
class ZeroHertzAdapterMode : public AdapterMode {
public:
ZeroHertzAdapterMode(FrameCadenceAdapterInterface::Callback* callback,
ZeroHertzAdapterMode(TaskQueueBase* queue,
Clock* clock,
FrameCadenceAdapterInterface::Callback* callback,
double max_fps);
// Adapter overrides.
@ -97,11 +104,37 @@ class ZeroHertzAdapterMode : public AdapterMode {
void UpdateFrameRate() override {}
private:
// Processes incoming frames on a delayed cadence.
void ProcessOnDelayedCadence() RTC_RUN_ON(sequence_checker_);
// Repeats a frame in the abscence of incoming frames. Slows down when QP
// convergence is attained, and stops the cadence terminally when new frames
// have arrived. `scheduled_delay` specifies the delay by which to modify the
// repeate frame's timestamps when it's sent.
void ProcessRepeatedFrameOnDelayedCadence(int frame_id,
TimeDelta scheduled_delay)
RTC_RUN_ON(sequence_checker_);
// Sends a frame, updating the timestamp to the current time.
void SendFrameNow(const VideoFrame& frame);
TaskQueueBase* const queue_;
Clock* const clock_;
FrameCadenceAdapterInterface::Callback* const callback_;
// The configured max_fps.
// TODO(crbug.com/1255737): support max_fps updates.
const double max_fps_;
// How much the incoming frame sequence is delayed by.
const TimeDelta frame_delay_ = TimeDelta::Seconds(1) / max_fps_;
RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
// A queue of incoming frames and repeated frames.
std::deque<VideoFrame> queued_frames_ RTC_GUARDED_BY(sequence_checker_);
// The current frame ID to use when starting to repeat frames. This is used
// for cancelling deferred repeated frame processing happening.
int current_frame_id_ RTC_GUARDED_BY(sequence_checker_) = 0;
// True when we are repeating frames.
bool is_repeating_ RTC_GUARDED_BY(sequence_checker_) = false;
ScopedTaskSafety safety_;
};
class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
@ -175,9 +208,11 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
};
ZeroHertzAdapterMode::ZeroHertzAdapterMode(
TaskQueueBase* queue,
Clock* clock,
FrameCadenceAdapterInterface::Callback* callback,
double max_fps)
: callback_(callback), max_fps_(max_fps) {
: queue_(queue), clock_(clock), callback_(callback), max_fps_(max_fps) {
sequence_checker_.Detach();
}
@ -185,8 +220,25 @@ void ZeroHertzAdapterMode::OnFrame(Timestamp post_time,
int frames_scheduled_for_processing,
const VideoFrame& frame) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
// TODO(crbug.com/1255737): fill with meaningful implementation.
callback_->OnFrame(post_time, frames_scheduled_for_processing, frame);
// Remove stored repeating frame if needed.
if (is_repeating_) {
RTC_DCHECK(queued_frames_.size() == 1);
RTC_LOG(LS_VERBOSE) << __func__ << " this " << this
<< " cancel repeat and restart with original";
queued_frames_.pop_front();
}
// Store the frame in the queue and schedule deferred processing.
queued_frames_.push_back(frame);
current_frame_id_++;
is_repeating_ = false;
queue_->PostDelayedTask(ToQueuedTask(safety_,
[this] {
RTC_DCHECK_RUN_ON(&sequence_checker_);
ProcessOnDelayedCadence();
}),
frame_delay_.ms());
}
absl::optional<uint32_t> ZeroHertzAdapterMode::GetInputFrameRateFps() {
@ -194,6 +246,83 @@ absl::optional<uint32_t> ZeroHertzAdapterMode::GetInputFrameRateFps() {
return max_fps_;
}
// RTC_RUN_ON(&sequence_checker_)
void ZeroHertzAdapterMode::ProcessOnDelayedCadence() {
RTC_DCHECK(!queued_frames_.empty());
SendFrameNow(queued_frames_.front());
// If there were two or more frames stored, we do not have to schedule repeats
// of the front frame.
if (queued_frames_.size() > 1) {
queued_frames_.pop_front();
return;
}
// There's only one frame to send. Schedule a repeat sequence, which is
// cancelled by |current_frame_id_| getting incremented should new frames
// arrive.
is_repeating_ = true;
int frame_id = current_frame_id_;
queue_->PostDelayedTask(ToQueuedTask(safety_,
[this, frame_id] {
RTC_DCHECK_RUN_ON(&sequence_checker_);
ProcessRepeatedFrameOnDelayedCadence(
frame_id, frame_delay_);
}),
frame_delay_.ms());
}
// RTC_RUN_ON(&sequence_checker_)
void ZeroHertzAdapterMode::ProcessRepeatedFrameOnDelayedCadence(
int frame_id,
TimeDelta scheduled_delay) {
RTC_DCHECK(!queued_frames_.empty());
// Cancel this invocation if new frames turned up.
if (frame_id != current_frame_id_)
return;
VideoFrame& frame = queued_frames_.front();
// Since this is a repeated frame, nothing changed compared to before.
VideoFrame::UpdateRect empty_update_rect;
empty_update_rect.MakeEmptyUpdate();
frame.set_update_rect(empty_update_rect);
// Adjust timestamps of the frame of the repeat, accounting for the delay in
// scheduling this method.
// NOTE: No need to update the RTP timestamp as the VideoStreamEncoder
// overwrites it based on its chosen NTP timestamp source.
if (frame.timestamp_us() > 0)
frame.set_timestamp_us(frame.timestamp_us() + scheduled_delay.us());
if (frame.ntp_time_ms())
frame.set_ntp_time_ms(frame.ntp_time_ms() + scheduled_delay.ms());
SendFrameNow(frame);
// TODO(crbug.com/1255737): Wire in a QP convergence signal here and adjust
// the delay on QP convergence to some lowest rate being a compromise between
// RTP receiver keyframe-requesting timeout (3s), backend limitations and some
// worst case RTT.
int delay_ms = frame_delay_.ms();
// Schedule another repeat depending on if QP converged.
queue_->PostDelayedTask(ToQueuedTask(safety_,
[this, frame_id] {
RTC_DCHECK_RUN_ON(&sequence_checker_);
ProcessRepeatedFrameOnDelayedCadence(
frame_id, frame_delay_);
}),
delay_ms);
}
void ZeroHertzAdapterMode::SendFrameNow(const VideoFrame& frame) {
// TODO(crbug.com/1255737): figure out if frames_scheduled_for_processing
// makes sense to compute in this implementation.
callback_->OnFrame(/*post_time=*/clock_->CurrentTime(),
/*frames_scheduled_for_processing=*/1, frame);
}
FrameCadenceAdapterImpl::FrameCadenceAdapterImpl(Clock* clock,
TaskQueueBase* queue)
: clock_(clock),
@ -284,7 +413,7 @@ void FrameCadenceAdapterImpl::MaybeReconfigureAdapters(
bool is_zero_hertz_enabled = IsZeroHertzScreenshareEnabled();
if (is_zero_hertz_enabled) {
if (!was_zero_hertz_enabled) {
zero_hertz_adapter_.emplace(callback_,
zero_hertz_adapter_.emplace(queue_, clock_, callback_,
source_constraints_->max_fps.value());
}
current_adapter_mode_ = &zero_hertz_adapter_.value();

161
video/frame_cadence_adapter_unittest.cc
View File

@ -18,7 +18,9 @@
#include "api/video/video_frame.h"
#include "rtc_base/rate_statistics.h"
#include "rtc_base/ref_counted_object.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/metrics.h"
#include "system_wrappers/include/ntp_time.h"
#include "test/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
@ -29,10 +31,9 @@ namespace {
using ::testing::_;
using ::testing::ElementsAre;
using ::testing::Invoke;
using ::testing::Mock;
using ::testing::Pair;
using ::testing::Ref;
using ::testing::UnorderedElementsAre;
VideoFrame CreateFrame() {
return VideoFrame::Builder()
@ -41,6 +42,16 @@ VideoFrame CreateFrame() {
.build();
}
VideoFrame CreateFrameWithTimestamps(
GlobalSimulatedTimeController* time_controller) {
return VideoFrame::Builder()
.set_video_frame_buffer(
rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16))
.set_ntp_time_ms(time_controller->GetClock()->CurrentNtpInMilliseconds())
.set_timestamp_us(time_controller->GetClock()->CurrentTime().us())
.build();
}
std::unique_ptr<FrameCadenceAdapterInterface> CreateAdapter(Clock* clock) {
return FrameCadenceAdapterInterface::Create(clock, TaskQueueBase::Current());
}
@ -186,6 +197,152 @@ TEST(FrameCadenceAdapterTest,
adapter->GetInputFrameRateFps());
}
TEST(FrameCadenceAdapterTest, ForwardsFramesDelayed) {
ZeroHertzFieldTrialEnabler enabler;
MockCallback callback;
GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
auto adapter = CreateAdapter(time_controller.GetClock());
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1});
constexpr int kNumFrames = 3;
NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime();
auto frame = CreateFrameWithTimestamps(&time_controller);
int64_t original_timestamp_us = frame.timestamp_us();
for (int index = 0; index != kNumFrames; ++index) {
EXPECT_CALL(callback, OnFrame).Times(0);
adapter->OnFrame(frame);
EXPECT_CALL(callback, OnFrame)
.WillOnce(Invoke([&](Timestamp post_time, int,
const VideoFrame& frame) {
EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime());
EXPECT_EQ(frame.timestamp_us(),
original_timestamp_us + index * rtc::kNumMicrosecsPerSec);
EXPECT_EQ(frame.ntp_time_ms(), original_ntp_time.ToMs() +
index * rtc::kNumMillisecsPerSec);
}));
time_controller.AdvanceTime(TimeDelta::Seconds(1));
frame = CreateFrameWithTimestamps(&time_controller);
}
}
TEST(FrameCadenceAdapterTest, RepeatsFramesDelayed) {
// Logic in the frame cadence adapter avoids modifying frame NTP and render
// timestamps if these timestamps looks unset, which is the case when the
// clock is initialized running from 0. For this reason we choose the
// `time_controller` initialization constant to something arbitrary which is
// not 0.
ZeroHertzFieldTrialEnabler enabler;
MockCallback callback;
GlobalSimulatedTimeController time_controller(Timestamp::Millis(47892223));
auto adapter = CreateAdapter(time_controller.GetClock());
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1});
NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime();
// Send one frame, expect 2 subsequent repeats.
auto frame = CreateFrameWithTimestamps(&time_controller);
int64_t original_timestamp_us = frame.timestamp_us();
adapter->OnFrame(frame);
EXPECT_CALL(callback, OnFrame)
.WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) {
EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime());
EXPECT_EQ(frame.timestamp_us(), original_timestamp_us);
EXPECT_EQ(frame.ntp_time_ms(), original_ntp_time.ToMs());
}));
time_controller.AdvanceTime(TimeDelta::Seconds(1));
Mock::VerifyAndClearExpectations(&callback);
EXPECT_CALL(callback, OnFrame)
.WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) {
EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime());
EXPECT_EQ(frame.timestamp_us(),
original_timestamp_us + rtc::kNumMicrosecsPerSec);
EXPECT_EQ(frame.ntp_time_ms(),
original_ntp_time.ToMs() + rtc::kNumMillisecsPerSec);
}));
time_controller.AdvanceTime(TimeDelta::Seconds(1));
Mock::VerifyAndClearExpectations(&callback);
EXPECT_CALL(callback, OnFrame)
.WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) {
EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime());
EXPECT_EQ(frame.timestamp_us(),
original_timestamp_us + 2 * rtc::kNumMicrosecsPerSec);
EXPECT_EQ(frame.ntp_time_ms(),
original_ntp_time.ToMs() + 2 * rtc::kNumMillisecsPerSec);
}));
time_controller.AdvanceTime(TimeDelta::Seconds(1));
}
TEST(FrameCadenceAdapterTest,
RepeatsFramesWithoutTimestampsWithUnsetTimestamps) {
// Logic in the frame cadence adapter avoids modifying frame NTP and render
// timestamps if these timestamps looks unset, which is the case when the
// clock is initialized running from 0. In this test we deliberately don't set
// it to zero, but select unset timestamps in the frames (via CreateFrame())
// and verify that the timestamp modifying logic doesn't depend on the current
// time.
ZeroHertzFieldTrialEnabler enabler;
MockCallback callback;
GlobalSimulatedTimeController time_controller(Timestamp::Millis(4711));
auto adapter = CreateAdapter(time_controller.GetClock());
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1});
// Send one frame, expect a repeat.
adapter->OnFrame(CreateFrame());
EXPECT_CALL(callback, OnFrame)
.WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) {
EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime());
EXPECT_EQ(frame.timestamp_us(), 0);
EXPECT_EQ(frame.ntp_time_ms(), 0);
}));
time_controller.AdvanceTime(TimeDelta::Seconds(1));
Mock::VerifyAndClearExpectations(&callback);
EXPECT_CALL(callback, OnFrame)
.WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) {
EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime());
EXPECT_EQ(frame.timestamp_us(), 0);
EXPECT_EQ(frame.ntp_time_ms(), 0);
}));
time_controller.AdvanceTime(TimeDelta::Seconds(1));
}
TEST(FrameCadenceAdapterTest, StopsRepeatingFramesDelayed) {
// At 1s, the initially scheduled frame appears.
// At 2s, the repeated initial frame appears.
// At 2.5s, we schedule another new frame.
// At 3.5s, we receive this frame.
ZeroHertzFieldTrialEnabler enabler;
MockCallback callback;
GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
auto adapter = CreateAdapter(time_controller.GetClock());
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1});
NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime();
// Send one frame, expect 1 subsequent repeat.
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller));
EXPECT_CALL(callback, OnFrame).Times(2);
time_controller.AdvanceTime(TimeDelta::Seconds(2.5));
Mock::VerifyAndClearExpectations(&callback);
// Send the new frame at 2.5s, which should appear after 3.5s.
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller));
EXPECT_CALL(callback, OnFrame)
.WillOnce(Invoke([&](Timestamp, int, const VideoFrame& frame) {
EXPECT_EQ(frame.timestamp_us(), 5 * rtc::kNumMicrosecsPerSec / 2);
EXPECT_EQ(frame.ntp_time_ms(),
original_ntp_time.ToMs() + 5u * rtc::kNumMillisecsPerSec / 2);
}));
time_controller.AdvanceTime(TimeDelta::Seconds(1));
}
class FrameCadenceAdapterMetricsTest : public ::testing::Test {
public:
FrameCadenceAdapterMetricsTest() : time_controller_(Timestamp::Millis(1)) {