admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "FrameCadenceAdapter keep track of Input framerate"

Browse Source 
This reverts commit 784af1f42e89735587c442855fa01fc90475c449.

Reason for revert: Seems like test test_support_unittests 
 ResolutionAdaptsToAvailableBandwidth is flaky with this cl.

Original change's description:
> FrameCadenceAdapter keep track of Input framerate
>
> Refactor FrameCandenceAdapter to keep track of input frame rate.
>
> Also a field trial WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp is added to control if VideoFrame timestamp should be used or local clock when calculating frame rate.
> Uma is recorded to tell if input frame timestamp is monotonically increasing.
>
> Bug: webrtc:10481, webrtc:15887
> Change-Id: I6d698e9f9dcfe8c023d2d35371435c47f70102b0
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/342760
> Commit-Queue: Per Kjellander <perkj@webrtc.org>
> Reviewed-by: Erik Språng <sprang@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#41967}

Bug: webrtc:10481, webrtc:15887
Change-Id: Id9672764768f2f40f8e711e990ad8ac18c28efcc
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/344560
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
Owners-Override: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41969}
This commit is contained in:
Per Kjellander 2024-03-26 12:53:35 +00:00 committed by WebRTC LUCI CQ
parent d1e577dd80
commit d427e83a15
7 changed files with 72 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
experiments/field_trials.py
View File

@ -80,9 +80,6 @@ ACTIVE_FIELD_TRIALS: FrozenSet[FieldTrial] = frozenset([
FieldTrial('WebRTC-EncoderDataDumpDirectory', FieldTrial('WebRTC-EncoderDataDumpDirectory',
'b/296242528', 'b/296242528',
date(2024, 4, 1)), date(2024, 4, 1)),
FieldTrial('WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp',
'webrtc:15887',
date(2024, 10, 1)),
FieldTrial('WebRTC-IPv6NetworkResolutionFixes', FieldTrial('WebRTC-IPv6NetworkResolutionFixes',
'webrtc:14334', 'webrtc:14334',
date(2024, 4, 1)), date(2024, 4, 1)),

3
modules/congestion_controller/pcc/pcc_network_controller_unittest.cc
View File

@ -112,8 +112,7 @@ TEST(PccNetworkControllerTest, UpdatesTargetSendRate) {
ret_net->UpdateConfig( ret_net->UpdateConfig(
[](NetworkSimulationConfig* c) { c->delay = TimeDelta::Millis(200); }); [](NetworkSimulationConfig* c) { c->delay = TimeDelta::Millis(200); });
s.RunFor(TimeDelta::Seconds(35)); s.RunFor(TimeDelta::Seconds(35));
EXPECT_LE(client->target_rate().kbps(), 200); EXPECT_NEAR(client->target_rate().kbps(), 170, 50);
EXPECT_GT(client->target_rate().kbps(), 90);
} }
} // namespace test } // namespace test

69
video/frame_cadence_adapter.cc
View File

@ -12,7 +12,6 @@
#include <algorithm> #include <algorithm>
#include <atomic> #include <atomic>
#include <cstdint>
#include <deque> #include <deque>
#include <memory> #include <memory>
#include <utility> #include <utility>
@ -59,15 +58,15 @@ class AdapterMode {
virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0; virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0;
// Updates the frame rate. // Updates the frame rate.
virtual void UpdateFrameRate(Timestamp frame_timestamp) = 0; virtual void UpdateFrameRate() = 0;
}; };
// Implements a pass-through adapter. Single-threaded. // Implements a pass-through adapter. Single-threaded.
class PassthroughAdapterMode : public AdapterMode { class PassthroughAdapterMode : public AdapterMode {
public: public:
explicit PassthroughAdapterMode( PassthroughAdapterMode(Clock* clock,
FrameCadenceAdapterInterface::Callback* callback) FrameCadenceAdapterInterface::Callback* callback)
: callback_(callback) { : clock_(clock), callback_(callback) {
sequence_checker_.Detach(); sequence_checker_.Detach();
} }
@ -81,18 +80,16 @@ class PassthroughAdapterMode : public AdapterMode {
absl::optional<uint32_t> GetInputFrameRateFps() override { absl::optional<uint32_t> GetInputFrameRateFps() override {
RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_RUN_ON(&sequence_checker_);
return last_frame_rate_; return input_framerate_.Rate(clock_->TimeInMilliseconds());
} }
void UpdateFrameRate(Timestamp frame_timestamp) override { void UpdateFrameRate() override {
RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_RUN_ON(&sequence_checker_);
// RateStatistics will calculate a too high rate immediately after Update. input_framerate_.Update(1, clock_->TimeInMilliseconds());
last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms());
input_framerate_.Update(1, frame_timestamp.ms());
} }
private: private:
absl::optional<uint64_t> last_frame_rate_; Clock* const clock_;
FrameCadenceAdapterInterface::Callback* const callback_; FrameCadenceAdapterInterface::Callback* const callback_;
RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
// Input frame rate statistics for use when not in zero-hertz mode. // Input frame rate statistics for use when not in zero-hertz mode.
@ -128,7 +125,7 @@ class ZeroHertzAdapterMode : public AdapterMode {
bool queue_overload, bool queue_overload,
const VideoFrame& frame) override; const VideoFrame& frame) override;
absl::optional<uint32_t> GetInputFrameRateFps() override; absl::optional<uint32_t> GetInputFrameRateFps() override;
void UpdateFrameRate(Timestamp frame_timestamp) override {} void UpdateFrameRate() override {}
// Notified on dropped frames. // Notified on dropped frames.
void OnDiscardedFrame(); void OnDiscardedFrame();
@ -286,14 +283,12 @@ class VSyncEncodeAdapterMode : public AdapterMode {
absl::optional<uint32_t> GetInputFrameRateFps() override { absl::optional<uint32_t> GetInputFrameRateFps() override {
RTC_DCHECK_RUN_ON(&queue_sequence_checker_); RTC_DCHECK_RUN_ON(&queue_sequence_checker_);
return last_frame_rate_; return input_framerate_.Rate(clock_->TimeInMilliseconds());
} }
void UpdateFrameRate(Timestamp frame_timestamp) override { void UpdateFrameRate() override {
RTC_DCHECK_RUN_ON(&queue_sequence_checker_); RTC_DCHECK_RUN_ON(&queue_sequence_checker_);
// RateStatistics will calculate a too high rate immediately after Update. input_framerate_.Update(1, clock_->TimeInMilliseconds());
last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms());
input_framerate_.Update(1, frame_timestamp.ms());
} }
void EncodeAllEnqueuedFrames(); void EncodeAllEnqueuedFrames();
@ -313,8 +308,6 @@ class VSyncEncodeAdapterMode : public AdapterMode {
RTC_NO_UNIQUE_ADDRESS SequenceChecker queue_sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker queue_sequence_checker_;
rtc::scoped_refptr<PendingTaskSafetyFlag> queue_safety_flag_; rtc::scoped_refptr<PendingTaskSafetyFlag> queue_safety_flag_;
// Input frame rate statistics for use when not in zero-hertz mode. // Input frame rate statistics for use when not in zero-hertz mode.
absl::optional<uint64_t> last_frame_rate_
RTC_GUARDED_BY(queue_sequence_checker_);
RateStatistics input_framerate_ RTC_GUARDED_BY(queue_sequence_checker_){ RateStatistics input_framerate_ RTC_GUARDED_BY(queue_sequence_checker_){
FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000}; FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000};
FrameCadenceAdapterInterface::Callback* const callback_; FrameCadenceAdapterInterface::Callback* const callback_;
@ -322,8 +315,8 @@ class VSyncEncodeAdapterMode : public AdapterMode {
Metronome* metronome_; Metronome* metronome_;
TaskQueueBase* const worker_queue_; TaskQueueBase* const worker_queue_;
RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_;
// `worker_safety_` protects tasks on the worker queue related to // `worker_safety_` protects tasks on the worker queue related to `metronome_`
// `metronome_` since metronome usage must happen on worker thread. // since metronome usage must happen on worker thread.
ScopedTaskSafetyDetached worker_safety_; ScopedTaskSafetyDetached worker_safety_;
Timestamp expected_next_tick_ RTC_GUARDED_BY(worker_sequence_checker_) = Timestamp expected_next_tick_ RTC_GUARDED_BY(worker_sequence_checker_) =
Timestamp::PlusInfinity(); Timestamp::PlusInfinity();
@ -346,6 +339,7 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
void SetZeroHertzModeEnabled( void SetZeroHertzModeEnabled(
absl::optional<ZeroHertzModeParams> params) override; absl::optional<ZeroHertzModeParams> params) override;
absl::optional<uint32_t> GetInputFrameRateFps() override; absl::optional<uint32_t> GetInputFrameRateFps() override;
void UpdateFrameRate() override;
void UpdateLayerQualityConvergence(size_t spatial_index, void UpdateLayerQualityConvergence(size_t spatial_index,
bool quality_converged) override; bool quality_converged) override;
void UpdateLayerStatus(size_t spatial_index, bool enabled) override; void UpdateLayerStatus(size_t spatial_index, bool enabled) override;
@ -360,7 +354,6 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
const VideoTrackSourceConstraints& constraints) override; const VideoTrackSourceConstraints& constraints) override;
private: private:
void UpdateFrameRate(Timestamp frame_timestamp);
// Called from OnFrame in both pass-through and zero-hertz mode. // Called from OnFrame in both pass-through and zero-hertz mode.
void OnFrameOnMainQueue(Timestamp post_time, void OnFrameOnMainQueue(Timestamp post_time,
bool queue_overload, bool queue_overload,
@ -390,13 +383,6 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
// Kill-switch for the queue overload mechanism in zero-hertz mode. // Kill-switch for the queue overload mechanism in zero-hertz mode.
const bool frame_cadence_adapter_zero_hertz_queue_overload_enabled_; const bool frame_cadence_adapter_zero_hertz_queue_overload_enabled_;
// Field trial for using timestamp from video frames, rather than clock when
// calculating input frame rate.
const bool use_video_frame_timestamp_;
// Used for verifying that timestamps are monotonically increasing.
absl::optional<Timestamp> last_incoming_frame_timestamp_;
bool incoming_frame_timestamp_monotonically_increasing_ = true;
// The three possible modes we're under. // The three possible modes we're under.
absl::optional<PassthroughAdapterMode> passthrough_adapter_; absl::optional<PassthroughAdapterMode> passthrough_adapter_;
absl::optional<ZeroHertzAdapterMode> zero_hertz_adapter_; absl::optional<ZeroHertzAdapterMode> zero_hertz_adapter_;
@ -851,8 +837,6 @@ FrameCadenceAdapterImpl::FrameCadenceAdapterImpl(
!field_trials.IsDisabled("WebRTC-ZeroHertzScreenshare")), !field_trials.IsDisabled("WebRTC-ZeroHertzScreenshare")),
frame_cadence_adapter_zero_hertz_queue_overload_enabled_( frame_cadence_adapter_zero_hertz_queue_overload_enabled_(
!field_trials.IsDisabled("WebRTC-ZeroHertzQueueOverload")), !field_trials.IsDisabled("WebRTC-ZeroHertzQueueOverload")),
use_video_frame_timestamp_(field_trials.IsEnabled(
"WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp")),
metronome_(metronome), metronome_(metronome),
worker_queue_(worker_queue) {} worker_queue_(worker_queue) {}
@ -865,10 +849,6 @@ FrameCadenceAdapterImpl::~FrameCadenceAdapterImpl() {
absl::Cleanup cleanup = [adapter = std::move(vsync_encode_adapter_)] {}; absl::Cleanup cleanup = [adapter = std::move(vsync_encode_adapter_)] {};
worker_queue_->PostTask([cleanup = std::move(cleanup)] {}); worker_queue_->PostTask([cleanup = std::move(cleanup)] {});
} }
RTC_HISTOGRAM_BOOLEAN(
"WebRTC.Video.InputFrameTimestampMonotonicallyIncreasing",
incoming_frame_timestamp_monotonically_increasing_);
} }
void FrameCadenceAdapterImpl::Initialize(Callback* callback) { void FrameCadenceAdapterImpl::Initialize(Callback* callback) {
@ -879,7 +859,7 @@ void FrameCadenceAdapterImpl::Initialize(Callback* callback) {
vsync_encode_adapter_ = std::make_unique<VSyncEncodeAdapterMode>( vsync_encode_adapter_ = std::make_unique<VSyncEncodeAdapterMode>(
clock_, queue_, safety_.flag(), metronome_, worker_queue_, callback_); clock_, queue_, safety_.flag(), metronome_, worker_queue_, callback_);
} else { } else {
passthrough_adapter_.emplace(callback); passthrough_adapter_.emplace(clock_, callback);
} }
ConfigureCurrentAdapterWithoutZeroHertz(); ConfigureCurrentAdapterWithoutZeroHertz();
} }
@ -897,7 +877,7 @@ absl::optional<uint32_t> FrameCadenceAdapterImpl::GetInputFrameRateFps() {
return current_adapter_mode_->GetInputFrameRateFps(); return current_adapter_mode_->GetInputFrameRateFps();
} }
void FrameCadenceAdapterImpl::UpdateFrameRate(Timestamp frame_timestamp) { void FrameCadenceAdapterImpl::UpdateFrameRate() {
RTC_DCHECK_RUN_ON(queue_); RTC_DCHECK_RUN_ON(queue_);
// The frame rate need not be updated for the zero-hertz adapter. The // The frame rate need not be updated for the zero-hertz adapter. The
// vsync encode and passthrough adapter however uses it. Always pass frames // vsync encode and passthrough adapter however uses it. Always pass frames
@ -905,10 +885,10 @@ void FrameCadenceAdapterImpl::UpdateFrameRate(Timestamp frame_timestamp) {
// there be an adapter switch. // there be an adapter switch.
if (metronome_) { if (metronome_) {
RTC_CHECK(vsync_encode_adapter_); RTC_CHECK(vsync_encode_adapter_);
vsync_encode_adapter_->UpdateFrameRate(frame_timestamp); vsync_encode_adapter_->UpdateFrameRate();
} else { } else {
RTC_CHECK(passthrough_adapter_); RTC_CHECK(passthrough_adapter_);
passthrough_adapter_->UpdateFrameRate(frame_timestamp); passthrough_adapter_->UpdateFrameRate();
} }
} }
@ -999,19 +979,6 @@ void FrameCadenceAdapterImpl::OnFrameOnMainQueue(Timestamp post_time,
const VideoFrame& frame) { const VideoFrame& frame) {
RTC_DCHECK_RUN_ON(queue_); RTC_DCHECK_RUN_ON(queue_);
current_adapter_mode_->OnFrame(post_time, queue_overload, frame); current_adapter_mode_->OnFrame(post_time, queue_overload, frame);
if (last_incoming_frame_timestamp_ &&
last_incoming_frame_timestamp_ >=
Timestamp::Micros(frame.timestamp_us())) {
RTC_LOG(LS_ERROR)
<< "Incoming frame timestamp is not monotonically increasing"
<< " current: " << frame.timestamp_us()
<< " last: " << last_incoming_frame_timestamp_.value().us();
incoming_frame_timestamp_monotonically_increasing_ = false;
}
last_incoming_frame_timestamp_ = Timestamp::Micros(frame.timestamp_us());
Timestamp update_frame_rate_timestamp =
use_video_frame_timestamp_ ? *last_incoming_frame_timestamp_ : post_time;
UpdateFrameRate(update_frame_rate_timestamp);
} }
bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const { bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const {

4
video/frame_cadence_adapter.h
View File

@ -99,6 +99,10 @@ class FrameCadenceAdapterInterface
// zero-hertz mode is off, and returns the max framerate in zero-hertz mode. // zero-hertz mode is off, and returns the max framerate in zero-hertz mode.
virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0; virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0;
// Updates frame rate. This is done unconditionally irrespective of adapter
// mode.
virtual void UpdateFrameRate() = 0;
// Updates quality convergence status for an enabled spatial layer. // Updates quality convergence status for an enabled spatial layer.
// Convergence means QP has dropped to a low-enough level to warrant ceasing // Convergence means QP has dropped to a low-enough level to warrant ceasing
// to send identical frames at high frequency. // to send identical frames at high frequency.

62
video/frame_cadence_adapter_unittest.cc
View File

@ -10,8 +10,6 @@
#include "video/frame_cadence_adapter.h" #include "video/frame_cadence_adapter.h"
#include <cstdint>
#include <memory>
#include <utility> #include <utility>
#include <vector> #include <vector>
@ -136,8 +134,7 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsRateStatisticsByDefault) {
test::ScopedKeyValueConfig no_field_trials; test::ScopedKeyValueConfig no_field_trials;
GlobalSimulatedTimeController time_controller(Timestamp::Zero()); GlobalSimulatedTimeController time_controller(Timestamp::Zero());
auto adapter = CreateAdapter(no_field_trials, time_controller.GetClock()); auto adapter = CreateAdapter(no_field_trials, time_controller.GetClock());
MockCallback callback; adapter->Initialize(nullptr);
adapter->Initialize(&callback);
// Create an "oracle" rate statistics which should be followed on a sequence // Create an "oracle" rate statistics which should be followed on a sequence
// of frames. // of frames.
@ -146,13 +143,10 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsRateStatisticsByDefault) {
for (int frame = 0; frame != 10; ++frame) { for (int frame = 0; frame != 10; ++frame) {
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
absl::optional<int64_t> expected_fps =
rate.Rate(time_controller.GetClock()->TimeInMilliseconds());
rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds());
// FrameCadanceAdapter::OnFrame post the frame to another sequence. adapter->UpdateFrameRate();
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()),
time_controller.AdvanceTime(TimeDelta::Millis(0)); adapter->GetInputFrameRateFps())
EXPECT_EQ(expected_fps, adapter->GetInputFrameRateFps())
<< " failed for frame " << frame; << " failed for frame " << frame;
} }
} }
@ -162,8 +156,7 @@ TEST(FrameCadenceAdapterTest,
ZeroHertzFieldTrialDisabler feature_disabler; ZeroHertzFieldTrialDisabler feature_disabler;
GlobalSimulatedTimeController time_controller(Timestamp::Zero()); GlobalSimulatedTimeController time_controller(Timestamp::Zero());
auto adapter = CreateAdapter(feature_disabler, time_controller.GetClock()); auto adapter = CreateAdapter(feature_disabler, time_controller.GetClock());
MockCallback callback; adapter->Initialize(nullptr);
adapter->Initialize(&callback);
// Create an "oracle" rate statistics which should be followed on a sequence // Create an "oracle" rate statistics which should be followed on a sequence
// of frames. // of frames.
@ -172,14 +165,10 @@ TEST(FrameCadenceAdapterTest,
for (int frame = 0; frame != 10; ++frame) { for (int frame = 0; frame != 10; ++frame) {
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
absl::optional<int64_t> expected_fps =
rate.Rate(time_controller.GetClock()->TimeInMilliseconds());
rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds());
// FrameCadanceAdapter::OnFrame post the frame to another sequence. adapter->UpdateFrameRate();
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()),
time_controller.AdvanceTime(TimeDelta::Millis(0)); adapter->GetInputFrameRateFps())
EXPECT_EQ(adapter->GetInputFrameRateFps(), expected_fps)
<< " failed for frame " << frame; << " failed for frame " << frame;
} }
} }
@ -188,16 +177,13 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsMaxFpsWhenZeroHertzActivated) {
ZeroHertzFieldTrialEnabler enabler; ZeroHertzFieldTrialEnabler enabler;
GlobalSimulatedTimeController time_controller(Timestamp::Zero()); GlobalSimulatedTimeController time_controller(Timestamp::Zero());
auto adapter = CreateAdapter(enabler, time_controller.GetClock()); auto adapter = CreateAdapter(enabler, time_controller.GetClock());
MockCallback callback; adapter->Initialize(nullptr);
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled( adapter->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{}); FrameCadenceAdapterInterface::ZeroHertzModeParams{});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1});
for (int frame = 0; frame != 10; ++frame) { for (int frame = 0; frame != 10; ++frame) {
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
// FrameCadanceAdapter::OnFrame post the frame to another sequence. adapter->UpdateFrameRate();
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller));
time_controller.AdvanceTime(TimeDelta::Millis(0));
EXPECT_EQ(adapter->GetInputFrameRateFps(), 1u); EXPECT_EQ(adapter->GetInputFrameRateFps(), 1u);
} }
} }
@ -230,8 +216,7 @@ TEST(FrameCadenceAdapterTest,
ZeroHertzFieldTrialEnabler enabler; ZeroHertzFieldTrialEnabler enabler;
GlobalSimulatedTimeController time_controller(Timestamp::Zero()); GlobalSimulatedTimeController time_controller(Timestamp::Zero());
auto adapter = CreateAdapter(enabler, time_controller.GetClock()); auto adapter = CreateAdapter(enabler, time_controller.GetClock());
MockCallback callback; adapter->Initialize(nullptr);
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled( adapter->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{}); FrameCadenceAdapterInterface::ZeroHertzModeParams{});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1});
@ -241,16 +226,15 @@ TEST(FrameCadenceAdapterTest,
for (int frame = 0; frame != MAX; ++frame) { for (int frame = 0; frame != MAX; ++frame) {
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds());
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); adapter->UpdateFrameRate();
time_controller.AdvanceTime(TimeDelta::Millis(0));
} }
// Turn off zero hertz on the next-last frame; after the last frame we // Turn off zero hertz on the next-last frame; after the last frame we
// should see a value that tracks the rate oracle. // should see a value that tracks the rate oracle.
adapter->SetZeroHertzModeEnabled(absl::nullopt); adapter->SetZeroHertzModeEnabled(absl::nullopt);
// Last frame. // Last frame.
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds());
time_controller.AdvanceTime(TimeDelta::Millis(0)); adapter->UpdateFrameRate();
EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()), EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()),
adapter->GetInputFrameRateFps()); adapter->GetInputFrameRateFps());
@ -1054,24 +1038,6 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsTimeUntilFirstFrame) {
ElementsAre(Pair(666, 1))); ElementsAre(Pair(666, 1)));
} }
TEST_F(FrameCadenceAdapterMetricsTest,
RecordsFrameTimestampMonotonicallyIncreasing) {
MockCallback callback;
test::ScopedKeyValueConfig no_field_trials;
std::unique_ptr<FrameCadenceAdapterInterface> adapter =
CreateAdapter(no_field_trials, time_controller_.GetClock());
adapter->Initialize(&callback);
time_controller_.AdvanceTime(TimeDelta::Millis(666));
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller_));
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller_));
time_controller_.AdvanceTime(TimeDelta::Zero());
adapter = nullptr;
DepleteTaskQueues();
EXPECT_THAT(metrics::Samples(
"WebRTC.Video.InputFrameTimestampMonotonicallyIncreasing"),
ElementsAre(Pair(false, 1)));
}
TEST(FrameCadenceAdapterRealTimeTest, TimestampsDoNotDrift) { TEST(FrameCadenceAdapterRealTimeTest, TimestampsDoNotDrift) {
// This regression test must be performed in realtime because of limitations // This regression test must be performed in realtime because of limitations
// in GlobalSimulatedTimeController. // in GlobalSimulatedTimeController.

6
video/video_stream_encoder.cc
View File

@ -1784,7 +1784,13 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
// According to the testcase // According to the testcase
// InitialFrameDropOffWhenEncoderDisabledScaling, the return value // InitialFrameDropOffWhenEncoderDisabledScaling, the return value
// from GetScalingSettings should enable or disable the frame drop. // from GetScalingSettings should enable or disable the frame drop.
// Update input frame rate before we start using it. If we update it after
// any potential frame drop we are going to artificially increase frame sizes.
// Poll the rate before updating, otherwise we risk the rate being estimated
// a little too high at the start of the call when then window is small.
uint32_t framerate_fps = GetInputFramerateFps(); uint32_t framerate_fps = GetInputFramerateFps();
frame_cadence_adapter_->UpdateFrameRate();
int64_t now_ms = env_.clock().TimeInMilliseconds(); int64_t now_ms = env_.clock().TimeInMilliseconds();
if (pending_encoder_reconfiguration_) { if (pending_encoder_reconfiguration_) {

71
video/video_stream_encoder_unittest.cc
View File

@ -574,7 +574,7 @@ class AdaptingFrameForwarder : public test::FrameForwarder {
.set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
nullptr, out_width, out_height)) nullptr, out_width, out_height))
.set_ntp_time_ms(video_frame.ntp_time_ms()) .set_ntp_time_ms(video_frame.ntp_time_ms())
.set_timestamp_ms(video_frame.timestamp_us() * 1000) .set_timestamp_ms(99)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.build(); .build();
if (video_frame.has_update_rect()) { if (video_frame.has_update_rect()) {
@ -791,6 +791,7 @@ class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
(override)); (override));
MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override)); MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
MOCK_METHOD(void, UpdateFrameRate, (), (override));
MOCK_METHOD(void, MOCK_METHOD(void,
UpdateLayerQualityConvergence, UpdateLayerQualityConvergence,
(size_t spatial_index, bool converged), (size_t spatial_index, bool converged),
@ -951,7 +952,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
.set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
destruction_event, codec_width_, codec_height_)) destruction_event, codec_width_, codec_height_))
.set_ntp_time_ms(ntp_time_ms) .set_ntp_time_ms(ntp_time_ms)
.set_timestamp_ms(ntp_time_ms) .set_timestamp_ms(99)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.build(); .build();
} }
@ -963,7 +964,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
.set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
destruction_event, codec_width_, codec_height_)) destruction_event, codec_width_, codec_height_))
.set_ntp_time_ms(ntp_time_ms) .set_ntp_time_ms(ntp_time_ms)
.set_timestamp_ms(ntp_time_ms) .set_timestamp_ms(99)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1}) .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
.build(); .build();
@ -997,7 +998,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
.set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
destruction_event, width, height)) destruction_event, width, height))
.set_ntp_time_ms(ntp_time_ms) .set_ntp_time_ms(ntp_time_ms)
.set_timestamp_ms(ntp_time_ms) .set_timestamp_ms(99)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.build(); .build();
} }
@ -1010,7 +1011,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
.set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
destruction_event, width, height)) destruction_event, width, height))
.set_ntp_time_ms(ntp_time_ms) .set_ntp_time_ms(ntp_time_ms)
.set_timestamp_ms(ntp_time_ms) .set_timestamp_ms(99)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.build(); .build();
} }
@ -1532,7 +1533,6 @@ class VideoStreamEncoderTest : public ::testing::Test {
if (num_received_layers_ == num_expected_layers_) { if (num_received_layers_ == num_expected_layers_) {
encoded_frame_event_.Set(); encoded_frame_event_.Set();
} }
return Result(Result::OK, last_timestamp_); return Result(Result::OK, last_timestamp_);
} }
@ -1663,34 +1663,28 @@ TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
// The encoder will cache up to one frame for a short duration. Adding two // The encoder will cache up to one frame for a short duration. Adding two
// frames means that the first frame will be dropped and the second frame will // frames means that the first frame will be dropped and the second frame will
// be sent when the encoder is enabled. // be sent when the encoder is enabled.
const int64_t kFrame1TimestampMs = CurrentTimeMs(); video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
video_source_.IncomingCapturedFrame(
CreateFrame(kFrame1TimestampMs, &frame_destroyed_event));
AdvanceTime(TimeDelta::Millis(10)); AdvanceTime(TimeDelta::Millis(10));
const int64_t kFrame2TimestampMs = CurrentTimeMs(); video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
video_source_.IncomingCapturedFrame(CreateFrame(kFrame2TimestampMs, nullptr));
AdvanceTime(TimeDelta::Zero()); AdvanceTime(TimeDelta::Zero());
EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout)); EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
// The pending frame should be encoded. // The pending frame should be received.
WaitForEncodedFrame(kFrame2TimestampMs); WaitForEncodedFrame(2);
video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
const int64_t kFrame3TimestampMs = CurrentTimeMs(); WaitForEncodedFrame(3);
video_source_.IncomingCapturedFrame(CreateFrame(kFrame3TimestampMs, nullptr));
WaitForEncodedFrame(kFrame3TimestampMs);
video_stream_encoder_->Stop(); video_stream_encoder_->Stop();
} }
TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) { TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
int64_t time_ms = 123;
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr)); video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
WaitForEncodedFrame(time_ms); WaitForEncodedFrame(1);
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0); DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
@ -1698,17 +1692,14 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
// The encoder will cache up to one frame for a short duration. Adding two // The encoder will cache up to one frame for a short duration. Adding two
// frames means that the first frame will be dropped and the second frame will // frames means that the first frame will be dropped and the second frame will
// be sent when the encoder is resumed. // be sent when the encoder is resumed.
time_ms += 30; video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr)); video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
time_ms += 30;
video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr));
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
WaitForEncodedFrame(time_ms); WaitForEncodedFrame(3);
time_ms += 20; video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr)); WaitForEncodedFrame(4);
WaitForEncodedFrame(time_ms);
video_stream_encoder_->Stop(); video_stream_encoder_->Stop();
} }
@ -4716,24 +4707,20 @@ TEST_F(VideoStreamEncoderTest,
TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) { TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
const int kWidth = 640; const int kWidth = 640;
const int kHeight = 360; const int kHeight = 360;
int64_t ntp_timestamp_ms = 123;
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
video_source_.IncomingCapturedFrame( video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
CreateFrame(ntp_timestamp_ms, kWidth, kHeight)); WaitForEncodedFrame(i);
WaitForEncodedFrame(ntp_timestamp_ms);
ntp_timestamp_ms += 20;
} }
video_stream_encoder_->TriggerCpuOveruse(); video_stream_encoder_->TriggerCpuOveruse();
for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
video_source_.IncomingCapturedFrame( video_source_.IncomingCapturedFrame(CreateFrame(
CreateFrame(ntp_timestamp_ms, kWidth, kHeight)); SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
WaitForEncodedFrame(ntp_timestamp_ms); WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
ntp_timestamp_ms += 20;
} }
video_stream_encoder_->Stop(); video_stream_encoder_->Stop();
@ -4752,16 +4739,13 @@ TEST_F(VideoStreamEncoderTest,
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
const int kWidth = 640; const int kWidth = 640;
const int kHeight = 360; const int kHeight = 360;
int64_t ntp_timestamp_ms = 123;
video_stream_encoder_->SetSource(&video_source_, video_stream_encoder_->SetSource(&video_source_,
webrtc::DegradationPreference::DISABLED); webrtc::DegradationPreference::DISABLED);
for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
video_source_.IncomingCapturedFrame( video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
CreateFrame(ntp_timestamp_ms, kWidth, kHeight)); WaitForEncodedFrame(i);
WaitForEncodedFrame(ntp_timestamp_ms);
ntp_timestamp_ms += 20;
} }
video_stream_encoder_->Stop(); video_stream_encoder_->Stop();
@ -5532,7 +5516,7 @@ TEST_F(VideoStreamEncoderTest,
// Reconfigure the encoder with a new (higher max framerate), max fps should // Reconfigure the encoder with a new (higher max framerate), max fps should
// still respect the adaptation. // still respect the adaptation.
video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate; video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
source.IncomingCapturedFrame(CreateFrame(2, kFrameWidth, kFrameHeight)); source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
kMaxPayloadLength); kMaxPayloadLength);
video_stream_encoder_->WaitUntilTaskQueueIsIdle(); video_stream_encoder_->WaitUntilTaskQueueIsIdle();
@ -9529,6 +9513,7 @@ TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
/*max_data_payload_length=*/1000); /*max_data_payload_length=*/1000);
EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps); EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1); PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
factory.DepleteTaskQueues(); factory.DepleteTaskQueues();
} }