admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FrameCadenceAdapter: improves performance under repeat and high load

Browse Source 
This change ensures that the FCA now is informed about a new max fps
when VideoStreamEncoder::OnVideoSourceRestrictionsUpdated is called.

The latest restricted frame rate which is provided to the FCA will
only affect the cadence of repeated non-idle (quality has not
converged) frames and the main goal is to ensure that the FCA reduces
its repeat rate in situations where the video source is constrained.

UpdateVideoSourceRestrictions is added to the FrameCadenceAdapter API
and it is called from the VideoStreamEncoder when its source
parameters (resolution and/or frame rate) are restricted.

This modification has no effect on the flow driven by
ProcessOnDelayedCadence (non repeated frames).

Bug: webrtc:15539
Change-Id: I26dee6480e5137f82c5ccf57091b737cad82dbf6
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/328300
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Markus Handell <handellm@webrtc.org>
Commit-Queue: Henrik Andreassson <henrika@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41308}
This commit is contained in:
henrika 2023-12-04 14:03:52 +01:00 committed by WebRTC LUCI CQ
parent c925f50c1c
commit 8f16ce98c2
6 changed files with 223 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
video/frame_cadence_adapter.cc
View File

@ -131,6 +131,10 @@ class ZeroHertzAdapterMode : public AdapterMode {
// Callback::RequestRefreshFrame.
void ProcessKeyFrameRequest();
// Updates the restrictions of max frame rate for the video source.
// Always called during construction using latest `restricted_frame_delay_`.
void UpdateVideoSourceRestrictions(absl::optional<double> max_frame_rate);
private:
// The tracking state of each spatial layer. Used for determining when to
// stop repeating frames.
@ -173,10 +177,11 @@ class ZeroHertzAdapterMode : public AdapterMode {
// Processes incoming frames on a delayed cadence.
void ProcessOnDelayedCadence(Timestamp post_time)
RTC_RUN_ON(sequence_checker_);
// Schedules a later repeat with delay depending on state of layer trackers.
// Schedules a later repeat with delay depending on state of layer trackers
// and if UpdateVideoSourceRestrictions has been called or not.
// If true is passed in `idle_repeat`, the repeat is going to be
// kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the value of
// `frame_delay`.
// kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the maximum value of
// `frame_delay` or `restricted_frame_delay_` if it has been set.
void ScheduleRepeat(int frame_id, bool idle_repeat)
RTC_RUN_ON(sequence_checker_);
// Repeats a frame in the absence of incoming frames. Slows down when quality
@ -221,6 +226,10 @@ class ZeroHertzAdapterMode : public AdapterMode {
// they can be dropped in various places in the capture pipeline.
RepeatingTaskHandle refresh_frame_requester_
RTC_GUARDED_BY(sequence_checker_);
// Can be set by UpdateVideoSourceRestrictions when the video source restricts
// the max frame rate.
absl::optional<TimeDelta> restricted_frame_delay_
RTC_GUARDED_BY(sequence_checker_);
ScopedTaskSafety safety_;
};
@ -241,6 +250,8 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
void UpdateLayerQualityConvergence(size_t spatial_index,
bool quality_converged) override;
void UpdateLayerStatus(size_t spatial_index, bool enabled) override;
void UpdateVideoSourceRestrictions(
absl::optional<double> max_frame_rate) override;
void ProcessKeyFrameRequest() override;
// VideoFrameSink overrides.
@ -292,6 +303,11 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
absl::optional<VideoTrackSourceConstraints> source_constraints_
RTC_GUARDED_BY(queue_);
// Stores the latest restriction in max frame rate set by
// UpdateVideoSourceRestrictions. Ensures that a previously set restriction
// can be maintained during reconstructions of the adapter.
absl::optional<double> restricted_max_frame_rate_ RTC_GUARDED_BY(queue_);
// Race checker for incoming frames. This is the network thread in chromium,
// but may vary from test contexts.
rtc::RaceChecker incoming_frame_race_checker_;
@ -412,6 +428,20 @@ absl::optional<uint32_t> ZeroHertzAdapterMode::GetInputFrameRateFps() {
return max_fps_;
}
void ZeroHertzAdapterMode::UpdateVideoSourceRestrictions(
absl::optional<double> max_frame_rate) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__,
"max_frame_rate", max_frame_rate.value_or(-1));
if (max_frame_rate.value_or(0) > 0) {
// Set new, validated (> 0) and restricted frame rate.
restricted_frame_delay_ = TimeDelta::Seconds(1) / *max_frame_rate;
} else {
// Source reports that the frame rate is now unrestricted.
restricted_frame_delay_ = absl::nullopt;
}
}
void ZeroHertzAdapterMode::ProcessKeyFrameRequest() {
RTC_DCHECK_RUN_ON(&sequence_checker_);
TRACE_EVENT_INSTANT0("webrtc", __func__);
@ -574,9 +604,14 @@ void ZeroHertzAdapterMode::SendFrameNow(Timestamp post_time,
TimeDelta ZeroHertzAdapterMode::RepeatDuration(bool idle_repeat) const {
RTC_DCHECK_RUN_ON(&sequence_checker_);
// By default use `frame_delay_` in non-idle repeat mode but use the
// restricted frame delay instead if it is set in
// UpdateVideoSourceRestrictions.
TimeDelta frame_delay =
std::max(frame_delay_, restricted_frame_delay_.value_or(frame_delay_));
return idle_repeat
? FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod
: frame_delay_;
: frame_delay;
}
void ZeroHertzAdapterMode::MaybeStartRefreshFrameRequester() {
@ -649,6 +684,17 @@ void FrameCadenceAdapterImpl::UpdateLayerStatus(size_t spatial_index,
zero_hertz_adapter_->UpdateLayerStatus(spatial_index, enabled);
}
void FrameCadenceAdapterImpl::UpdateVideoSourceRestrictions(
absl::optional<double> max_frame_rate) {
RTC_DCHECK_RUN_ON(queue_);
// Store the restriction to ensure that it can be reapplied in possible
// future adapter creations on configuration changes.
restricted_max_frame_rate_ = max_frame_rate;
if (zero_hertz_adapter_) {
zero_hertz_adapter_->UpdateVideoSourceRestrictions(max_frame_rate);
}
}
void FrameCadenceAdapterImpl::ProcessKeyFrameRequest() {
RTC_DCHECK_RUN_ON(queue_);
if (zero_hertz_adapter_)
@ -742,11 +788,12 @@ void FrameCadenceAdapterImpl::MaybeReconfigureAdapters(
bool max_fps_has_changed = GetInputFrameRateFps().value_or(-1) !=
source_constraints_->max_fps.value_or(-1);
if (!was_zero_hertz_enabled || max_fps_has_changed) {
zero_hertz_adapter_.emplace(queue_, clock_, callback_,
source_constraints_->max_fps.value());
zero_hertz_adapter_is_active_.store(true, std::memory_order_relaxed);
RTC_LOG(LS_INFO) << "Zero hertz mode enabled (max_fps="
<< source_constraints_->max_fps.value() << ")";
zero_hertz_adapter_.emplace(queue_, clock_, callback_,
source_constraints_->max_fps.value());
zero_hertz_adapter_->UpdateVideoSourceRestrictions(
restricted_max_frame_rate_);
zero_hertz_adapter_created_timestamp_ = clock_->CurrentTime();
}
zero_hertz_adapter_->ReconfigureParameters(zero_hertz_params_.value());

6
video/frame_cadence_adapter.h
View File

@ -109,6 +109,12 @@ class FrameCadenceAdapterInterface
// Updates spatial layer enabled status.
virtual void UpdateLayerStatus(size_t spatial_index, bool enabled) = 0;
// Updates the restrictions of max frame rate for the video source.
// The new `max_frame_rate` will only affect the cadence of Callback::OnFrame
// for non-idle (non converged) repeated frames.
virtual void UpdateVideoSourceRestrictions(
absl::optional<double> max_frame_rate) = 0;
// Conditionally requests a refresh frame via
// Callback::RequestRefreshFrame.
virtual void ProcessKeyFrameRequest() = 0;

96
video/frame_cadence_adapter_unittest.cc
View File

@ -731,6 +731,8 @@ class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test {
static constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(100);
static constexpr TimeDelta kIdleFrameDelay =
FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod;
// Restricts non-idle repeat rate to 5 fps (default is 10 fps);
static constexpr int kRestrictedMaxFps = 5;
ZeroHertzLayerQualityConvergenceTest() {
adapter_->Initialize(&callback_);
@ -833,6 +835,100 @@ TEST_F(ZeroHertzLayerQualityConvergenceTest,
});
}
TEST_F(ZeroHertzLayerQualityConvergenceTest,
UnconvergedRepeatRateAdaptsDownWhenRestricted) {
PassFrame();
ScheduleDelayed(1.5 * kMinFrameDelay, [&] {
adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps);
});
ExpectFrameEntriesAtDelaysFromNow({
1 * kMinFrameDelay, // Original frame emitted at non-restricted rate.
// 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result
// in a new non-idle repeat delay of 2 * kMinFrameDelay.
2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate.
4 * kMinFrameDelay, // Unconverged repeats at restricted rate. This
// happens 2 * kMinFrameDelay after the last frame.
6 * kMinFrameDelay, // ...
});
}
TEST_F(ZeroHertzLayerQualityConvergenceTest,
UnconvergedRepeatRateAdaptsUpWhenGoingFromRestrictedToUnrestricted) {
PassFrame();
ScheduleDelayed(1.5 * kMinFrameDelay, [&] {
adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps);
});
ScheduleDelayed(5.5 * kMinFrameDelay, [&] {
adapter_->UpdateVideoSourceRestrictions(absl::nullopt);
});
ExpectFrameEntriesAtDelaysFromNow({
1 * kMinFrameDelay, // Original frame emitted at non-restricted rate.
// 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result
// in a new non-idle repeat delay of 2 * kMinFrameDelay.
2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate.
4 * kMinFrameDelay, // Unconverged repeat at restricted rate.
// 5.5 * kMinFrameDelay: removes frame-rate restriction and we should
// then go back to 10 fps as unconverged repeat rate.
6 * kMinFrameDelay, // Last unconverged repeat at restricted rate.
7 * kMinFrameDelay, // Back to unconverged repeat at non-restricted rate.
8 * kMinFrameDelay, // We are now unrestricted.
9 * kMinFrameDelay, // ...
});
}
TEST_F(ZeroHertzLayerQualityConvergenceTest,
UnconvergedRepeatRateMaintainsRestrictionOnReconfigureToHigherMaxFps) {
PassFrame();
ScheduleDelayed(1.5 * kMinFrameDelay, [&] {
adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps);
});
ScheduleDelayed(2.5 * kMinFrameDelay, [&] {
adapter_->OnConstraintsChanged(VideoTrackSourceConstraints{
/*min_fps=*/0, /*max_fps=*/2 * TimeDelta::Seconds(1) / kMinFrameDelay});
});
ScheduleDelayed(3 * kMinFrameDelay, [&] { PassFrame(); });
ScheduleDelayed(8 * kMinFrameDelay, [&] {
adapter_->OnConstraintsChanged(VideoTrackSourceConstraints{
/*min_fps=*/0,
/*max_fps=*/0.2 * TimeDelta::Seconds(1) / kMinFrameDelay});
});
ScheduleDelayed(9 * kMinFrameDelay, [&] { PassFrame(); });
ExpectFrameEntriesAtDelaysFromNow({
1 * kMinFrameDelay, // Original frame emitted at non-restricted rate.
// 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result
// in a new non-idle repeat delay of 2 * kMinFrameDelay.
2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate.
// 2.5 * kMinFrameDelay: new constraint asks for max rate of 20 fps.
// The 0Hz adapter is reconstructed for 20 fps but inherits the current
// restriction for rate of non-converged frames of 5 fps.
// A new frame is passed at 3 * kMinFrameDelay. The previous repeat
// cadence was stopped by the change in constraints.
3.5 * kMinFrameDelay, // Original frame emitted at non-restricted 20 fps.
// The delay is 0.5 * kMinFrameDelay.
5.5 * kMinFrameDelay, // Unconverged repeat at restricted rate.
// The delay is 2 * kMinFrameDelay when restricted.
7.5 * kMinFrameDelay, // ...
// 8 * kMinFrameDelay: new constraint asks for max rate of 2 fps.
// The 0Hz adapter is reconstructed for 2 fps and will therefore not obey
// the current restriction for rate of non-converged frames of 5 fps
// since the new max rate is lower.
// A new frame is passed at 9 * kMinFrameDelay. The previous repeat
// cadence was stopped by the change in constraints.
14 * kMinFrameDelay, // Original frame emitted at non-restricted 2 fps.
// The delay is 5 * kMinFrameDelay.
19 * kMinFrameDelay, // Unconverged repeat at non-restricted rate.
24 * kMinFrameDelay, // ...
});
}
class FrameCadenceAdapterMetricsTest : public ::testing::Test {
public:
FrameCadenceAdapterMetricsTest() : time_controller_(Timestamp::Millis(1)) {

5
video/video_stream_encoder.cc
View File

@ -2358,6 +2358,11 @@ void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
<< (reason ? reason->Name() : std::string("<null>"))
<< " to " << restrictions.ToString();
if (frame_cadence_adapter_) {
frame_cadence_adapter_->UpdateVideoSourceRestrictions(
restrictions.max_frame_rate());
}
// TODO(webrtc:14451) Split video_source_sink_controller_
// so that ownership on restrictions/wants is kept on &encoder_queue_
latest_restrictions_ = restrictions;

2
video/video_stream_encoder.h
View File

@ -132,6 +132,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
double cwnd_reduce_ratio);
protected:
friend class VideoStreamEncoderFrameCadenceRestrictionTest;
// Used for testing. For example the `ScalingObserverInterface` methods must
// be called on `encoder_queue_`.
TaskQueueBase* encoder_queue() { return encoder_queue_.Get(); }

60
video/video_stream_encoder_unittest.cc
View File

@ -783,6 +783,10 @@ class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
UpdateLayerStatus,
(size_t spatial_index, bool enabled),
(override));
MOCK_METHOD(void,
UpdateVideoSourceRestrictions,
(absl::optional<double>),
(override));
MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
};
@ -9580,4 +9584,60 @@ TEST(VideoStreamEncoderFrameCadenceTest,
kMaxFps);
}
class VideoStreamEncoderFrameCadenceRestrictionTest : public ::testing::Test {
public:
VideoStreamEncoderFrameCadenceRestrictionTest()
: adapter_ptr_(adapter_.get()),
fake_resource_(FakeResource::Create("FakeResource")),
video_stream_encoder_(
factory_.Create(std::move(adapter_), &encoder_queue_)) {}
~VideoStreamEncoderFrameCadenceRestrictionTest() {
factory_.DepleteTaskQueues();
}
void UpdateVideoSourceRestrictions(VideoSourceRestrictions restrictions) {
encoder_queue_->PostTask([this, restrictions] {
RTC_DCHECK_RUN_ON(encoder_queue_);
video_stream_encoder_->OnVideoSourceRestrictionsUpdated(
restrictions, VideoAdaptationCounters(), fake_resource_,
VideoSourceRestrictions());
});
}
protected:
SimpleVideoStreamEncoderFactory factory_;
std::unique_ptr<NiceMock<MockFrameCadenceAdapter>> adapter_{
std::make_unique<NiceMock<MockFrameCadenceAdapter>>()};
NiceMock<MockFrameCadenceAdapter>* adapter_ptr_;
TaskQueueBase* encoder_queue_{nullptr};
rtc::scoped_refptr<FakeResource> fake_resource_;
VideoSourceRestrictions restrictions_;
std::unique_ptr<SimpleVideoStreamEncoderFactory::AdaptedVideoStreamEncoder>
video_stream_encoder_;
};
TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest,
UpdatesVideoSourceRestrictionsUnRestricted) {
EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Eq(absl::nullopt)));
UpdateVideoSourceRestrictions(VideoSourceRestrictions());
}
TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest,
UpdatesVideoSourceRestrictionsWithMaxFrameRateRestriction) {
restrictions_.set_max_frame_rate(20);
EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Optional(20)));
UpdateVideoSourceRestrictions(restrictions_);
}
TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest,
UpdatesVideoSourceRestrictionsWithoutMaxFrameRateRestriction) {
// Restrictions in resolution count as restriction updated, even though the
// FPS is unlimited.
restrictions_.set_max_pixels_per_frame(99);
restrictions_.set_target_pixels_per_frame(101);
EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Eq(absl::nullopt)));
UpdateVideoSourceRestrictions(restrictions_);
}
} // namespace webrtc