From cb237f88224cb5fc9c011cda6d461a490a38681d Mon Sep 17 00:00:00 2001 From: Markus Handell Date: Wed, 29 Dec 2021 21:31:57 +0100 Subject: [PATCH] ZeroHertzAdapterMode: reset convergence info on key frame requests. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The QP value of encoded key frames is normally very large. However, the zero-hertz mode is retaining quality convergence info, leading to only a single short repeat on key frame request when idle repeating. Fix this by resetting quality convergence information on key frame requests, ensuring zero-hertz mode goes back to idle repeating only when quality has converged again. go/rtc-0hz-present Bug: chromium:1255737 Change-Id: Ia1ad686cc98007f01c8aaef9162011837575938c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/242862 Reviewed-by: Erik Språng Commit-Queue: Markus Handell Cr-Commit-Position: refs/heads/main@{#35594} --- video/frame_cadence_adapter.cc | 10 ++ video/frame_cadence_adapter_unittest.cc | 193 +++++++++++++----------- 2 files changed, 114 insertions(+), 89 deletions(-) diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc index 54cd5b2994..1a10484aec 100644 --- a/video/frame_cadence_adapter.cc +++ b/video/frame_cadence_adapter.cc @@ -367,6 +367,11 @@ bool ZeroHertzAdapterMode::ProcessKeyFrameRequest() { return true; } + // The next frame encoded will be a key frame. Reset quality convergence so we + // don't get idle repeats shortly after, because key frames need a lot of + // refinement frames. + ResetQualityConvergenceInfo(); + // If we're not repeating, or we're repeating with non-idle duration, we will // very soon send out a frame and don't need a refresh frame. if (!scheduled_repeat_.has_value() || !scheduled_repeat_->idle) { @@ -399,7 +404,12 @@ bool ZeroHertzAdapterMode::ProcessKeyFrameRequest() { // RTC_RUN_ON(&sequence_checker_) bool ZeroHertzAdapterMode::HasQualityConverged() const { + // 1. Define ourselves as unconverged with no spatial layers configured. This + // is to keep short repeating until the layer configuration comes. + // 2. Unset layers implicitly imply that they're converged to support + // disabling layers when they're not needed. const bool quality_converged = + !layer_trackers_.empty() && absl::c_all_of(layer_trackers_, [](const SpatialLayerTracker& tracker) { return tracker.quality_converged.value_or(true); }); diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc index a790c4ff46..7e3b3fd2ea 100644 --- a/video/frame_cadence_adapter_unittest.cc +++ b/video/frame_cadence_adapter_unittest.cc @@ -37,6 +37,7 @@ using ::testing::ElementsAre; using ::testing::Invoke; using ::testing::Mock; using ::testing::Pair; +using ::testing::Values; VideoFrame CreateFrame() { return VideoFrame::Builder() @@ -379,105 +380,119 @@ TEST(FrameCadenceAdapterTest, IgnoresKeyFrameRequestShortlyAfterFrame) { EXPECT_FALSE(adapter->ProcessKeyFrameRequest()); } -TEST(FrameCadenceAdapterTest, IgnoresKeyFrameRequestWhileShortRepeating) { - ZeroHertzFieldTrialEnabler enabler; - MockCallback callback; - GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); - auto adapter = CreateAdapter(time_controller.GetClock()); - adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled( - FrameCadenceAdapterInterface::ZeroHertzModeParams{ - /*num_simulcast_layers=*/1}); - constexpr int kMaxFpsHz = 10; - constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(1000 / kMaxFpsHz); - adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFpsHz}); - time_controller.AdvanceTime(TimeDelta::Zero()); - adapter->UpdateLayerStatus(0, true); - adapter->OnFrame(CreateFrame()); - time_controller.AdvanceTime(2 * kMinFrameDelay); - EXPECT_FALSE(adapter->ProcessKeyFrameRequest()); +class FrameCadenceAdapterSimulcastLayersParamTest + : public ::testing::TestWithParam { + public: + static constexpr int kMaxFpsHz = 8; + static constexpr TimeDelta kMinFrameDelay = + TimeDelta::Millis(1000 / kMaxFpsHz); + static constexpr TimeDelta kIdleFrameDelay = + FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod; - // Expect repeating as ususal. - EXPECT_CALL(callback, OnFrame).Times(8); - time_controller.AdvanceTime(8 * kMinFrameDelay); + FrameCadenceAdapterSimulcastLayersParamTest() { + adapter_->Initialize(&callback_); + adapter_->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFpsHz}); + time_controller_.AdvanceTime(TimeDelta::Zero()); + adapter_->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); + const int num_spatial_layers = GetParam(); + adapter_->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{num_spatial_layers}); + } + + int NumSpatialLayers() const { return GetParam(); } + + protected: + ZeroHertzFieldTrialEnabler enabler_; + MockCallback callback_; + GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)}; + const std::unique_ptr adapter_{ + CreateAdapter(time_controller_.GetClock())}; +}; + +TEST_P(FrameCadenceAdapterSimulcastLayersParamTest, + LayerReconfigurationResetsConvergenceInfo) { + // Assumes layer reconfiguration has just happened. + // Verify the state is unconverged. + adapter_->OnFrame(CreateFrame()); + EXPECT_CALL(callback_, OnFrame).Times(kMaxFpsHz); + time_controller_.AdvanceTime(kMaxFpsHz * kMinFrameDelay); } -TEST(FrameCadenceAdapterTest, IgnoresKeyFrameRequestJustBeforeIdleRepeating) { - ZeroHertzFieldTrialEnabler enabler; - MockCallback callback; - GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); - auto adapter = CreateAdapter(time_controller.GetClock()); - adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled( - FrameCadenceAdapterInterface::ZeroHertzModeParams{}); - constexpr int kMaxFpsHz = 10; - constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(1000 / kMaxFpsHz); - constexpr TimeDelta kIdleFrameDelay = - FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod; - adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFpsHz}); - time_controller.AdvanceTime(TimeDelta::Zero()); - adapter->OnFrame(CreateFrame()); - time_controller.AdvanceTime(kIdleFrameDelay); +TEST_P(FrameCadenceAdapterSimulcastLayersParamTest, + IgnoresKeyFrameRequestWhileShortRepeating) { + // Plot: + // 1. 0 * kMinFrameDelay: Start unconverged. Frame -> adapter. + // 2. 1 * kMinFrameDelay: Frame -> callback. + // 3. 2 * kMinFrameDelay: 1st short repeat. + // Since we're unconverged we assume the process continues. + adapter_->OnFrame(CreateFrame()); + time_controller_.AdvanceTime(2 * kMinFrameDelay); + EXPECT_FALSE(adapter_->ProcessKeyFrameRequest()); + + // Expect short repeating as ususal. + EXPECT_CALL(callback_, OnFrame).Times(8); + time_controller_.AdvanceTime(8 * kMinFrameDelay); +} + +TEST_P(FrameCadenceAdapterSimulcastLayersParamTest, + IgnoresKeyFrameRequestJustBeforeIdleRepeating) { + // (Only for > 0 spatial layers as we assume not converged with 0 layers) + if (NumSpatialLayers() == 0) + return; + + // Plot: + // 1. 0 * kMinFrameDelay: Start converged. Frame -> adapter. + // 2. 1 * kMinFrameDelay: Frame -> callback. New repeat scheduled at + // (kMaxFpsHz + 1) * kMinFrameDelay. + // 3. kMaxFpsHz * kMinFrameDelay: Process keyframe. + // 4. (kMaxFpsHz + N) * kMinFrameDelay (1 <= N <= kMaxFpsHz): Short repeats + // due to not converged. + for (int i = 0; i != NumSpatialLayers(); i++) { + adapter_->UpdateLayerStatus(i, /*enabled=*/true); + adapter_->UpdateLayerQualityConvergence(i, /*converged=*/true); + } + adapter_->OnFrame(CreateFrame()); + time_controller_.AdvanceTime(kIdleFrameDelay); // We process the key frame request kMinFrameDelay before the first idle - // repeat should happen. The repeat should happen at T = kMinFrameDelay + - // kIdleFrameDelay as originally expected. - EXPECT_FALSE(adapter->ProcessKeyFrameRequest()); - EXPECT_CALL(callback, OnFrame); - time_controller.AdvanceTime(kMinFrameDelay); - EXPECT_CALL(callback, OnFrame); - time_controller.AdvanceTime(kIdleFrameDelay); + // repeat should happen. The resulting repeats should happen spaced by + // kMinFrameDelay before we get new convergence info. + EXPECT_FALSE(adapter_->ProcessKeyFrameRequest()); + EXPECT_CALL(callback_, OnFrame).Times(kMaxFpsHz); + time_controller_.AdvanceTime(kMaxFpsHz * kMinFrameDelay); } -TEST(FrameCadenceAdapterTest, - IgnoresKeyFrameRequestShortRepeatsBeforeIdleRepeat) { - ZeroHertzFieldTrialEnabler enabler; - MockCallback callback; - GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); - auto adapter = CreateAdapter(time_controller.GetClock()); - adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled( - FrameCadenceAdapterInterface::ZeroHertzModeParams{}); - constexpr int kMaxFpsHz = 10; - constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(1000 / kMaxFpsHz); - constexpr TimeDelta kIdleFrameDelay = - FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod; - adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFpsHz}); - time_controller.AdvanceTime(TimeDelta::Zero()); - adapter->OnFrame(CreateFrame()); - time_controller.AdvanceTime(2 * kMinFrameDelay); +TEST_P(FrameCadenceAdapterSimulcastLayersParamTest, + IgnoresKeyFrameRequestShortRepeatsBeforeIdleRepeat) { + // (Only for > 0 spatial layers as we assume not converged with 0 layers) + if (NumSpatialLayers() == 0) + return; + // Plot: + // 1. 0 * kMinFrameDelay: Start converged. Frame -> adapter. + // 2. 1 * kMinFrameDelay: Frame -> callback. New repeat scheduled at + // (kMaxFpsHz + 1) * kMinFrameDelay. + // 3. 2 * kMinFrameDelay: Process keyframe. + // 4. (2 + N) * kMinFrameDelay (1 <= N <= kMaxFpsHz): Short repeats due to not + // converged. + for (int i = 0; i != NumSpatialLayers(); i++) { + adapter_->UpdateLayerStatus(i, /*enabled=*/true); + adapter_->UpdateLayerQualityConvergence(i, /*converged=*/true); + } + adapter_->OnFrame(CreateFrame()); + time_controller_.AdvanceTime(2 * kMinFrameDelay); - // We process the key frame request 9 * kMinFrameDelay before the first idle - // repeat should happen. We should get a short repeat in kMinFrameDelay and an - // idle repeat after that. - EXPECT_FALSE(adapter->ProcessKeyFrameRequest()); - EXPECT_CALL(callback, OnFrame); - time_controller.AdvanceTime(kMinFrameDelay); - EXPECT_CALL(callback, OnFrame); - time_controller.AdvanceTime(kIdleFrameDelay); + // We process the key frame request (kMaxFpsHz - 1) * kMinFrameDelay before + // the first idle repeat should happen. The resulting repeats should happen + // spaced kMinFrameDelay before we get new convergence info. + EXPECT_FALSE(adapter_->ProcessKeyFrameRequest()); + EXPECT_CALL(callback_, OnFrame).Times(kMaxFpsHz); + time_controller_.AdvanceTime(kMaxFpsHz * kMinFrameDelay); } -TEST(FrameCadenceAdapterTest, LayerReconfigurationResetsConvergenceInfo) { - ZeroHertzFieldTrialEnabler enabler; - MockCallback callback; - GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); - auto adapter = CreateAdapter(time_controller.GetClock()); - adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled( - FrameCadenceAdapterInterface::ZeroHertzModeParams{}); - constexpr int kMaxFpsHz = 10; - constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(1000 / kMaxFpsHz); - adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFpsHz}); - time_controller.AdvanceTime(TimeDelta::Zero()); - - // Now setup 2 simulcast layers. The state should be unconverged. - adapter->SetZeroHertzModeEnabled( - FrameCadenceAdapterInterface::ZeroHertzModeParams{ - /*num_simulcast_layers=*/2}); - adapter->OnFrame(CreateFrame()); - EXPECT_CALL(callback, OnFrame).Times(kMaxFpsHz); - time_controller.AdvanceTime(kMaxFpsHz * kMinFrameDelay); -} +INSTANTIATE_TEST_SUITE_P(, + FrameCadenceAdapterSimulcastLayersParamTest, + Values(0, 1, 2)); class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test { public: