diff --git a/api/video/encoded_image.h b/api/video/encoded_image.h index 987645b569..88df34916b 100644 --- a/api/video/encoded_image.h +++ b/api/video/encoded_image.h @@ -154,6 +154,16 @@ class RTC_EXPORT EncodedImage { return encoded_data_ ? encoded_data_->data() : nullptr; } + // Returns whether the encoded image can be considered to be of target + // quality. + bool IsAtTargetQuality() const { return at_target_quality_; } + + // Sets that the encoded image can be considered to be of target quality to + // true or false. + void SetAtTargetQuality(bool at_target_quality) { + at_target_quality_ = at_target_quality; + } + uint32_t _encodedWidth = 0; uint32_t _encodedHeight = 0; // NTP time of the capture time in local timebase in milliseconds. @@ -200,6 +210,8 @@ class RTC_EXPORT EncodedImage { // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources RtpPacketInfos packet_infos_; bool retransmission_allowed_ = true; + // True if the encoded image can be considered to be of target quality. + bool at_target_quality_ = false; }; } // namespace webrtc diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 8661725679..17220b14dd 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -1184,6 +1184,8 @@ int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image, libvpx_->codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER, &qp_128); encoded_images_[encoder_idx].qp_ = qp_128; + encoded_images_[encoder_idx].SetAtTargetQuality( + qp_128 <= variable_framerate_experiment_.steady_state_qp); encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx], &codec_specific); const size_t steady_state_size = SteadyStateSize( diff --git a/video/BUILD.gn b/video/BUILD.gn index f4d9668861..c1e7292675 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -268,6 +268,7 @@ rtc_library("frame_cadence_adapter") { "../api/task_queue", "../api/units:time_delta", "../api/video:video_frame", + "../rtc_base:checks", "../rtc_base:logging", "../rtc_base:macromagic", "../rtc_base:rtc_base_approved", @@ -279,6 +280,7 @@ rtc_library("frame_cadence_adapter") { "../system_wrappers:field_trial", "../system_wrappers:metrics", ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] } rtc_library("video_stream_encoder_impl") { diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc index 3b961c8107..18b99f5383 100644 --- a/video/frame_cadence_adapter.cc +++ b/video/frame_cadence_adapter.cc @@ -14,11 +14,14 @@ #include #include #include +#include +#include "absl/algorithm/container.h" #include "api/sequence_checker.h" #include "api/task_queue/task_queue_base.h" #include "api/units/time_delta.h" #include "api/video/video_frame.h" +#include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/race_checker.h" #include "rtc_base/rate_statistics.h" @@ -31,6 +34,7 @@ #include "system_wrappers/include/clock.h" #include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/ntp_time.h" namespace webrtc { namespace { @@ -91,10 +95,18 @@ class PassthroughAdapterMode : public AdapterMode { // Implements a frame cadence adapter supporting zero-hertz input. class ZeroHertzAdapterMode : public AdapterMode { public: - ZeroHertzAdapterMode(TaskQueueBase* queue, - Clock* clock, - FrameCadenceAdapterInterface::Callback* callback, - double max_fps); + ZeroHertzAdapterMode( + TaskQueueBase* queue, + Clock* clock, + FrameCadenceAdapterInterface::Callback* callback, + double max_fps, + FrameCadenceAdapterInterface::ZeroHertzModeParams params); + + // Updates spatial layer quality convergence status. + void UpdateLayerQualityConvergence(int spatial_index, bool quality_converged); + + // Updates spatial layer enabled status. + void UpdateLayerStatus(int spatial_index, bool enabled); // Adapter overrides. void OnFrame(Timestamp post_time, @@ -104,9 +116,19 @@ class ZeroHertzAdapterMode : public AdapterMode { void UpdateFrameRate() override {} private: + // The tracking state of each spatial layer. Used for determining when to + // stop repeating frames. + struct SpatialLayerTracker { + // If unset, the layer is disabled. Otherwise carries the quality + // convergence status of the layer. + absl::optional quality_converged; + }; + // Processes incoming frames on a delayed cadence. void ProcessOnDelayedCadence() RTC_RUN_ON(sequence_checker_); - // Repeats a frame in the abscence of incoming frames. Slows down when QP + // Schedules a later repeat with delay depending on state of layer trackers. + void ScheduleRepeat(int frame_id) RTC_RUN_ON(sequence_checker_); + // Repeats a frame in the abscence of incoming frames. Slows down when quality // convergence is attained, and stops the cadence terminally when new frames // have arrived. `scheduled_delay` specifies the delay by which to modify the // repeate frame's timestamps when it's sent. @@ -133,6 +155,9 @@ class ZeroHertzAdapterMode : public AdapterMode { int current_frame_id_ RTC_GUARDED_BY(sequence_checker_) = 0; // True when we are repeating frames. bool is_repeating_ RTC_GUARDED_BY(sequence_checker_) = false; + // Convergent state of each of the configured simulcast layers. + std::vector layer_trackers_ + RTC_GUARDED_BY(sequence_checker_); ScopedTaskSafety safety_; }; @@ -143,9 +168,13 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { // FrameCadenceAdapterInterface overrides. void Initialize(Callback* callback) override; - void SetZeroHertzModeEnabled(bool enabled) override; + void SetZeroHertzModeEnabled( + absl::optional params) override; absl::optional GetInputFrameRateFps() override; void UpdateFrameRate() override; + void UpdateLayerQualityConvergence(int spatial_index, + bool quality_converged) override; + void UpdateLayerStatus(int spatial_index, bool enabled) override; // VideoFrameSink overrides. void OnFrame(const VideoFrame& frame) override; @@ -182,6 +211,8 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { // The two possible modes we're under. absl::optional passthrough_adapter_; absl::optional zero_hertz_adapter_; + // If set, zero-hertz mode has been enabled. + absl::optional zero_hertz_params_; // Cache for the current adapter mode. AdapterMode* current_adapter_mode_ = nullptr; @@ -192,16 +223,13 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { absl::optional source_constraints_ RTC_GUARDED_BY(queue_); - // Whether zero-hertz and UMA reporting is enabled. - bool zero_hertz_and_uma_reporting_enabled_ RTC_GUARDED_BY(queue_) = false; - // Race checker for incoming frames. This is the network thread in chromium, // but may vary from test contexts. rtc::RaceChecker incoming_frame_race_checker_; bool has_reported_screenshare_frame_rate_umas_ RTC_GUARDED_BY(queue_) = false; // Number of frames that are currently scheduled for processing on the - // |queue_|. + // `queue_`. std::atomic frames_scheduled_for_processing_{0}; ScopedTaskSafetyDetached safety_; @@ -211,21 +239,64 @@ ZeroHertzAdapterMode::ZeroHertzAdapterMode( TaskQueueBase* queue, Clock* clock, FrameCadenceAdapterInterface::Callback* callback, - double max_fps) - : queue_(queue), clock_(clock), callback_(callback), max_fps_(max_fps) { + double max_fps, + FrameCadenceAdapterInterface::ZeroHertzModeParams params) + : queue_(queue), + clock_(clock), + callback_(callback), + max_fps_(max_fps), + layer_trackers_(params.num_simulcast_layers) { sequence_checker_.Detach(); } +void ZeroHertzAdapterMode::UpdateLayerQualityConvergence( + int spatial_index, + bool quality_converged) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_LT(spatial_index, layer_trackers_.size()); + RTC_LOG(LS_INFO) << __func__ << " layer " << spatial_index + << " quality has converged: " << quality_converged; + if (layer_trackers_[spatial_index].quality_converged.has_value()) + layer_trackers_[spatial_index].quality_converged = quality_converged; +} + +void ZeroHertzAdapterMode::UpdateLayerStatus(int spatial_index, bool enabled) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_LT(spatial_index, layer_trackers_.size()); + if (enabled) { + if (!layer_trackers_[spatial_index].quality_converged.has_value()) { + // Assume quality has not converged until hearing otherwise. + layer_trackers_[spatial_index].quality_converged = false; + } + } else { + layer_trackers_[spatial_index].quality_converged = absl::nullopt; + } + RTC_LOG(LS_INFO) + << __func__ << " layer " << spatial_index + << (enabled + ? (layer_trackers_[spatial_index].quality_converged.has_value() + ? " enabled." + : " enabled and it's assumed quality has not converged.") + : " disabled."); +} + void ZeroHertzAdapterMode::OnFrame(Timestamp post_time, int frames_scheduled_for_processing, const VideoFrame& frame) { RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this; + + // Assume all enabled layers are unconverged after frame entry. + for (auto& layer_tracker : layer_trackers_) { + if (layer_tracker.quality_converged.has_value()) + layer_tracker.quality_converged = false; + } // Remove stored repeating frame if needed. if (is_repeating_) { RTC_DCHECK(queued_frames_.size() == 1); - RTC_LOG(LS_VERBOSE) << __func__ << " this " << this - << " cancel repeat and restart with original"; + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this + << " cancel repeat and restart with original"; queued_frames_.pop_front(); } @@ -249,6 +320,7 @@ absl::optional ZeroHertzAdapterMode::GetInputFrameRateFps() { // RTC_RUN_ON(&sequence_checker_) void ZeroHertzAdapterMode::ProcessOnDelayedCadence() { RTC_DCHECK(!queued_frames_.empty()); + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this; SendFrameNow(queued_frames_.front()); @@ -260,23 +332,42 @@ void ZeroHertzAdapterMode::ProcessOnDelayedCadence() { } // There's only one frame to send. Schedule a repeat sequence, which is - // cancelled by |current_frame_id_| getting incremented should new frames + // cancelled by `current_frame_id_` getting incremented should new frames // arrive. is_repeating_ = true; - int frame_id = current_frame_id_; + ScheduleRepeat(current_frame_id_); +} + +// RTC_RUN_ON(&sequence_checker_) +void ZeroHertzAdapterMode::ScheduleRepeat(int frame_id) { + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this << " frame_id " + << frame_id; + // Determine if quality has converged. Adjust the time for the next repeat + // accordingly. + const bool quality_converged = + absl::c_all_of(layer_trackers_, [](const SpatialLayerTracker& tracker) { + return !tracker.quality_converged.has_value() || + tracker.quality_converged.value(); + }); + TimeDelta repeat_delay = + quality_converged + ? FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod + : frame_delay_; queue_->PostDelayedTask(ToQueuedTask(safety_, - [this, frame_id] { + [this, frame_id, repeat_delay] { RTC_DCHECK_RUN_ON(&sequence_checker_); ProcessRepeatedFrameOnDelayedCadence( - frame_id, frame_delay_); + frame_id, repeat_delay); }), - frame_delay_.ms()); + repeat_delay.ms()); } // RTC_RUN_ON(&sequence_checker_) void ZeroHertzAdapterMode::ProcessRepeatedFrameOnDelayedCadence( int frame_id, TimeDelta scheduled_delay) { + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this << " frame_id " + << frame_id; RTC_DCHECK(!queued_frames_.empty()); // Cancel this invocation if new frames turned up. @@ -300,23 +391,12 @@ void ZeroHertzAdapterMode::ProcessRepeatedFrameOnDelayedCadence( frame.set_ntp_time_ms(frame.ntp_time_ms() + scheduled_delay.ms()); SendFrameNow(frame); - // TODO(crbug.com/1255737): Wire in a QP convergence signal here and adjust - // the delay on QP convergence to some lowest rate being a compromise between - // RTP receiver keyframe-requesting timeout (3s), backend limitations and some - // worst case RTT. - int delay_ms = frame_delay_.ms(); - - // Schedule another repeat depending on if QP converged. - queue_->PostDelayedTask(ToQueuedTask(safety_, - [this, frame_id] { - RTC_DCHECK_RUN_ON(&sequence_checker_); - ProcessRepeatedFrameOnDelayedCadence( - frame_id, frame_delay_); - }), - delay_ms); + // Schedule another repeat. + ScheduleRepeat(frame_id); } void ZeroHertzAdapterMode::SendFrameNow(const VideoFrame& frame) { + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this; // TODO(crbug.com/1255737): figure out if frames_scheduled_for_processing // makes sense to compute in this implementation. callback_->OnFrame(/*post_time=*/clock_->CurrentTime(), @@ -336,12 +416,13 @@ void FrameCadenceAdapterImpl::Initialize(Callback* callback) { current_adapter_mode_ = &passthrough_adapter_.value(); } -void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled(bool enabled) { +void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled( + absl::optional params) { RTC_DCHECK_RUN_ON(queue_); - bool was_zero_hertz_enabled = zero_hertz_and_uma_reporting_enabled_; - if (enabled && !zero_hertz_and_uma_reporting_enabled_) + bool was_zero_hertz_enabled = zero_hertz_params_.has_value(); + if (params.has_value() && !was_zero_hertz_enabled) has_reported_screenshare_frame_rate_umas_ = false; - zero_hertz_and_uma_reporting_enabled_ = enabled; + zero_hertz_params_ = params; MaybeReconfigureAdapters(was_zero_hertz_enabled); } @@ -358,6 +439,20 @@ void FrameCadenceAdapterImpl::UpdateFrameRate() { passthrough_adapter_->UpdateFrameRate(); } +void FrameCadenceAdapterImpl::UpdateLayerQualityConvergence( + int spatial_index, + bool quality_converged) { + if (zero_hertz_adapter_.has_value()) + zero_hertz_adapter_->UpdateLayerQualityConvergence(spatial_index, + quality_converged); +} + +void FrameCadenceAdapterImpl::UpdateLayerStatus(int spatial_index, + bool enabled) { + if (zero_hertz_adapter_.has_value()) + zero_hertz_adapter_->UpdateLayerStatus(spatial_index, enabled); +} + void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { // This method is called on the network thread under Chromium, or other // various contexts in test. @@ -404,7 +499,7 @@ bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const { return zero_hertz_screenshare_enabled_ && source_constraints_.has_value() && source_constraints_->max_fps.value_or(-1) > 0 && source_constraints_->min_fps.value_or(-1) == 0 && - zero_hertz_and_uma_reporting_enabled_; + zero_hertz_params_.has_value(); } // RTC_RUN_ON(queue_) @@ -414,7 +509,9 @@ void FrameCadenceAdapterImpl::MaybeReconfigureAdapters( if (is_zero_hertz_enabled) { if (!was_zero_hertz_enabled) { zero_hertz_adapter_.emplace(queue_, clock_, callback_, - source_constraints_->max_fps.value()); + source_constraints_->max_fps.value(), + zero_hertz_params_.value()); + RTC_LOG(LS_INFO) << "FrameCadenceAdapterImpl: Zero hertz mode activated."; } current_adapter_mode_ = &zero_hertz_adapter_.value(); } else { @@ -429,7 +526,7 @@ void FrameCadenceAdapterImpl::MaybeReportFrameRateConstraintUmas() { if (has_reported_screenshare_frame_rate_umas_) return; has_reported_screenshare_frame_rate_umas_ = true; - if (!zero_hertz_and_uma_reporting_enabled_) + if (!zero_hertz_params_.has_value()) return; RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Exists", source_constraints_.has_value()); diff --git a/video/frame_cadence_adapter.h b/video/frame_cadence_adapter.h index 8685f37f97..aeecea992d 100644 --- a/video/frame_cadence_adapter.h +++ b/video/frame_cadence_adapter.h @@ -14,6 +14,7 @@ #include #include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" #include "api/video/video_frame.h" #include "api/video/video_sink_interface.h" #include "rtc_base/synchronization/mutex.h" @@ -31,7 +32,18 @@ class FrameCadenceAdapterInterface public: // Averaging window spanning 90 frames at default 30fps, matching old media // optimization module defaults. + // TODO(crbug.com/1255737): Use TimeDelta. static constexpr int64_t kFrameRateAveragingWindowSizeMs = (1000 / 30) * 90; + // In zero-hertz mode, the idle repeat rate is a compromise between + // RTP receiver keyframe-requesting timeout (3s), other backend limitations + // and some worst case RTT. + static constexpr TimeDelta kZeroHertzIdleRepeatRatePeriod = + TimeDelta::Millis(1000); + + struct ZeroHertzModeParams { + // The number of simulcast layers used in this configuration. + int num_simulcast_layers = 0; + }; // Callback interface used to inform instance owners. class Callback { @@ -68,8 +80,11 @@ class FrameCadenceAdapterInterface // Call before using the rest of the API. virtual void Initialize(Callback* callback) = 0; - // Pass true in |enabled| as a prerequisite to enable zero-hertz operation. - virtual void SetZeroHertzModeEnabled(bool enabled) = 0; + // Pass zero hertz parameters in |params| as a prerequisite to enable + // zero-hertz operation. If absl:::nullopt is passed, the cadence adapter will + // switch to passthrough mode. + virtual void SetZeroHertzModeEnabled( + absl::optional params) = 0; // Returns the input framerate. This is measured by RateStatistics when // zero-hertz mode is off, and returns the max framerate in zero-hertz mode. @@ -78,6 +93,13 @@ class FrameCadenceAdapterInterface // Updates frame rate. This is done unconditionally irrespective of adapter // mode. virtual void UpdateFrameRate() = 0; + + // Updates quality convergence status for an enabled spatial layer. + virtual void UpdateLayerQualityConvergence(int spatial_index, + bool converged) = 0; + + // Updates spatial layer enabled status. + virtual void UpdateLayerStatus(int spatial_index, bool enabled) = 0; }; } // namespace webrtc diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc index 946fe0c949..38827d6e64 100644 --- a/video/frame_cadence_adapter_unittest.cc +++ b/video/frame_cadence_adapter_unittest.cc @@ -14,10 +14,13 @@ #include #include "api/task_queue/task_queue_base.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/units/timestamp.h" #include "api/video/nv12_buffer.h" #include "api/video/video_frame.h" #include "rtc_base/rate_statistics.h" #include "rtc_base/ref_counted_object.h" +#include "rtc_base/task_utils/to_queued_task.h" #include "rtc_base/time_utils.h" #include "system_wrappers/include/metrics.h" #include "system_wrappers/include/ntp_time.h" @@ -159,7 +162,8 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsMaxFpsWhenZeroHertzActivated) { GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); auto adapter = CreateAdapter(time_controller.GetClock()); adapter->Initialize(nullptr); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); for (int frame = 0; frame != 10; ++frame) { time_controller.AdvanceTime(TimeDelta::Millis(10)); @@ -175,7 +179,8 @@ TEST(FrameCadenceAdapterTest, GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); auto adapter = CreateAdapter(time_controller.GetClock()); adapter->Initialize(nullptr); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); RateStatistics rate( FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000); @@ -187,7 +192,7 @@ TEST(FrameCadenceAdapterTest, } // Turn off zero hertz on the next-last frame; after the last frame we // should see a value that tracks the rate oracle. - adapter->SetZeroHertzModeEnabled(false); + adapter->SetZeroHertzModeEnabled(absl::nullopt); // Last frame. time_controller.AdvanceTime(TimeDelta::Millis(10)); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); @@ -203,7 +208,8 @@ TEST(FrameCadenceAdapterTest, ForwardsFramesDelayed) { GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); auto adapter = CreateAdapter(time_controller.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); constexpr int kNumFrames = 3; NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime(); @@ -237,7 +243,8 @@ TEST(FrameCadenceAdapterTest, RepeatsFramesDelayed) { GlobalSimulatedTimeController time_controller(Timestamp::Millis(47892223)); auto adapter = CreateAdapter(time_controller.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime(); @@ -290,7 +297,8 @@ TEST(FrameCadenceAdapterTest, GlobalSimulatedTimeController time_controller(Timestamp::Millis(4711)); auto adapter = CreateAdapter(time_controller.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); // Send one frame, expect a repeat. @@ -322,7 +330,8 @@ TEST(FrameCadenceAdapterTest, StopsRepeatingFramesDelayed) { GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); auto adapter = CreateAdapter(time_controller.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); NtpTime original_ntp_time = time_controller.GetClock()->CurrentNtpTime(); @@ -343,6 +352,120 @@ TEST(FrameCadenceAdapterTest, StopsRepeatingFramesDelayed) { time_controller.AdvanceTime(TimeDelta::Seconds(1)); } +class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test { + public: + static constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(100); + static constexpr TimeDelta kIdleFrameDelay = + FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod; + + ZeroHertzLayerQualityConvergenceTest() { + adapter_->Initialize(&callback_); + adapter_->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{ + /*num_simulcast_layers=*/2}); + adapter_->OnConstraintsChanged(VideoTrackSourceConstraints{ + /*min_fps=*/0, /*max_fps=*/TimeDelta::Seconds(1) / kMinFrameDelay}); + time_controller_.AdvanceTime(TimeDelta::Zero()); + } + + void PassFrame() { adapter_->OnFrame(CreateFrame()); } + + void ExpectFrameEntriesAtDelaysFromNow( + std::initializer_list list) { + Timestamp origin = time_controller_.GetClock()->CurrentTime(); + for (auto delay : list) { + EXPECT_CALL(callback_, OnFrame(origin + delay, _, _)); + time_controller_.AdvanceTime(origin + delay - + time_controller_.GetClock()->CurrentTime()); + } + } + + void ScheduleDelayed(TimeDelta delay, std::function function) { + TaskQueueBase::Current()->PostDelayedTask( + ToQueuedTask([function = std::move(function)] { function(); }), + delay.ms()); + } + + protected: + ZeroHertzFieldTrialEnabler field_trial_enabler_; + MockCallback callback_; + GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)}; + std::unique_ptr adapter_{ + CreateAdapter(time_controller_.GetClock())}; +}; + +TEST_F(ZeroHertzLayerQualityConvergenceTest, InitialStateConverged) { + // We start out assuming we're disabled in all layers, therefore converged. In + // reality we expect layer enabledness to be set up very early on + // initialization, but to cover this case we prefer being converged over being + // unconverged due to lower CPU usage. + PassFrame(); + ExpectFrameEntriesAtDelaysFromNow({ + kMinFrameDelay, // Original frame emitted + kMinFrameDelay + + kIdleFrameDelay, // Idle repeats after convergence at 100. + kMinFrameDelay + 2 * kIdleFrameDelay, // ... + kMinFrameDelay + 3 * kIdleFrameDelay, // ... + }); +} + +TEST_F(ZeroHertzLayerQualityConvergenceTest, UnconvergedAfterLayersEnabled) { + // With newly enabled layers we assume quality is unconverged. + adapter_->UpdateLayerStatus(0, /*enabled=*/true); + adapter_->UpdateLayerStatus(1, /*enabled=*/true); + PassFrame(); + ExpectFrameEntriesAtDelaysFromNow({ + kMinFrameDelay, // Original frame emitted + 2 * kMinFrameDelay, // Unconverged repeats. + 3 * kMinFrameDelay, // ... + }); +} + +TEST_F(ZeroHertzLayerQualityConvergenceTest, + RepeatsPassedFramesUntilConvergence) { + ScheduleDelayed(TimeDelta::Zero(), [&] { + adapter_->UpdateLayerStatus(0, /*enabled=*/true); + adapter_->UpdateLayerStatus(1, /*enabled=*/true); + PassFrame(); + }); + ScheduleDelayed(2.5 * kMinFrameDelay, [&] { + adapter_->UpdateLayerQualityConvergence(/*spatial_index=*/1, true); + }); + ScheduleDelayed(3.5 * kMinFrameDelay, [&] { + adapter_->UpdateLayerQualityConvergence(/*spatial_index=*/0, true); + }); + ScheduleDelayed(8 * kMinFrameDelay, [&] { PassFrame(); }); + ScheduleDelayed(9.5 * kMinFrameDelay, [&] { + adapter_->UpdateLayerQualityConvergence(/*spatial_index=*/0, true); + }); + ScheduleDelayed(10.5 * kMinFrameDelay, [&] { + adapter_->UpdateLayerQualityConvergence(/*spatial_index=*/1, true); + }); + ExpectFrameEntriesAtDelaysFromNow({ + kMinFrameDelay, // Original frame emitted + 2 * kMinFrameDelay, // Repeat from kMinFrameDelay. + + // 2.5 * kMinFrameDelay: Converged in layer 1, layer 0 still unconverged. + 3 * kMinFrameDelay, // Repeat from 2 * kMinFrameDelay. + + // 3.5 * kMinFrameDelay: Converged in layer 0 as well. + 4 * kMinFrameDelay, // Repeat from 3 * kMinFrameDelay. An idle repeat is + // scheduled for kIdleFrameDelay + 3 * + // kMinFrameDelay. + + // A new frame is passed at 8 * kMinFrameDelay. + 9 * kMinFrameDelay, // Original frame emitted + + // 9.5 * kMinFrameDelay: Converged in layer 0, layer 1 still unconverged. + 10 * kMinFrameDelay, // Repeat from 9 * kMinFrameDelay. + // 10.5 * kMinFrameDelay: Converged in layer 0 as well. + 11 * kMinFrameDelay, // Idle repeats from 1000. + 11 * kMinFrameDelay + kIdleFrameDelay, // ... + 11 * kMinFrameDelay + 2 * kIdleFrameDelay, // ... + // ... + }); +} + class FrameCadenceAdapterMetricsTest : public ::testing::Test { public: FrameCadenceAdapterMetricsTest() : time_controller_(Timestamp::Millis(1)) { @@ -439,7 +562,8 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoConstraintsIfUnsetOnFrame) { MockCallback callback; auto adapter = CreateAdapter(time_controller_.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnFrame(CreateFrame()); DepleteTaskQueues(); EXPECT_THAT( @@ -451,7 +575,8 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsEmptyConstraintsIfSetOnFrame) { MockCallback callback; auto adapter = CreateAdapter(time_controller_.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged( VideoTrackSourceConstraints{absl::nullopt, absl::nullopt}); adapter->OnFrame(CreateFrame()); @@ -490,7 +615,8 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsMaxConstraintIfSetOnFrame) { MockCallback callback; auto adapter = CreateAdapter(time_controller_.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged( VideoTrackSourceConstraints{absl::nullopt, 2.0}); adapter->OnFrame(CreateFrame()); @@ -526,7 +652,8 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinConstraintIfSetOnFrame) { MockCallback callback; auto adapter = CreateAdapter(time_controller_.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged( VideoTrackSourceConstraints{3.0, absl::nullopt}); adapter->OnFrame(CreateFrame()); @@ -562,7 +689,8 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinGtMaxConstraintIfSetOnFrame) { MockCallback callback; auto adapter = CreateAdapter(time_controller_.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5.0, 4.0}); adapter->OnFrame(CreateFrame()); DepleteTaskQueues(); @@ -597,7 +725,8 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinLtMaxConstraintIfSetOnFrame) { MockCallback callback; auto adapter = CreateAdapter(time_controller_.GetClock()); adapter->Initialize(&callback); - adapter->SetZeroHertzModeEnabled(true); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4.0, 5.0}); adapter->OnFrame(CreateFrame()); DepleteTaskQueues(); diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 640c230c68..d581f0847c 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -51,6 +51,7 @@ #include "system_wrappers/include/metrics.h" #include "video/adaptation/video_stream_encoder_resource_manager.h" #include "video/alignment_adjuster.h" +#include "video/frame_cadence_adapter.h" namespace webrtc { @@ -821,10 +822,9 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, [this, config = std::move(config), max_data_payload_length]() mutable { RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK(sink_); - RTC_LOG(LS_INFO) << "ConfigureEncoder requested."; + RTC_LOG(LS_ERROR) << "ConfigureEncoder requested. simulcast_layers = " + << config.simulcast_layers.size(); - frame_cadence_adapter_->SetZeroHertzModeEnabled( - config.content_type == VideoEncoderConfig::ContentType::kScreen); pending_encoder_creation_ = (!encoder_ || encoder_config_.video_format != config.video_format || max_data_payload_length_ != max_data_payload_length); @@ -1252,6 +1252,13 @@ void VideoStreamEncoder::OnEncoderSettingsChanged() { bool is_screenshare = encoder_settings.encoder_config().content_type == VideoEncoderConfig::ContentType::kScreen; degradation_preference_manager_->SetIsScreenshare(is_screenshare); + if (is_screenshare) { + frame_cadence_adapter_->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{ + send_codec_.numberOfSimulcastStreams}); + } else { + frame_cadence_adapter_->SetZeroHertzModeEnabled(absl::nullopt); + } } void VideoStreamEncoder::OnFrame(Timestamp post_time, @@ -1449,8 +1456,16 @@ void VideoStreamEncoder::SetEncoderRates( last_encoder_rate_settings_ = rate_settings; } - if (!encoder_) { + if (!encoder_) return; + + // Make the cadence adapter know if streams were disabled. + for (int spatial_index = 0; + spatial_index != send_codec_.numberOfSimulcastStreams; ++spatial_index) { + frame_cadence_adapter_->UpdateLayerStatus( + spatial_index, + /*enabled=*/rate_settings.rate_control.target_bitrate + .GetSpatialLayerSum(spatial_index) > 0); } // `bitrate_allocation` is 0 it means that the network is down or the send @@ -1459,9 +1474,8 @@ void VideoStreamEncoder::SetEncoderRates( // bitrate. // TODO(perkj): Make sure all known encoder implementations handle zero // target bitrate and remove this check. - if (rate_settings.rate_control.bitrate.get_sum_bps() == 0) { + if (rate_settings.rate_control.bitrate.get_sum_bps() == 0) return; - } if (rate_control_changed) { encoder_->SetRates(rate_settings.rate_control); @@ -1867,14 +1881,24 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( RTC_CHECK(videocontenttypehelpers::SetSimulcastId( &image_copy.content_type_, static_cast(spatial_idx + 1))); - // Currently internal quality scaler is used for VP9 instead of webrtc qp - // scaler (in no-svc case or if only a single spatial layer is encoded). - // It has to be explicitly detected and reported to adaptation metrics. - // Post a task because `send_codec_` requires `encoder_queue_` lock. + // Post a task because `send_codec_` requires `encoder_queue_` lock and we + // need to update on quality convergence. unsigned int image_width = image_copy._encodedWidth; unsigned int image_height = image_copy._encodedHeight; - encoder_queue_.PostTask([this, codec_type, image_width, image_height] { + encoder_queue_.PostTask([this, codec_type, image_width, image_height, + spatial_idx, + at_target_quality = image_copy.IsAtTargetQuality()] { RTC_DCHECK_RUN_ON(&encoder_queue_); + + // Let the frame cadence adapter know about quality convergence. + if (frame_cadence_adapter_) + frame_cadence_adapter_->UpdateLayerQualityConvergence(spatial_idx, + at_target_quality); + + // Currently, the internal quality scaler is used for VP9 instead of the + // webrtc qp scaler (in the no-svc case or if only a single spatial layer is + // encoded). It has to be explicitly detected and reported to adaptation + // metrics. if (codec_type == VideoCodecType::kVideoCodecVP9 && send_codec_.VP9()->automaticResizeOn) { unsigned int expected_width = send_codec_.width; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index c5ff292a00..0a8c2a6c9d 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -23,6 +23,7 @@ #include "api/test/mock_fec_controller_override.h" #include "api/test/mock_video_encoder.h" #include "api/test/mock_video_encoder_factory.h" +#include "api/units/data_rate.h" #include "api/video/builtin_video_bitrate_allocator_factory.h" #include "api/video/i420_buffer.h" #include "api/video/nv12_buffer.h" @@ -82,6 +83,7 @@ using ::testing::Lt; using ::testing::Matcher; using ::testing::Mock; using ::testing::NiceMock; +using ::testing::Not; using ::testing::Optional; using ::testing::Return; using ::testing::SizeIs; @@ -118,6 +120,21 @@ const uint8_t kCodedFrameVp8Qp25[] = { 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c, 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0}; +void PassAFrame( + TaskQueueBase* encoder_queue, + FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback, + int64_t ntp_time_ms) { + encoder_queue->PostTask( + ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] { + video_stream_encoder_callback->OnFrame( + Timestamp::Millis(ntp_time_ms), 1, + VideoFrame::Builder() + .set_video_frame_buffer(rtc::make_ref_counted( + /*width=*/16, /*height=*/16)) + .build()); + })); +} + class TestBuffer : public webrtc::I420Buffer { public: TestBuffer(rtc::Event* event, int width, int height) @@ -641,6 +658,16 @@ class SimpleVideoStreamEncoderFactory { ~AdaptedVideoStreamEncoder() { Stop(); } }; + class MockFakeEncoder : public test::FakeEncoder { + public: + using FakeEncoder::FakeEncoder; + MOCK_METHOD(CodecSpecificInfo, + EncodeHook, + (EncodedImage & encoded_image, + rtc::scoped_refptr buffer), + (override)); + }; + SimpleVideoStreamEncoderFactory() { encoder_settings_.encoder_factory = &encoder_factory_; encoder_settings_.bitrate_allocator_factory = @@ -668,6 +695,7 @@ class SimpleVideoStreamEncoderFactory { } void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); } + MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; } private: class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink { @@ -701,18 +729,29 @@ class SimpleVideoStreamEncoderFactory { CreateBuiltinVideoBitrateAllocatorFactory(); VideoStreamEncoderSettings encoder_settings_{ VideoEncoder::Capabilities(/*loss_notification=*/false)}; - test::FakeEncoder fake_encoder_{time_controller_.GetClock()}; - test::VideoEncoderProxyFactory encoder_factory_{&fake_encoder_}; + MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()}; + test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_}; NullEncoderSink sink_; }; class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface { public: MOCK_METHOD(void, Initialize, (Callback * callback), (override)); - MOCK_METHOD(void, SetZeroHertzModeEnabled, (bool), (override)); + MOCK_METHOD(void, + SetZeroHertzModeEnabled, + (absl::optional), + (override)); MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); MOCK_METHOD(absl::optional, GetInputFrameRateFps, (), (override)); MOCK_METHOD(void, UpdateFrameRate, (), (override)); + MOCK_METHOD(void, + UpdateLayerQualityConvergence, + (int spatial_index, bool converged), + (override)); + MOCK_METHOD(void, + UpdateLayerStatus, + (int spatial_index, bool enabled), + (override)); }; class MockEncoderSelector @@ -8716,19 +8755,32 @@ TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) { auto adapter = std::make_unique(); auto* adapter_ptr = adapter.get(); SimpleVideoStreamEncoderFactory factory; - auto video_stream_encoder = factory.Create(std::move(adapter)); + FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback = + nullptr; + EXPECT_CALL(*adapter_ptr, Initialize) + .WillOnce(Invoke([&video_stream_encoder_callback]( + FrameCadenceAdapterInterface::Callback* callback) { + video_stream_encoder_callback = callback; + })); + TaskQueueBase* encoder_queue = nullptr; + auto video_stream_encoder = + factory.Create(std::move(adapter), &encoder_queue); - EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(true)); + EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Not(Eq(absl::nullopt)))); VideoEncoderConfig config; + test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config); config.content_type = VideoEncoderConfig::ContentType::kScreen; video_stream_encoder->ConfigureEncoder(std::move(config), 0); + PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1); factory.DepleteTaskQueues(); Mock::VerifyAndClearExpectations(adapter_ptr); - EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(false)); + EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt))); VideoEncoderConfig config2; + test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2); config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; video_stream_encoder->ConfigureEncoder(std::move(config2), 0); + PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2); factory.DepleteTaskQueues(); } @@ -8745,12 +8797,7 @@ TEST(VideoStreamEncoderFrameCadenceTest, EXPECT_CALL(*adapter_ptr, OnFrame); auto buffer = rtc::make_ref_counted(/*width=*/16, /*height=*/16); video_source.IncomingCapturedFrame( - VideoFrame::Builder() - .set_video_frame_buffer(std::move(buffer)) - .set_ntp_time_ms(0) - .set_timestamp_ms(0) - .set_rotation(kVideoRotation_0) - .build()); + VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build()); } TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) { @@ -8781,18 +8828,125 @@ TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) { EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps); EXPECT_CALL(*adapter_ptr, UpdateFrameRate); - encoder_queue->PostTask(ToQueuedTask([video_stream_encoder_callback] { - video_stream_encoder_callback->OnFrame( - Timestamp::Millis(1), 1, - VideoFrame::Builder() - .set_video_frame_buffer( - rtc::make_ref_counted(/*width=*/16, /*height=*/16)) - .set_ntp_time_ms(0) - .set_timestamp_ms(0) - .set_rotation(kVideoRotation_0) - .build()); - })); + PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1); factory.DepleteTaskQueues(); } +TEST(VideoStreamEncoderFrameCadenceTest, + DeactivatesActivatesLayersOnBitrateChanges) { + auto adapter = std::make_unique(); + auto* adapter_ptr = adapter.get(); + SimpleVideoStreamEncoderFactory factory; + FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback = + nullptr; + EXPECT_CALL(*adapter_ptr, Initialize) + .WillOnce(Invoke([&video_stream_encoder_callback]( + FrameCadenceAdapterInterface::Callback* callback) { + video_stream_encoder_callback = callback; + })); + TaskQueueBase* encoder_queue = nullptr; + auto video_stream_encoder = + factory.Create(std::move(adapter), &encoder_queue); + + // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to + // {150000, 450000}. + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config); + video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + // Ensure an encoder is created. + PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1); + + // Both layers enabled at 1 MBit/s. + video_stream_encoder->OnBitrateUpdated( + DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000), + DataRate::KilobitsPerSec(1000), 0, 0, 0); + EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true)); + EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true)); + factory.DepleteTaskQueues(); + Mock::VerifyAndClearExpectations(adapter_ptr); + + // Layer 1 disabled at 200 KBit/s. + video_stream_encoder->OnBitrateUpdated( + DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200), + DataRate::KilobitsPerSec(200), 0, 0, 0); + EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true)); + EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false)); + factory.DepleteTaskQueues(); + Mock::VerifyAndClearExpectations(adapter_ptr); + + // All layers off at suspended video. + video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(), + DataRate::Zero(), 0, 0, 0); + EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false)); + EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false)); + factory.DepleteTaskQueues(); + Mock::VerifyAndClearExpectations(adapter_ptr); + + // Both layers enabled again back at 1 MBit/s. + video_stream_encoder->OnBitrateUpdated( + DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000), + DataRate::KilobitsPerSec(1000), 0, 0, 0); + EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true)); + EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true)); + factory.DepleteTaskQueues(); +} + +TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) { + auto adapter = std::make_unique(); + auto* adapter_ptr = adapter.get(); + SimpleVideoStreamEncoderFactory factory; + FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback = + nullptr; + EXPECT_CALL(*adapter_ptr, Initialize) + .WillOnce(Invoke([&video_stream_encoder_callback]( + FrameCadenceAdapterInterface::Callback* callback) { + video_stream_encoder_callback = callback; + })); + TaskQueueBase* encoder_queue = nullptr; + auto video_stream_encoder = + factory.Create(std::move(adapter), &encoder_queue); + + // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder. + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config); + video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder->OnBitrateUpdated( + DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000), + DataRate::KilobitsPerSec(1000), 0, 0, 0); + + // Pass a frame which has unconverged results. + PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1); + EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook) + .WillRepeatedly(Invoke([](EncodedImage& encoded_image, + rtc::scoped_refptr buffer) { + EXPECT_FALSE(encoded_image.IsAtTargetQuality()); + CodecSpecificInfo codec_specific; + codec_specific.codecType = kVideoCodecGeneric; + return codec_specific; + })); + EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false)); + EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false)); + factory.DepleteTaskQueues(); + Mock::VerifyAndClearExpectations(adapter_ptr); + Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder()); + + // Pass a frame which converges in layer 0 and not in layer 1. + PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2); + EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook) + .WillRepeatedly(Invoke([](EncodedImage& encoded_image, + rtc::scoped_refptr buffer) { + encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0); + CodecSpecificInfo codec_specific; + codec_specific.codecType = kVideoCodecGeneric; + return codec_specific; + })); + EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true)); + EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false)); + factory.DepleteTaskQueues(); + Mock::VerifyAndClearExpectations(adapter_ptr); + Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder()); +} + } // namespace webrtc