diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc index 36ef740079..54cd5b2994 100644 --- a/video/frame_cadence_adapter.cc +++ b/video/frame_cadence_adapter.cc @@ -96,12 +96,15 @@ class PassthroughAdapterMode : public AdapterMode { // Implements a frame cadence adapter supporting zero-hertz input. class ZeroHertzAdapterMode : public AdapterMode { public: - ZeroHertzAdapterMode( - TaskQueueBase* queue, - Clock* clock, - FrameCadenceAdapterInterface::Callback* callback, - double max_fps, - FrameCadenceAdapterInterface::ZeroHertzModeParams params); + ZeroHertzAdapterMode(TaskQueueBase* queue, + Clock* clock, + FrameCadenceAdapterInterface::Callback* callback, + double max_fps); + + // Reconfigures according to parameters. + // All spatial layer trackers are initialized as unconverged by this method. + void ReconfigureParameters( + const FrameCadenceAdapterInterface::ZeroHertzModeParams& params); // Updates spatial layer quality convergence status. void UpdateLayerQualityConvergence(int spatial_index, bool quality_converged); @@ -143,6 +146,9 @@ class ZeroHertzAdapterMode : public AdapterMode { // Convergence means QP has dropped to a low-enough level to warrant ceasing // to send identical frames at high frequency. bool HasQualityConverged() const RTC_RUN_ON(sequence_checker_); + // Resets quality convergence information. HasQualityConverged() returns false + // after this call. + void ResetQualityConvergenceInfo() RTC_RUN_ON(sequence_checker_); // Processes incoming frames on a delayed cadence. void ProcessOnDelayedCadence() RTC_RUN_ON(sequence_checker_); // Schedules a later repeat with delay depending on state of layer trackers. @@ -191,6 +197,7 @@ class ZeroHertzAdapterMode : public AdapterMode { class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { public: FrameCadenceAdapterImpl(Clock* clock, TaskQueueBase* queue); + ~FrameCadenceAdapterImpl(); // FrameCadenceAdapterInterface overrides. void Initialize(Callback* callback) override; @@ -266,22 +273,29 @@ ZeroHertzAdapterMode::ZeroHertzAdapterMode( TaskQueueBase* queue, Clock* clock, FrameCadenceAdapterInterface::Callback* callback, - double max_fps, - FrameCadenceAdapterInterface::ZeroHertzModeParams params) - : queue_(queue), - clock_(clock), - callback_(callback), - max_fps_(max_fps), - layer_trackers_(params.num_simulcast_layers) { + double max_fps) + : queue_(queue), clock_(clock), callback_(callback), max_fps_(max_fps) { sequence_checker_.Detach(); } +void ZeroHertzAdapterMode::ReconfigureParameters( + const FrameCadenceAdapterInterface::ZeroHertzModeParams& params) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_LOG(LS_INFO) << __func__ << " this " << this << " num_simulcast_layers " + << params.num_simulcast_layers; + + // Start as unconverged. + layer_trackers_.clear(); + layer_trackers_.resize(params.num_simulcast_layers, + SpatialLayerTracker{false}); +} + void ZeroHertzAdapterMode::UpdateLayerQualityConvergence( int spatial_index, bool quality_converged) { RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_LT(spatial_index, layer_trackers_.size()); - RTC_LOG(LS_INFO) << __func__ << " layer " << spatial_index + RTC_LOG(LS_INFO) << __func__ << " this " << this << " layer " << spatial_index << " quality has converged: " << quality_converged; if (layer_trackers_[spatial_index].quality_converged.has_value()) layer_trackers_[spatial_index].quality_converged = quality_converged; @@ -299,7 +313,7 @@ void ZeroHertzAdapterMode::UpdateLayerStatus(int spatial_index, bool enabled) { layer_trackers_[spatial_index].quality_converged = absl::nullopt; } RTC_LOG(LS_INFO) - << __func__ << " layer " << spatial_index + << __func__ << " this " << this << " layer " << spatial_index << (enabled ? (layer_trackers_[spatial_index].quality_converged.has_value() ? " enabled." @@ -311,13 +325,11 @@ void ZeroHertzAdapterMode::OnFrame(Timestamp post_time, int frames_scheduled_for_processing, const VideoFrame& frame) { RTC_DCHECK_RUN_ON(&sequence_checker_); - RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this; + RTC_DLOG(LS_VERBOSE) << "ZeroHertzAdapterMode::" << __func__ << " this " + << this; // Assume all enabled layers are unconverged after frame entry. - for (auto& layer_tracker : layer_trackers_) { - if (layer_tracker.quality_converged.has_value()) - layer_tracker.quality_converged = false; - } + ResetQualityConvergenceInfo(); // Remove stored repeating frame if needed. if (scheduled_repeat_.has_value()) { @@ -349,9 +361,9 @@ bool ZeroHertzAdapterMode::ProcessKeyFrameRequest() { // If no frame was ever passed to us, request a refresh frame from the source. if (current_frame_id_ == 0) { - RTC_LOG(LS_INFO) << __func__ << " this " << this - << " recommending requesting refresh frame due to no " - "frames received yet."; + RTC_LOG(LS_INFO) + << __func__ << " this " << this + << " requesting refresh frame due to no frames received yet."; return true; } @@ -359,7 +371,7 @@ bool ZeroHertzAdapterMode::ProcessKeyFrameRequest() { // very soon send out a frame and don't need a refresh frame. if (!scheduled_repeat_.has_value() || !scheduled_repeat_->idle) { RTC_LOG(LS_INFO) << __func__ << " this " << this - << " ignoring key frame request because of recently " + << " not requesting refresh frame because of recently " "incoming frame or short repeating."; return false; } @@ -370,16 +382,17 @@ bool ZeroHertzAdapterMode::ProcessKeyFrameRequest() { if (scheduled_repeat_->scheduled + RepeatDuration(/*idle_repeat=*/true) - now <= frame_delay_) { - RTC_LOG(LS_INFO) - << __func__ << " this " << this - << " ignoring key frame request because of soon happening idle repeat"; + RTC_LOG(LS_INFO) << __func__ << " this " << this + << " not requesting refresh frame because of soon " + "happening idle repeat"; return false; } // Cancel the current repeat and reschedule a short repeat now. No need for a // new refresh frame. RTC_LOG(LS_INFO) << __func__ << " this " << this - << " scheduling a short repeat due to key frame request"; + << " not requesting refresh frame and scheduling a short " + "repeat due to key frame request"; ScheduleRepeat(++current_frame_id_, /*idle_repeat=*/false); return false; } @@ -393,6 +406,15 @@ bool ZeroHertzAdapterMode::HasQualityConverged() const { return quality_converged; } +// RTC_RUN_ON(&sequence_checker_) +void ZeroHertzAdapterMode::ResetQualityConvergenceInfo() { + RTC_DLOG(LS_INFO) << __func__ << " this " << this; + for (auto& layer_tracker : layer_trackers_) { + if (layer_tracker.quality_converged.has_value()) + layer_tracker.quality_converged = false; + } +} + // RTC_RUN_ON(&sequence_checker_) void ZeroHertzAdapterMode::ProcessOnDelayedCadence() { RTC_DCHECK(!queued_frames_.empty()); @@ -484,6 +506,10 @@ FrameCadenceAdapterImpl::FrameCadenceAdapterImpl(Clock* clock, zero_hertz_screenshare_enabled_( field_trial::IsEnabled("WebRTC-ZeroHertzScreenshare")) {} +FrameCadenceAdapterImpl::~FrameCadenceAdapterImpl() { + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this; +} + void FrameCadenceAdapterImpl::Initialize(Callback* callback) { callback_ = callback; passthrough_adapter_.emplace(clock_, callback); @@ -539,6 +565,8 @@ void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { // This method is called on the network thread under Chromium, or other // various contexts in test. RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); + RTC_DLOG(LS_VERBOSE) << "FrameCadenceAdapterImpl::" << __func__ << " this " + << this; // Local time in webrtc time base. Timestamp post_time = clock_->CurrentTime(); @@ -556,7 +584,7 @@ void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { void FrameCadenceAdapterImpl::OnConstraintsChanged( const VideoTrackSourceConstraints& constraints) { - RTC_LOG(LS_INFO) << __func__ << " min_fps " + RTC_LOG(LS_INFO) << __func__ << " this " << this << " min_fps " << constraints.min_fps.value_or(-1) << " max_fps " << constraints.max_fps.value_or(-1); queue_->PostTask(ToQueuedTask(safety_.flag(), [this, constraints] { @@ -591,10 +619,10 @@ void FrameCadenceAdapterImpl::MaybeReconfigureAdapters( if (is_zero_hertz_enabled) { if (!was_zero_hertz_enabled) { zero_hertz_adapter_.emplace(queue_, clock_, callback_, - source_constraints_->max_fps.value(), - zero_hertz_params_.value()); - RTC_LOG(LS_INFO) << "FrameCadenceAdapterImpl: Zero hertz mode activated."; + source_constraints_->max_fps.value()); + RTC_LOG(LS_INFO) << "Zero hertz mode activated."; } + zero_hertz_adapter_->ReconfigureParameters(zero_hertz_params_.value()); current_adapter_mode_ = &zero_hertz_adapter_.value(); } else { if (was_zero_hertz_enabled) diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc index 8932c7f410..a790c4ff46 100644 --- a/video/frame_cadence_adapter_unittest.cc +++ b/video/frame_cadence_adapter_unittest.cc @@ -457,6 +457,28 @@ TEST(FrameCadenceAdapterTest, time_controller.AdvanceTime(kIdleFrameDelay); } +TEST(FrameCadenceAdapterTest, LayerReconfigurationResetsConvergenceInfo) { + ZeroHertzFieldTrialEnabler enabler; + MockCallback callback; + GlobalSimulatedTimeController time_controller(Timestamp::Millis(0)); + auto adapter = CreateAdapter(time_controller.GetClock()); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); + constexpr int kMaxFpsHz = 10; + constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(1000 / kMaxFpsHz); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFpsHz}); + time_controller.AdvanceTime(TimeDelta::Zero()); + + // Now setup 2 simulcast layers. The state should be unconverged. + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{ + /*num_simulcast_layers=*/2}); + adapter->OnFrame(CreateFrame()); + EXPECT_CALL(callback, OnFrame).Times(kMaxFpsHz); + time_controller.AdvanceTime(kMaxFpsHz * kMinFrameDelay); +} + class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test { public: static constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(100); @@ -499,18 +521,13 @@ class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test { CreateAdapter(time_controller_.GetClock())}; }; -TEST_F(ZeroHertzLayerQualityConvergenceTest, InitialStateConverged) { - // We start out assuming we're disabled in all layers, therefore converged. In - // reality we expect layer enabledness to be set up very early on - // initialization, but to cover this case we prefer being converged over being - // unconverged due to lower CPU usage. +TEST_F(ZeroHertzLayerQualityConvergenceTest, InitialStateUnconverged) { + // As the layer count is just configured, assume we start out as unconverged. PassFrame(); ExpectFrameEntriesAtDelaysFromNow({ - kMinFrameDelay, // Original frame emitted - kMinFrameDelay + - kIdleFrameDelay, // Idle repeats after convergence at 100. - kMinFrameDelay + 2 * kIdleFrameDelay, // ... - kMinFrameDelay + 3 * kIdleFrameDelay, // ... + 1 * kMinFrameDelay, // Original frame emitted + 2 * kMinFrameDelay, // Short repeats. + 3 * kMinFrameDelay, // ... }); } diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 572f82cb99..3baf10a8ad 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -822,8 +822,21 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, [this, config = std::move(config), max_data_payload_length]() mutable { RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK(sink_); - RTC_LOG(LS_ERROR) << "ConfigureEncoder requested. simulcast_layers = " - << config.simulcast_layers.size(); + RTC_LOG(LS_INFO) << "ConfigureEncoder requested."; + + // Set up the frame cadence adapter according to if we're going to do + // screencast. The final number of spatial layers is based on info + // in `send_codec_`, which is computed based on incoming frame + // dimensions which can only be determined later. + // + // Note: zero-hertz mode isn't enabled by this alone. Constraints also + // have to be set up with min_fps = 0 and max_fps > 0. + if (config.content_type == VideoEncoderConfig::ContentType::kScreen) { + frame_cadence_adapter_->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); + } else { + frame_cadence_adapter_->SetZeroHertzModeEnabled(absl::nullopt); + } pending_encoder_creation_ = (!encoder_ || encoder_config_.video_format != config.video_format || @@ -1256,8 +1269,6 @@ void VideoStreamEncoder::OnEncoderSettingsChanged() { frame_cadence_adapter_->SetZeroHertzModeEnabled( FrameCadenceAdapterInterface::ZeroHertzModeParams{ send_codec_.numberOfSimulcastStreams}); - } else { - frame_cadence_adapter_->SetZeroHertzModeEnabled(absl::nullopt); } } @@ -1815,15 +1826,20 @@ void VideoStreamEncoder::SendKeyFrame() { TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); RTC_DCHECK(!next_frame_types_.empty()); - if (!encoder_) - return; // Shutting down. - - if (frame_cadence_adapter_ && - frame_cadence_adapter_->ProcessKeyFrameRequest()) { - worker_queue_->PostTask(ToQueuedTask(task_safety_, [this] { - RTC_DCHECK_RUN_ON(worker_queue_); - video_source_sink_controller_.RequestRefreshFrame(); - })); + if (frame_cadence_adapter_) { + if (frame_cadence_adapter_->ProcessKeyFrameRequest()) { + RTC_DLOG(LS_INFO) << __func__ << " RequestRefreshFrame()."; + worker_queue_->PostTask(ToQueuedTask(task_safety_, [this] { + RTC_DCHECK_RUN_ON(worker_queue_); + video_source_sink_controller_.RequestRefreshFrame(); + })); + } else { + RTC_DLOG(LS_INFO) << __func__ << " No RequestRefreshFrame()."; + } + } + if (!encoder_) { + RTC_DLOG(LS_INFO) << __func__ << " no encoder."; + return; // Shutting down, or not configured yet. } // TODO(webrtc:10615): Map keyframe request to spatial layer. diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 3d6956ebf7..c1d89d73b2 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -72,7 +72,6 @@ namespace webrtc { using ::testing::_; using ::testing::AllOf; -using ::testing::AtLeast; using ::testing::Eq; using ::testing::Field; using ::testing::Ge; @@ -83,7 +82,6 @@ using ::testing::Lt; using ::testing::Matcher; using ::testing::Mock; using ::testing::NiceMock; -using ::testing::Not; using ::testing::Optional; using ::testing::Return; using ::testing::SizeIs; @@ -8781,15 +8779,29 @@ TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) { auto video_stream_encoder = factory.Create(std::move(adapter), &encoder_queue); - EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Not(Eq(absl::nullopt)))); + // First a call before we know the frame size and hence cannot compute the + // number of simulcast layers. + EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field( + &FrameCadenceAdapterInterface:: + ZeroHertzModeParams::num_simulcast_layers, + Eq(0))))); VideoEncoderConfig config; test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config); config.content_type = VideoEncoderConfig::ContentType::kScreen; video_stream_encoder->ConfigureEncoder(std::move(config), 0); + factory.DepleteTaskQueues(); + + // Then a call as we've computed the number of simulcast layers after a passed + // frame. + EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field( + &FrameCadenceAdapterInterface:: + ZeroHertzModeParams::num_simulcast_layers, + Gt(0))))); PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1); factory.DepleteTaskQueues(); Mock::VerifyAndClearExpectations(adapter_ptr); + // Expect a disabled zero-hertz mode after passing realtime video. EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt))); VideoEncoderConfig config2; test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);