diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 364aa0efdb..b1438392ae 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -87,6 +87,7 @@ rtc_library("video_coding") { "../../api/video:video_bitrate_allocation", "../../api/video:video_bitrate_allocator_factory", "../../rtc_base:deprecation", + "../../rtc_base/task_utils:to_queued_task", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", "../rtp_rtcp:rtp_video_header", @@ -303,6 +304,7 @@ rtc_library("video_coding_utility") { deps = [ ":video_codec_interface", "..:module_api", + "../../api:scoped_refptr", "../../api/video:encoded_frame", "../../api/video:encoded_image", "../../api/video:video_adaptation", @@ -315,6 +317,7 @@ rtc_library("video_coding_utility") { "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_numerics", "../../rtc_base:rtc_task_queue", + "../../rtc_base:weak_ptr", "../../rtc_base/experiments:quality_scaler_settings", "../../rtc_base/experiments:quality_scaling_experiment", "../../rtc_base/experiments:rate_control_settings", @@ -323,6 +326,7 @@ rtc_library("video_coding_utility") { "../../rtc_base/system:arch", "../../rtc_base/system:file_wrapper", "../../rtc_base/task_utils:repeating_task", + "../../rtc_base/task_utils:to_queued_task", "../../system_wrappers:field_trial", "../rtp_rtcp:rtp_rtcp_format", "//third_party/abseil-cpp/absl/types:optional", diff --git a/modules/video_coding/utility/quality_scaler.cc b/modules/video_coding/utility/quality_scaler.cc index c3d8b2e425..e909b2f88e 100644 --- a/modules/video_coding/utility/quality_scaler.cc +++ b/modules/video_coding/utility/quality_scaler.cc @@ -19,6 +19,8 @@ #include "rtc_base/logging.h" #include "rtc_base/numerics/exp_filter.h" #include "rtc_base/task_queue.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "rtc_base/weak_ptr.h" // TODO(kthelgason): Some versions of Android have issues with log2. // See https://code.google.com/p/android/issues/detail?id=212634 for details @@ -69,15 +71,192 @@ class QualityScaler::QpSmoother { rtc::ExpFilter smoother_; }; -QualityScaler::QualityScaler(AdaptationObserverInterface* observer, +// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The +// task will either run to completion and trigger a new task being queued, or it +// will be destroyed because the QualityScaler is destroyed. +// +// When high or low QP is reported, the task will be pending until a callback is +// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage +// asynchronously and prevents checking for QP until the stream has potentially +// been reconfigured. +class QualityScaler::CheckQpTask { + public: + // The result of one CheckQpTask may influence the delay of the next + // CheckQpTask. + struct Result { + bool observed_enough_frames = false; + bool qp_usage_reported = false; + bool clear_qp_samples = false; + }; + + CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result) + : quality_scaler_(quality_scaler), + state_(State::kNotStarted), + previous_task_result_(previous_task_result), + weak_ptr_factory_(this) {} + + void StartDelayedTask() { + RTC_DCHECK_EQ(state_, State::kNotStarted); + state_ = State::kCheckingQp; + TaskQueueBase::Current()->PostDelayedTask( + ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] { + if (!this_weak_ptr) { + // The task has been cancelled through destruction. + return; + } + RTC_DCHECK_EQ(state_, State::kCheckingQp); + RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_); + switch (quality_scaler_->CheckQp()) { + case QualityScaler::CheckQpResult::kInsufficientSamples: { + result_.observed_enough_frames = false; + // After this line, |this| may be deleted. + DoCompleteTask(); + return; + } + case QualityScaler::CheckQpResult::kNormalQp: { + result_.observed_enough_frames = true; + // After this line, |this| may be deleted. + DoCompleteTask(); + return; + } + case QualityScaler::CheckQpResult::kHighQp: { + result_.observed_enough_frames = true; + result_.qp_usage_reported = true; + state_ = State::kAwaitingQpUsageHandled; + rtc::scoped_refptr + callback = ConstructCallback(); + quality_scaler_->fast_rampup_ = false; + // After this line, |this| may be deleted. + quality_scaler_->handler_->OnReportQpUsageHigh(callback); + return; + } + case QualityScaler::CheckQpResult::kLowQp: { + result_.observed_enough_frames = true; + result_.qp_usage_reported = true; + state_ = State::kAwaitingQpUsageHandled; + rtc::scoped_refptr + callback = ConstructCallback(); + // After this line, |this| may be deleted. + quality_scaler_->handler_->OnReportQpUsageLow(callback); + return; + } + } + }), + GetCheckingQpDelayMs()); + } + + void OnQpUsageHandled(bool clear_qp_samples) { + RTC_DCHECK_EQ(state_, State::kAwaitingQpUsageHandled); + result_.clear_qp_samples = clear_qp_samples; + if (clear_qp_samples) + quality_scaler_->ClearSamples(); + DoCompleteTask(); + } + + bool HasCompletedTask() const { return state_ == State::kCompleted; } + + Result result() const { + RTC_DCHECK(HasCompletedTask()); + return result_; + } + + private: + enum class State { + kNotStarted, + kCheckingQp, + kAwaitingQpUsageHandled, + kCompleted, + }; + + // Defined after the definition of QualityScaler::CheckQpTaskHandlerCallback. + // Gets around a forward declaration issue. + rtc::scoped_refptr + ConstructCallback(); + + // Determines the sampling period of CheckQpTasks. + int64_t GetCheckingQpDelayMs() const { + RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_); + if (quality_scaler_->fast_rampup_) { + return quality_scaler_->sampling_period_ms_; + } + if (quality_scaler_->experiment_enabled_ && + !previous_task_result_.observed_enough_frames) { + // Use half the interval while waiting for enough frames. + return quality_scaler_->sampling_period_ms_ / 2; + } + if (!previous_task_result_.clear_qp_samples) { + // Check shortly again. + return quality_scaler_->sampling_period_ms_ / 8; + } + if (quality_scaler_->scale_factor_ && + !previous_task_result_.qp_usage_reported) { + // Last CheckQp did not call AdaptDown/Up, possibly reduce interval. + return quality_scaler_->sampling_period_ms_ * + quality_scaler_->scale_factor_.value(); + } + return quality_scaler_->sampling_period_ms_ * + quality_scaler_->initial_scale_factor_; + } + + void DoCompleteTask() { + RTC_DCHECK(state_ == State::kCheckingQp || + state_ == State::kAwaitingQpUsageHandled); + state_ = State::kCompleted; + // Starting the next task deletes the pending task. After this line, |this| + // has been deleted. + quality_scaler_->StartNextCheckQpTask(); + } + + QualityScaler* const quality_scaler_; + State state_; + const Result previous_task_result_; + Result result_; + + rtc::WeakPtrFactory weak_ptr_factory_; +}; + +class QualityScaler::CheckQpTaskHandlerCallback + : public QualityScalerQpUsageHandlerCallbackInterface { + public: + CheckQpTaskHandlerCallback( + rtc::WeakPtr check_qp_task) + : QualityScalerQpUsageHandlerCallbackInterface(), + check_qp_task_(std::move(check_qp_task)), + was_handled_(false) {} + + ~CheckQpTaskHandlerCallback() { RTC_DCHECK(was_handled_); } + + void OnQpUsageHandled(bool clear_qp_samples) { + RTC_DCHECK(!was_handled_); + was_handled_ = true; + if (!check_qp_task_) { + // The task has been cancelled through destruction; the result of the + // operation is ignored. + return; + } + check_qp_task_->OnQpUsageHandled(clear_qp_samples); + } + + private: + // The callback may outlive the QualityScaler and its task. + rtc::WeakPtr const check_qp_task_; + bool was_handled_; +}; + +rtc::scoped_refptr +QualityScaler::CheckQpTask::ConstructCallback() { + return new CheckQpTaskHandlerCallback(weak_ptr_factory_.GetWeakPtr()); +} + +QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler, VideoEncoder::QpThresholds thresholds) - : QualityScaler(observer, thresholds, kMeasureMs) {} + : QualityScaler(handler, thresholds, kMeasureMs) {} // Protected ctor, should not be called directly. -QualityScaler::QualityScaler(AdaptationObserverInterface* observer, +QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler, VideoEncoder::QpThresholds thresholds, int64_t sampling_period_ms) - : observer_(observer), + : handler_(handler), thresholds_(thresholds), sampling_period_ms_(sampling_period_ms), fast_rampup_(true), @@ -86,7 +265,6 @@ QualityScaler::QualityScaler(AdaptationObserverInterface* observer, framedrop_percent_media_opt_(5 * 30), framedrop_percent_all_(5 * 30), experiment_enabled_(QualityScalingExperiment::Enabled()), - observed_enough_frames_(false), min_frames_needed_( QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or( kMinFramesNeededToScale)), @@ -94,49 +272,33 @@ QualityScaler::QualityScaler(AdaptationObserverInterface* observer, .InitialScaleFactor() .value_or(kSamplePeriodScaleFactor)), scale_factor_( - QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()), - adapt_called_(false), - adapt_failed_(false) { + QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()) { RTC_DCHECK_RUN_ON(&task_checker_); if (experiment_enabled_) { config_ = QualityScalingExperiment::GetConfig(); qp_smoother_high_.reset(new QpSmoother(config_.alpha_high)); qp_smoother_low_.reset(new QpSmoother(config_.alpha_low)); } - RTC_DCHECK(observer_ != nullptr); - check_qp_task_ = RepeatingTaskHandle::DelayedStart( - TaskQueueBase::Current(), TimeDelta::Millis(GetSamplingPeriodMs()), - [this]() { - CheckQp(); - return TimeDelta::Millis(GetSamplingPeriodMs()); - }); + RTC_DCHECK(handler_ != nullptr); + StartNextCheckQpTask(); RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low << ", high: " << thresholds_.high; } QualityScaler::~QualityScaler() { RTC_DCHECK_RUN_ON(&task_checker_); - check_qp_task_.Stop(); } -int64_t QualityScaler::GetSamplingPeriodMs() const { +void QualityScaler::StartNextCheckQpTask() { RTC_DCHECK_RUN_ON(&task_checker_); - if (fast_rampup_) { - return sampling_period_ms_; + RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask()) + << "A previous CheckQpTask has not completed yet!"; + CheckQpTask::Result previous_task_result; + if (pending_qp_task_) { + previous_task_result = pending_qp_task_->result(); } - if (experiment_enabled_ && !observed_enough_frames_) { - // Use half the interval while waiting for enough frames. - return sampling_period_ms_ / 2; - } - if (adapt_failed_) { - // Check shortly again. - return sampling_period_ms_ / 8; - } - if (scale_factor_ && !adapt_called_) { - // Last CheckQp did not call AdaptDown/Up, possibly reduce interval. - return sampling_period_ms_ * scale_factor_.value(); - } - return sampling_period_ms_ * initial_scale_factor_; + pending_qp_task_ = std::make_unique(this, previous_task_result); + pending_qp_task_->StartDelayedTask(); } void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) { @@ -181,12 +343,10 @@ bool QualityScaler::QpFastFilterLow() const { return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false; } -void QualityScaler::CheckQp() { +QualityScaler::CheckQpResult QualityScaler::CheckQp() const { RTC_DCHECK_RUN_ON(&task_checker_); // Should be set through InitEncode -> Should be set by now. RTC_DCHECK_GE(thresholds_.low, 0); - adapt_failed_ = false; - adapt_called_ = false; // If we have not observed at least this many frames we can't make a good // scaling decision. @@ -194,10 +354,8 @@ void QualityScaler::CheckQp() { ? framedrop_percent_all_.Size() : framedrop_percent_media_opt_.Size(); if (frames < min_frames_needed_) { - observed_enough_frames_ = false; - return; + return CheckQpResult::kInsufficientSamples; } - observed_enough_frames_ = true; // Check if we should scale down due to high frame drop. const absl::optional drop_rate = @@ -206,8 +364,7 @@ void QualityScaler::CheckQp() { : framedrop_percent_media_opt_.GetAverageRoundedDown(); if (drop_rate && *drop_rate >= kFramedropPercentThreshold) { RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate; - ReportQpHigh(); - return; + return CheckQpResult::kHighQp; } // Check if we should scale up or down based on QP. @@ -221,38 +378,14 @@ void QualityScaler::CheckQp() { RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " (" << *avg_qp_low << ")."; if (*avg_qp_high > thresholds_.high) { - ReportQpHigh(); - return; + return CheckQpResult::kHighQp; } if (*avg_qp_low <= thresholds_.low) { // QP has been low. We want to try a higher resolution. - ReportQpLow(); - return; + return CheckQpResult::kLowQp; } } -} - -void QualityScaler::ReportQpLow() { - RTC_DCHECK_RUN_ON(&task_checker_); - ClearSamples(); - observer_->AdaptUp(VideoAdaptationReason::kQuality); - adapt_called_ = true; -} - -void QualityScaler::ReportQpHigh() { - RTC_DCHECK_RUN_ON(&task_checker_); - - if (observer_->AdaptDown(VideoAdaptationReason::kQuality)) { - ClearSamples(); - } else { - adapt_failed_ = true; - } - - // If we've scaled down, wait longer before scaling up again. - if (fast_rampup_) { - fast_rampup_ = false; - } - adapt_called_ = true; + return CheckQpResult::kNormalQp; } void QualityScaler::ClearSamples() { @@ -265,4 +398,13 @@ void QualityScaler::ClearSamples() { if (qp_smoother_low_) qp_smoother_low_->Reset(); } + +QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {} + +QualityScalerQpUsageHandlerCallbackInterface:: + QualityScalerQpUsageHandlerCallbackInterface() {} + +QualityScalerQpUsageHandlerCallbackInterface:: + ~QualityScalerQpUsageHandlerCallbackInterface() {} + } // namespace webrtc diff --git a/modules/video_coding/utility/quality_scaler.h b/modules/video_coding/utility/quality_scaler.h index d6fd599139..cfd2fced3f 100644 --- a/modules/video_coding/utility/quality_scaler.h +++ b/modules/video_coding/utility/quality_scaler.h @@ -17,45 +17,30 @@ #include #include "absl/types/optional.h" -#include "api/video/video_adaptation_reason.h" +#include "api/scoped_refptr.h" #include "api/video_codecs/video_encoder.h" #include "rtc_base/experiments/quality_scaling_experiment.h" #include "rtc_base/numerics/moving_average.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/ref_counted_object.h" #include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/task_queue.h" -#include "rtc_base/task_utils/repeating_task.h" namespace webrtc { -// An interface for signaling requests to limit or increase the resolution or -// framerate of the captured video stream. -// TODO(hbos): Can we remove AdaptationObserverInterface in favor of -// ResourceUsageListener? If we need to adapt that is because of resource usage. -// A multi-stream and multi-resource aware solution needs to sparate the notion -// of being resource constrained from the decision to downgrade a specific -// stream. -class AdaptationObserverInterface { - public: - // Called to signal that we can handle larger or more frequent frames. - virtual void AdaptUp(VideoAdaptationReason reason) = 0; - // Called to signal that the source should reduce the resolution or framerate. - // Returns false if a downgrade was requested but the request did not result - // in a new limiting resolution or fps. - virtual bool AdaptDown(VideoAdaptationReason reason) = 0; - - protected: - virtual ~AdaptationObserverInterface() {} -}; +class QualityScalerQpUsageHandlerCallbackInterface; +class QualityScalerQpUsageHandlerInterface; // QualityScaler runs asynchronously and monitors QP values of encoded frames. -// It holds a reference to an AdaptationObserverInterface implementation to -// signal an intent to scale up or down. +// It holds a reference to a QualityScalerQpUsageHandlerInterface implementation +// to signal an overuse or underuse of QP (which indicate a desire to scale the +// video stream down or up). class QualityScaler { public: - // Construct a QualityScaler with given |thresholds| and |observer|. + // Construct a QualityScaler with given |thresholds| and |handler|. // This starts the quality scaler periodically checking what the average QP // has been recently. - QualityScaler(AdaptationObserverInterface* observer, + QualityScaler(QualityScalerQpUsageHandlerInterface* handler, VideoEncoder::QpThresholds thresholds); virtual ~QualityScaler(); // Should be called each time a frame is dropped at encoding. @@ -69,21 +54,34 @@ class QualityScaler { // The following members declared protected for testing purposes. protected: - QualityScaler(AdaptationObserverInterface* observer, + QualityScaler(QualityScalerQpUsageHandlerInterface* handler, VideoEncoder::QpThresholds thresholds, int64_t sampling_period_ms); private: class QpSmoother; + class CheckQpTask; + class CheckQpTaskHandlerCallback; - void CheckQp(); + enum class CheckQpResult { + kInsufficientSamples, + kNormalQp, + kHighQp, + kLowQp, + }; + + // Starts checking for QP in a delayed task. When the resulting CheckQpTask + // completes, it will invoke this method again, ensuring that we always + // periodically check for QP. See CheckQpTask for more details. We never run + // more than one CheckQpTask at a time. + void StartNextCheckQpTask(); + + CheckQpResult CheckQp() const; void ClearSamples(); - void ReportQpLow(); - void ReportQpHigh(); - int64_t GetSamplingPeriodMs() const; - RepeatingTaskHandle check_qp_task_ RTC_GUARDED_BY(&task_checker_); - AdaptationObserverInterface* const observer_ RTC_GUARDED_BY(&task_checker_); + std::unique_ptr pending_qp_task_ RTC_GUARDED_BY(&task_checker_); + QualityScalerQpUsageHandlerInterface* const handler_ + RTC_GUARDED_BY(&task_checker_); SequenceChecker task_checker_; VideoEncoder::QpThresholds thresholds_ RTC_GUARDED_BY(&task_checker_); @@ -99,14 +97,55 @@ class QualityScaler { QualityScalingExperiment::Config config_ RTC_GUARDED_BY(&task_checker_); std::unique_ptr qp_smoother_high_ RTC_GUARDED_BY(&task_checker_); std::unique_ptr qp_smoother_low_ RTC_GUARDED_BY(&task_checker_); - bool observed_enough_frames_ RTC_GUARDED_BY(&task_checker_); const size_t min_frames_needed_; const double initial_scale_factor_; const absl::optional scale_factor_; - bool adapt_called_ RTC_GUARDED_BY(&task_checker_); - bool adapt_failed_ RTC_GUARDED_BY(&task_checker_); }; + +// Reacts to QP being too high or too low. For best quality, when QP is high it +// is desired to decrease the resolution or frame rate of the stream and when QP +// is low it is desired to increase the resolution or frame rate of the stream. +// Whether to reconfigure the stream is ultimately up to the handler, which is +// able to respond asynchronously. +class QualityScalerQpUsageHandlerInterface { + public: + virtual ~QualityScalerQpUsageHandlerInterface(); + + // Reacts to QP usage being too high or too low. The |callback| MUST be + // invoked when the handler is done, allowing the QualityScaler to resume + // checking for QP. + virtual void OnReportQpUsageHigh( + rtc::scoped_refptr + callback) = 0; + virtual void OnReportQpUsageLow( + rtc::scoped_refptr + callback) = 0; +}; + +// When QP is reported as high or low by the QualityScaler, it pauses checking +// for QP until the QP usage has been handled. When OnQpUsageHandled() is +// invoked, the QualityScaler resumes checking for QP. This ensures that if the +// stream is reconfigured in response to QP usage we do not include QP samples +// from before the reconfiguration the next time we check for QP. +// +// OnQpUsageHandled() MUST be invoked exactly once before this object is +// destroyed. +class QualityScalerQpUsageHandlerCallbackInterface + : public rtc::RefCountedObject { + public: + virtual ~QualityScalerQpUsageHandlerCallbackInterface(); + + // If |clear_qp_samples| is true, existing QP samples are cleared before the + // next time QualityScaler checks for QP. This is usually a good idea when the + // stream is reconfigured. If |clear_qp_samples| is false, samples are not + // cleared and QualityScaler increases its frequency of checking for QP. + virtual void OnQpUsageHandled(bool clear_qp_samples) = 0; + + protected: + QualityScalerQpUsageHandlerCallbackInterface(); +}; + } // namespace webrtc #endif // MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ diff --git a/modules/video_coding/utility/quality_scaler_unittest.cc b/modules/video_coding/utility/quality_scaler_unittest.cc index a000504607..275b327960 100644 --- a/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/modules/video_coding/utility/quality_scaler_unittest.cc @@ -13,7 +13,6 @@ #include #include -#include "api/video/video_adaptation_reason.h" #include "rtc_base/checks.h" #include "rtc_base/event.h" #include "rtc_base/task_queue_for_test.h" @@ -29,31 +28,45 @@ static const int kMinFramesNeededToScale = 60; // From quality_scaler.cc. static const size_t kDefaultTimeoutMs = 150; } // namespace -class MockAdaptationObserver : public AdaptationObserverInterface { +class MockQpUsageHandler : public QualityScalerQpUsageHandlerInterface { public: - virtual ~MockAdaptationObserver() {} + virtual ~MockQpUsageHandler() {} - void AdaptUp(VideoAdaptationReason r) override { - adapt_up_events_++; - event.Set(); - } - bool AdaptDown(VideoAdaptationReason r) override { + // QualityScalerQpUsageHandlerInterface implementation. + void OnReportQpUsageHigh( + rtc::scoped_refptr callback) + override { + callback_ = callback; adapt_down_events_++; event.Set(); - return true; + if (synchronously_invoke_callback) + callback_->OnQpUsageHandled(true); + } + + void OnReportQpUsageLow( + rtc::scoped_refptr callback) + override { + callback_ = callback; + adapt_up_events_++; + event.Set(); + if (synchronously_invoke_callback) + callback_->OnQpUsageHandled(true); } rtc::Event event; int adapt_up_events_ = 0; int adapt_down_events_ = 0; + bool synchronously_invoke_callback = true; + rtc::scoped_refptr callback_ = + nullptr; }; // Pass a lower sampling period to speed up the tests. class QualityScalerUnderTest : public QualityScaler { public: - explicit QualityScalerUnderTest(AdaptationObserverInterface* observer, + explicit QualityScalerUnderTest(QualityScalerQpUsageHandlerInterface* handler, VideoEncoder::QpThresholds thresholds) - : QualityScaler(observer, thresholds, 5) {} + : QualityScaler(handler, thresholds, 5) {} }; class QualityScalerTest : public ::testing::Test, @@ -70,11 +83,11 @@ class QualityScalerTest : public ::testing::Test, QualityScalerTest() : scoped_field_trial_(GetParam()), task_queue_("QualityScalerTestQueue"), - observer_(new MockAdaptationObserver()) { + handler_(new MockQpUsageHandler()) { task_queue_.SendTask( [this] { qs_ = std::unique_ptr(new QualityScalerUnderTest( - observer_.get(), VideoEncoder::QpThresholds(kLowQp, kHighQp))); + handler_.get(), VideoEncoder::QpThresholds(kLowQp, kHighQp))); }, RTC_FROM_HERE); } @@ -108,7 +121,7 @@ class QualityScalerTest : public ::testing::Test, test::ScopedFieldTrials scoped_field_trial_; TaskQueueForTest task_queue_; std::unique_ptr qs_; - std::unique_ptr observer_; + std::unique_ptr handler_; }; INSTANTIATE_TEST_SUITE_P( @@ -120,25 +133,25 @@ INSTANTIATE_TEST_SUITE_P( TEST_P(QualityScalerTest, DownscalesAfterContinuousFramedrop) { task_queue_.SendTask([this] { TriggerScale(kScaleDown); }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(1, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, KeepsScaleAtHighQp) { task_queue_.SendTask([this] { TriggerScale(kKeepScaleAtHighQp); }, RTC_FROM_HERE); - EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(0, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, DownscalesAboveHighQp) { task_queue_.SendTask([this] { TriggerScale(kScaleDownAboveHighQp); }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(1, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { @@ -151,9 +164,9 @@ TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { } }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(1, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) { @@ -165,9 +178,9 @@ TEST_P(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) { } }, RTC_FROM_HERE); - EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(0, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsIfFieldTrialEnabled) { @@ -181,35 +194,35 @@ TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsIfFieldTrialEnabled) { } }, RTC_FROM_HERE); - EXPECT_EQ(kDownScaleExpected, observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(kDownScaleExpected ? 1 : 0, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_EQ(kDownScaleExpected, handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(kDownScaleExpected ? 1 : 0, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, KeepsScaleOnNormalQp) { task_queue_.SendTask([this] { TriggerScale(kKeepScaleAboveLowQp); }, RTC_FROM_HERE); - EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(0, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, UpscalesAfterLowQp) { task_queue_.SendTask([this] { TriggerScale(kScaleUp); }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(0, observer_->adapt_down_events_); - EXPECT_EQ(1, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, ScalesDownAndBackUp) { task_queue_.SendTask([this] { TriggerScale(kScaleDown); }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(1, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); task_queue_.SendTask([this] { TriggerScale(kScaleUp); }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(1, observer_->adapt_down_events_); - EXPECT_EQ(1, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) { @@ -221,7 +234,7 @@ TEST_P(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) { } }, RTC_FROM_HERE); - EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs)); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs)); task_queue_.SendTask( [this] { // Send 1 more. Enough frames observed, should result in an adapt @@ -229,9 +242,9 @@ TEST_P(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) { qs_->ReportQp(kLowQp, 0); }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(0, observer_->adapt_down_events_); - EXPECT_EQ(1, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); // Samples should be cleared after an adapt request. task_queue_.SendTask( @@ -240,9 +253,9 @@ TEST_P(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) { qs_->ReportQp(kLowQp, 0); }, RTC_FROM_HERE); - EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(0, observer_->adapt_down_events_); - EXPECT_EQ(1, observer_->adapt_up_events_); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); } TEST_P(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) { @@ -253,9 +266,9 @@ TEST_P(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) { } }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(1, observer_->adapt_down_events_); - EXPECT_EQ(0, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); // Samples cleared. task_queue_.SendTask( [this] { @@ -264,9 +277,39 @@ TEST_P(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) { } }, RTC_FROM_HERE); - EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs)); - EXPECT_EQ(1, observer_->adapt_down_events_); - EXPECT_EQ(1, observer_->adapt_up_events_); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, CheckingQpAgainRequiresResolvingCallback) { + handler_->synchronously_invoke_callback = false; + task_queue_.SendTask([this] { TriggerScale(kScaleDown); }, RTC_FROM_HERE); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(1, handler_->adapt_down_events_); + // Without invoking the callback, another downscale should not happen. + handler_->event.Reset(); + rtc::Event event; + task_queue_.SendTask( + [this, &event] { + TriggerScale(kScaleDown); + event.Set(); + }, + RTC_FROM_HERE); + EXPECT_TRUE(event.Wait(kDefaultTimeoutMs)); + EXPECT_FALSE(handler_->event.Wait(0)); + EXPECT_EQ(1, handler_->adapt_down_events_); + // Resume checking for QP again by invoking the callback. + task_queue_.SendTask( + [this] { + handler_->callback_->OnQpUsageHandled(true); + TriggerScale(kScaleDown); + }, + RTC_FROM_HERE); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs)); + EXPECT_EQ(2, handler_->adapt_down_events_); + task_queue_.SendTask([this] { handler_->callback_->OnQpUsageHandled(true); }, + RTC_FROM_HERE); } } // namespace webrtc diff --git a/video/adaptation/encode_usage_resource.cc b/video/adaptation/encode_usage_resource.cc index 45cba1ad79..7a42878aa9 100644 --- a/video/adaptation/encode_usage_resource.cc +++ b/video/adaptation/encode_usage_resource.cc @@ -66,15 +66,12 @@ void EncodeUsageResource::OnEncodeCompleted( encode_duration_us); } -void EncodeUsageResource::AdaptUp(VideoAdaptationReason reason) { - RTC_DCHECK_EQ(reason, VideoAdaptationReason::kCpu); +void EncodeUsageResource::AdaptUp() { OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); } -bool EncodeUsageResource::AdaptDown(VideoAdaptationReason reason) { - RTC_DCHECK_EQ(reason, VideoAdaptationReason::kCpu); - return OnResourceUsageStateMeasured(ResourceUsageState::kOveruse) != - ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; +void EncodeUsageResource::AdaptDown() { + OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); } int EncodeUsageResource::TargetFrameRateAsInt() { diff --git a/video/adaptation/encode_usage_resource.h b/video/adaptation/encode_usage_resource.h index a41211ee03..7147569ffb 100644 --- a/video/adaptation/encode_usage_resource.h +++ b/video/adaptation/encode_usage_resource.h @@ -17,7 +17,6 @@ #include "absl/types/optional.h" #include "api/video/video_adaptation_reason.h" #include "call/adaptation/resource.h" -#include "modules/video_coding/utility/quality_scaler.h" #include "video/adaptation/overuse_frame_detector.h" namespace webrtc { @@ -27,10 +26,8 @@ namespace webrtc { // indirectly by usage in the ResourceAdaptationProcessor (which is only tested // because of its usage in VideoStreamEncoder); all tests are currently in // video_stream_encoder_unittest.cc. -// TODO(https://crbug.com/webrtc/11222): Move this class to the -// video/adaptation/ subdirectory. class EncodeUsageResource : public Resource, - public AdaptationObserverInterface { + public OveruseFrameDetectorObserverInterface { public: explicit EncodeUsageResource( std::unique_ptr overuse_detector); @@ -48,11 +45,9 @@ class EncodeUsageResource : public Resource, int64_t capture_time_us, absl::optional encode_duration_us); - // AdaptationObserverInterface implementation. - // TODO(https://crbug.com/webrtc/11222, 11172): This resource also needs to - // signal when its stable to support multi-stream aware modules. - void AdaptUp(VideoAdaptationReason reason) override; - bool AdaptDown(VideoAdaptationReason reason) override; + // OveruseFrameDetectorObserverInterface implementation. + void AdaptUp() override; + void AdaptDown() override; std::string name() const override { return "EncoderUsageResource"; } diff --git a/video/adaptation/overuse_frame_detector.cc b/video/adaptation/overuse_frame_detector.cc index abd6f953de..9703ac8025 100644 --- a/video/adaptation/overuse_frame_detector.cc +++ b/video/adaptation/overuse_frame_detector.cc @@ -20,7 +20,6 @@ #include #include -#include "api/video/video_adaptation_reason.h" #include "api/video/video_frame.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -65,8 +64,6 @@ const float kMaxSampleDiffMarginFactor = 1.35f; const int kMinFramerate = 7; const int kMaxFramerate = 30; -const auto kScaleReasonCpu = VideoAdaptationReason::kCpu; - // Class for calculating the processing usage on the send-side (the average // processing time of a frame divided by the average time difference between // captured frames). @@ -543,7 +540,7 @@ OveruseFrameDetector::~OveruseFrameDetector() {} void OveruseFrameDetector::StartCheckForOveruse( TaskQueueBase* task_queue_base, const CpuOveruseOptions& options, - AdaptationObserverInterface* overuse_observer) { + OveruseFrameDetectorObserverInterface* overuse_observer) { RTC_DCHECK_RUN_ON(&task_checker_); RTC_DCHECK(!check_overuse_task_.Running()); RTC_DCHECK(overuse_observer != nullptr); @@ -633,7 +630,7 @@ void OveruseFrameDetector::FrameSent(uint32_t timestamp, } void OveruseFrameDetector::CheckForOveruse( - AdaptationObserverInterface* observer) { + OveruseFrameDetectorObserverInterface* observer) { RTC_DCHECK_RUN_ON(&task_checker_); RTC_DCHECK(observer); ++num_process_times_; @@ -666,12 +663,12 @@ void OveruseFrameDetector::CheckForOveruse( checks_above_threshold_ = 0; ++num_overuse_detections_; - observer->AdaptDown(kScaleReasonCpu); + observer->AdaptDown(); } else if (IsUnderusing(*encode_usage_percent_, now_ms)) { last_rampup_time_ms_ = now_ms; in_quick_rampup_ = true; - observer->AdaptUp(kScaleReasonCpu); + observer->AdaptUp(); } int rampup_delay = diff --git a/video/adaptation/overuse_frame_detector.h b/video/adaptation/overuse_frame_detector.h index e8c667dfdc..16217fff84 100644 --- a/video/adaptation/overuse_frame_detector.h +++ b/video/adaptation/overuse_frame_detector.h @@ -17,7 +17,6 @@ #include "absl/types/optional.h" #include "api/task_queue/task_queue_base.h" #include "api/video/video_stream_encoder_observer.h" -#include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/numerics/exp_filter.h" @@ -47,6 +46,17 @@ struct CpuOveruseOptions { int filter_time_ms; // Time constant for averaging }; +class OveruseFrameDetectorObserverInterface { + public: + // Called to signal that we can handle larger or more frequent frames. + virtual void AdaptUp() = 0; + // Called to signal that the source should reduce the resolution or framerate. + virtual void AdaptDown() = 0; + + protected: + virtual ~OveruseFrameDetectorObserverInterface() {} +}; + // Use to detect system overuse based on the send-side processing time of // incoming frames. All methods must be called on a single task queue but it can // be created and destroyed on an arbitrary thread. @@ -58,9 +68,10 @@ class OveruseFrameDetector { virtual ~OveruseFrameDetector(); // Start to periodically check for overuse. - void StartCheckForOveruse(TaskQueueBase* task_queue_base, - const CpuOveruseOptions& options, - AdaptationObserverInterface* overuse_observer); + void StartCheckForOveruse( + TaskQueueBase* task_queue_base, + const CpuOveruseOptions& options, + OveruseFrameDetectorObserverInterface* overuse_observer); // StopCheckForOveruse must be called before destruction if // StartCheckForOveruse has been called. @@ -105,7 +116,7 @@ class OveruseFrameDetector { protected: // Protected for test purposes. - void CheckForOveruse(AdaptationObserverInterface* overuse_observer); + void CheckForOveruse(OveruseFrameDetectorObserverInterface* overuse_observer); void SetOptions(const CpuOveruseOptions& options); CpuOveruseOptions options_; diff --git a/video/adaptation/overuse_frame_detector_unittest.cc b/video/adaptation/overuse_frame_detector_unittest.cc index d3eeb53905..bb34224b02 100644 --- a/video/adaptation/overuse_frame_detector_unittest.cc +++ b/video/adaptation/overuse_frame_detector_unittest.cc @@ -36,25 +36,22 @@ const int kFrameIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; const int kProcessTimeUs = 5 * rtc::kNumMicrosecsPerMillisec; } // namespace -class MockCpuOveruseObserver : public AdaptationObserverInterface { +class MockCpuOveruseObserver : public OveruseFrameDetectorObserverInterface { public: MockCpuOveruseObserver() {} virtual ~MockCpuOveruseObserver() {} - MOCK_METHOD1(AdaptUp, void(VideoAdaptationReason)); - MOCK_METHOD1(AdaptDown, bool(VideoAdaptationReason)); + MOCK_METHOD0(AdaptUp, void()); + MOCK_METHOD0(AdaptDown, void()); }; -class CpuOveruseObserverImpl : public AdaptationObserverInterface { +class CpuOveruseObserverImpl : public OveruseFrameDetectorObserverInterface { public: CpuOveruseObserverImpl() : overuse_(0), normaluse_(0) {} virtual ~CpuOveruseObserverImpl() {} - bool AdaptDown(VideoAdaptationReason) override { - ++overuse_; - return true; - } - void AdaptUp(VideoAdaptationReason) override { ++normaluse_; } + void AdaptDown() override { ++overuse_; } + void AdaptUp() override { ++normaluse_; } int overuse_; int normaluse_; @@ -232,11 +229,9 @@ class OveruseFrameDetectorTest : public ::testing::Test, CpuOveruseOptions options_; rtc::ScopedFakeClock clock_; MockCpuOveruseObserver mock_observer_; - AdaptationObserverInterface* observer_; + OveruseFrameDetectorObserverInterface* observer_; std::unique_ptr overuse_detector_; int encode_usage_percent_ = -1; - - static const auto reason_ = VideoAdaptationReason::kCpu; }; // UsagePercent() > high_encode_usage_threshold_percent => overuse. @@ -244,26 +239,26 @@ class OveruseFrameDetectorTest : public ::testing::Test, TEST_F(OveruseFrameDetectorTest, TriggerOveruse) { // usage > high => overuse overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); TriggerOveruse(options_.high_threshold_consecutive_count); } TEST_F(OveruseFrameDetectorTest, OveruseAndRecover) { // usage > high => overuse overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); TriggerOveruse(options_.high_threshold_consecutive_count); // usage < low => underuse - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); TriggerUnderuse(); } TEST_F(OveruseFrameDetectorTest, DoubleOveruseAndRecover) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(2); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(2); TriggerOveruse(options_.high_threshold_consecutive_count); TriggerOveruse(options_.high_threshold_consecutive_count); - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); TriggerUnderuse(); } @@ -284,8 +279,8 @@ TEST_F(OveruseFrameDetectorTest, TriggerUnderuseWithMinProcessCount) { TEST_F(OveruseFrameDetectorTest, ConstantOveruseGivesNoNormalUsage) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(0); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(64); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(64); for (size_t i = 0; i < 64; ++i) { TriggerOveruse(options_.high_threshold_consecutive_count); } @@ -293,7 +288,7 @@ TEST_F(OveruseFrameDetectorTest, ConstantOveruseGivesNoNormalUsage) { TEST_F(OveruseFrameDetectorTest, ConsecutiveCountTriggersOveruse) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); options_.high_threshold_consecutive_count = 2; overuse_detector_->SetOptions(options_); TriggerOveruse(2); @@ -301,7 +296,7 @@ TEST_F(OveruseFrameDetectorTest, ConsecutiveCountTriggersOveruse) { TEST_F(OveruseFrameDetectorTest, IncorrectConsecutiveCountTriggersNoOveruse) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); options_.high_threshold_consecutive_count = 2; overuse_detector_->SetOptions(options_); TriggerOveruse(1); @@ -374,7 +369,7 @@ TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) { TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(::testing::AtLeast(1)); static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; static const size_t kNumFramesEncodingDelay = 3; VideoFrame frame = @@ -401,7 +396,7 @@ TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) { // >85% encoding time should trigger overuse. overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(::testing::AtLeast(1)); static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; static const int kDelayUs = 30 * rtc::kNumMicrosecsPerMillisec; VideoFrame frame = @@ -442,7 +437,7 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { rtc::Event event; // Expect NormalUsage(). When called, stop the |overuse_detector_| and then // set |event| to end the test. - EXPECT_CALL(mock_observer_, AdaptUp(reason_)) + EXPECT_CALL(mock_observer_, AdaptUp()) .WillOnce(InvokeWithoutArgs([this, &event] { overuse_detector_->StopCheckForOveruse(); event.Set(); @@ -470,7 +465,7 @@ TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) { // Processing time just below over use limit given kEncodeMaxFrameRate. int64_t processing_time_us = (98 * OveruseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, processing_time_us); @@ -480,7 +475,7 @@ TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) { // Simulate frame rate reduction and normal usage. frame_interval_us = rtc::kNumMicrosecsPerSec / kEncodeMaxFrameRate; overuse_detector_->OnTargetFramerateUpdated(kEncodeMaxFrameRate); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, processing_time_us); @@ -490,7 +485,7 @@ TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) { // Reduce processing time to trigger underuse. processing_time_us = (98 * UnderuseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(1); InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, processing_time_us); overuse_detector_->CheckForOveruse(observer_); @@ -506,7 +501,7 @@ TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) { // Processing time just below over use limit given kEncodeMaxFrameRate. int64_t processing_time_us = (98 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, processing_time_us); @@ -516,7 +511,7 @@ TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) { // Over the limit to overuse. processing_time_us = (102 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, processing_time_us); @@ -525,7 +520,7 @@ TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) { // Reduce input frame rate. Should still trigger overuse. overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate - 1); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, processing_time_us); @@ -548,7 +543,7 @@ TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) { // Processing time just below overuse limit given kMaxFrameRate. int64_t processing_time_us = (98 * max_processing_time_us) / 100; - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, kHeight, processing_time_us); @@ -557,7 +552,7 @@ TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) { // Go above limit, trigger overuse. processing_time_us = (102 * max_processing_time_us) / 100; - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, kHeight, processing_time_us); @@ -566,7 +561,7 @@ TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) { // Increase frame interval, should still trigger overuse. max_frame_interval_us *= 2; - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, kHeight, processing_time_us); @@ -581,8 +576,8 @@ TEST_F(OveruseFrameDetectorTest, NoOveruseForLargeRandomFrameInterval) { // behavior is improved in this scenario, with only AdaptUp events, // and estimated load closer to the true average. - // EXPECT_CALL(mock_observer_, AdaptDown(_)).Times(0); - // EXPECT_CALL(mock_observer_, AdaptUp(reason_)) + // EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + // EXPECT_CALL(mock_observer_, AdaptUp()) // .Times(::testing::AtLeast(1)); overuse_detector_->SetOptions(options_); @@ -610,8 +605,8 @@ TEST_F(OveruseFrameDetectorTest, NoOveruseForRandomFrameIntervalWithReset) { // TODO(bugs.webrtc.org/8504): When new estimator is relanded, // behavior is improved in this scenario, and we get AdaptUp events. overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(_)).Times(0); - // EXPECT_CALL(mock_observer_, AdaptUp(reason_)) + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + // EXPECT_CALL(mock_observer_, AdaptUp()) // .Times(::testing::AtLeast(1)); const int kNumFrames = 500; @@ -639,7 +634,7 @@ TEST_F(OveruseFrameDetectorTest, NoOveruseForRandomFrameIntervalWithReset) { // Load estimate should be based on the maximum encode time per input frame. TEST_F(OveruseFrameDetectorTest, NoOveruseForSimulcast) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(_)).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); constexpr int kNumFrames = 500; constexpr int kEncodeTimesUs[] = { @@ -726,26 +721,26 @@ class OveruseFrameDetectorTest2 : public OveruseFrameDetectorTest { TEST_F(OveruseFrameDetectorTest2, TriggerOveruse) { // usage > high => overuse overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); TriggerOveruse(options_.high_threshold_consecutive_count); } TEST_F(OveruseFrameDetectorTest2, OveruseAndRecover) { // usage > high => overuse overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); TriggerOveruse(options_.high_threshold_consecutive_count); // usage < low => underuse - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); TriggerUnderuse(); } TEST_F(OveruseFrameDetectorTest2, DoubleOveruseAndRecover) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(2); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(2); TriggerOveruse(options_.high_threshold_consecutive_count); TriggerOveruse(options_.high_threshold_consecutive_count); - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); TriggerUnderuse(); } @@ -766,22 +761,22 @@ TEST_F(OveruseFrameDetectorTest2, TriggerUnderuseWithMinProcessCount) { TEST_F(OveruseFrameDetectorTest2, ConstantOveruseGivesNoNormalUsage) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(0); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(64); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(64); for (size_t i = 0; i < 64; ++i) { TriggerOveruse(options_.high_threshold_consecutive_count); } } TEST_F(OveruseFrameDetectorTest2, ConsecutiveCountTriggersOveruse) { - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); options_.high_threshold_consecutive_count = 2; overuse_detector_->SetOptions(options_); TriggerOveruse(2); } TEST_F(OveruseFrameDetectorTest2, IncorrectConsecutiveCountTriggersNoOveruse) { - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); options_.high_threshold_consecutive_count = 2; overuse_detector_->SetOptions(options_); TriggerOveruse(1); @@ -856,7 +851,7 @@ TEST_F(OveruseFrameDetectorTest2, InitialProcessingUsage) { TEST_F(OveruseFrameDetectorTest2, MeasuresMultipleConcurrentSamples) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(::testing::AtLeast(1)); static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; static const size_t kNumFramesEncodingDelay = 3; VideoFrame frame = @@ -883,7 +878,7 @@ TEST_F(OveruseFrameDetectorTest2, MeasuresMultipleConcurrentSamples) { TEST_F(OveruseFrameDetectorTest2, UpdatesExistingSamples) { // >85% encoding time should trigger overuse. overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(::testing::AtLeast(1)); static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; static const int kDelayUs = 30 * rtc::kNumMicrosecsPerMillisec; VideoFrame frame = @@ -924,7 +919,7 @@ TEST_F(OveruseFrameDetectorTest2, RunOnTqNormalUsage) { rtc::Event event; // Expect NormalUsage(). When called, stop the |overuse_detector_| and then // set |event| to end the test. - EXPECT_CALL(mock_observer_, AdaptUp(reason_)) + EXPECT_CALL(mock_observer_, AdaptUp()) .WillOnce(InvokeWithoutArgs([this, &event] { overuse_detector_->StopCheckForOveruse(); event.Set(); @@ -946,8 +941,8 @@ TEST_F(OveruseFrameDetectorTest2, RunOnTqNormalUsage) { // to encode. TEST_F(OveruseFrameDetectorTest2, NoOveruseForLargeRandomFrameInterval) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(_)).Times(0); - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); const int kNumFrames = 500; const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; @@ -966,8 +961,8 @@ TEST_F(OveruseFrameDetectorTest2, NoOveruseForLargeRandomFrameInterval) { // exceeding the timeout interval. TEST_F(OveruseFrameDetectorTest2, NoOveruseForRandomFrameIntervalWithReset) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(_)).Times(0); - EXPECT_CALL(mock_observer_, AdaptUp(reason_)).Times(::testing::AtLeast(1)); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); const int kNumFrames = 500; const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; @@ -1005,7 +1000,7 @@ TEST_F(OveruseFrameDetectorTest2, ToleratesOutOfOrderFrames) { // Load estimate should be based on the maximum encode time per input frame. TEST_F(OveruseFrameDetectorTest2, NoOveruseForSimulcast) { overuse_detector_->SetOptions(options_); - EXPECT_CALL(mock_observer_, AdaptDown(_)).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); constexpr int kNumFrames = 500; constexpr int kEncodeTimesUs[] = { diff --git a/video/adaptation/quality_scaler_resource.cc b/video/adaptation/quality_scaler_resource.cc index 1d5f7dd0a4..42271f9324 100644 --- a/video/adaptation/quality_scaler_resource.cc +++ b/video/adaptation/quality_scaler_resource.cc @@ -62,15 +62,18 @@ void QualityScalerResource::OnFrameDropped( } } -void QualityScalerResource::AdaptUp(VideoAdaptationReason reason) { - RTC_DCHECK_EQ(reason, VideoAdaptationReason::kQuality); - OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); +void QualityScalerResource::OnReportQpUsageHigh( + rtc::scoped_refptr callback) { + bool clear_qp_samples = + OnResourceUsageStateMeasured(ResourceUsageState::kOveruse) != + ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; + callback->OnQpUsageHandled(clear_qp_samples); } -bool QualityScalerResource::AdaptDown(VideoAdaptationReason reason) { - RTC_DCHECK_EQ(reason, VideoAdaptationReason::kQuality); - return OnResourceUsageStateMeasured(ResourceUsageState::kOveruse) != - ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; +void QualityScalerResource::OnReportQpUsageLow( + rtc::scoped_refptr callback) { + OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); + callback->OnQpUsageHandled(true); } } // namespace webrtc diff --git a/video/adaptation/quality_scaler_resource.h b/video/adaptation/quality_scaler_resource.h index 77bb60690f..eb7d22a8fd 100644 --- a/video/adaptation/quality_scaler_resource.h +++ b/video/adaptation/quality_scaler_resource.h @@ -26,10 +26,8 @@ namespace webrtc { // indirectly by usage in the ResourceAdaptationProcessor (which is only tested // because of its usage in VideoStreamEncoder); all tests are currently in // video_stream_encoder_unittest.cc. -// TODO(https://crbug.com/webrtc/11222): Move this class to the -// video/adaptation/ subdirectory. class QualityScalerResource : public Resource, - public AdaptationObserverInterface { + public QualityScalerQpUsageHandlerInterface { public: QualityScalerResource(); @@ -44,11 +42,13 @@ class QualityScalerResource : public Resource, int64_t time_sent_in_us); void OnFrameDropped(EncodedImageCallback::DropReason reason); - // AdaptationObserverInterface implementation. - // TODO(https://crbug.com/webrtc/11222, 11172): This resource also needs to - // signal when its stable to support multi-stream aware modules. - void AdaptUp(VideoAdaptationReason reason) override; - bool AdaptDown(VideoAdaptationReason reason) override; + // QualityScalerQpUsageHandlerInterface implementation. + void OnReportQpUsageHigh( + rtc::scoped_refptr callback) + override; + void OnReportQpUsageLow( + rtc::scoped_refptr callback) + override; std::string name() const override { return "QualityScalerResource"; }