diff --git a/video/overuse_frame_detector.cc b/video/overuse_frame_detector.cc index 8519ce7804..7b64c10597 100644 --- a/video/overuse_frame_detector.cc +++ b/video/overuse_frame_detector.cc @@ -23,6 +23,7 @@ #include "common_video/include/frame_callback.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" #include "rtc_base/timeutils.h" #include "system_wrappers/include/field_trial.h" @@ -47,15 +48,31 @@ const double kRampUpBackoffFactor = 2.0; // Max number of overuses detected before always applying the rampup delay. const int kMaxOverusesBeforeApplyRampupDelay = 4; +// The maximum exponent to use in VCMExpFilter. +const float kMaxExp = 7.0f; +// Default value used before first reconfiguration. +const int kDefaultFrameRate = 30; +// Default sample diff, default frame rate. +const float kDefaultSampleDiffMs = 1000.0f / kDefaultFrameRate; +// A factor applied to the sample diff on OnTargetFramerateUpdated to determine +// a max limit for the sample diff. For instance, with a framerate of 30fps, +// the sample diff is capped to (1000 / 30) * 1.35 = 45ms. This prevents +// triggering too soon if there are individual very large outliers. +const float kMaxSampleDiffMarginFactor = 1.35f; +// Minimum framerate allowed for usage calculation. This prevents crazy long +// encode times from being accepted if the frame rate happens to be low. +const int kMinFramerate = 7; +const int kMaxFramerate = 30; + const auto kScaleReasonCpu = AdaptationObserverInterface::AdaptReason::kCpu; } // namespace CpuOveruseOptions::CpuOveruseOptions() : high_encode_usage_threshold_percent(85), frame_timeout_interval_ms(1500), + min_frame_samples(120), min_process_count(3), - high_threshold_consecutive_count(2), - filter_time_ms(5000) { + high_threshold_consecutive_count(2) { #if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) // This is proof-of-concept code for letting the physical core count affect // the interval into which we attempt to scale. For now, the code is Mac OS @@ -106,43 +123,72 @@ CpuOveruseOptions::CpuOveruseOptions() class OveruseFrameDetector::SendProcessingUsage { public: explicit SendProcessingUsage(const CpuOveruseOptions& options) - : options_(options) { + : kWeightFactorFrameDiff(0.998f), + kWeightFactorProcessing(0.995f), + kInitialSampleDiffMs(40.0f), + count_(0), + options_(options), + max_sample_diff_ms_(kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor), + filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)), + filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) { Reset(); } virtual ~SendProcessingUsage() {} void Reset() { - // Start in between the underuse and overuse threshold. - load_estimate_ = (options_.low_encode_usage_threshold_percent + - options_.high_encode_usage_threshold_percent) / - 200.0; + count_ = 0; + max_sample_diff_ms_ = kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor; + filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff); + filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs); + filtered_processing_ms_->Reset(kWeightFactorProcessing); + filtered_processing_ms_->Apply(1.0f, InitialProcessingMs()); } - void AddSample(double encode_time, double diff_time) { - RTC_CHECK_GE(diff_time, 0.0); + void SetMaxSampleDiffMs(float diff_ms) { max_sample_diff_ms_ = diff_ms; } - // Use the filter update - // - // load <-- x/d (1-exp (-d/T)) + exp (-d/T) load - // - // where we must take care for small d, using the proper limit - // (1 - exp(-d/tau)) / d = 1/tau - d/2tau^2 + O(d^2) - double tau = (1e-3 * options_.filter_time_ms); - double e = diff_time / tau; - double c; - if (e < 0.0001) { - c = (1 - e / 2) / tau; - } else { - c = -expm1(-e) / diff_time; + void AddCaptureSample(float sample_ms) { + float exp = sample_ms / kDefaultSampleDiffMs; + exp = std::min(exp, kMaxExp); + filtered_frame_diff_ms_->Apply(exp, sample_ms); + } + + void AddSample(float processing_ms, int64_t diff_last_sample_ms) { + ++count_; + float exp = diff_last_sample_ms / kDefaultSampleDiffMs; + exp = std::min(exp, kMaxExp); + filtered_processing_ms_->Apply(exp, processing_ms); + } + + virtual int Value() { + if (count_ < static_cast(options_.min_frame_samples)) { + return static_cast(InitialUsageInPercent() + 0.5f); } - load_estimate_ = c * encode_time + exp(-e) * load_estimate_; + float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f); + frame_diff_ms = std::min(frame_diff_ms, max_sample_diff_ms_); + float encode_usage_percent = + 100.0f * filtered_processing_ms_->filtered() / frame_diff_ms; + return static_cast(encode_usage_percent + 0.5); } - virtual int Value() { return static_cast(100.0 * load_estimate_ + 0.5); } - private: + float InitialUsageInPercent() const { + // Start in between the underuse and overuse threshold. + return (options_.low_encode_usage_threshold_percent + + options_.high_encode_usage_threshold_percent) / 2.0f; + } + + float InitialProcessingMs() const { + return InitialUsageInPercent() * kInitialSampleDiffMs / 100; + } + + const float kWeightFactorFrameDiff; + const float kWeightFactorProcessing; + const float kInitialSampleDiffMs; + uint64_t count_; const CpuOveruseOptions options_; - double load_estimate_; + float max_sample_diff_ms_; + std::unique_ptr filtered_processing_ms_; + std::unique_ptr filtered_frame_diff_ms_; }; // Class used for manual testing of overuse, enabled via field trial flag. @@ -172,7 +218,6 @@ class OveruseFrameDetector::OverdoseInjector int64_t now_ms = rtc::TimeMillis(); if (last_toggling_ms_ == -1) { last_toggling_ms_ = now_ms; - } else { switch (state_) { case State::kNormal: @@ -210,6 +255,7 @@ class OveruseFrameDetector::OverdoseInjector overried_usage_value.emplace(5); break; } + return overried_usage_value.value_or(SendProcessingUsage::Value()); } @@ -302,6 +348,7 @@ OveruseFrameDetector::OveruseFrameDetector( last_capture_time_us_(-1), last_processed_capture_time_us_(-1), num_pixels_(0), + max_framerate_(kDefaultFrameRate), last_overuse_time_ms_(-1), checks_above_threshold_(0), num_overuse_detections_(0), @@ -352,41 +399,93 @@ bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now_us) const { options_.frame_timeout_interval_ms * rtc::kNumMicrosecsPerMillisec; } -void OveruseFrameDetector::ResetAll() { +void OveruseFrameDetector::ResetAll(int num_pixels) { + // Reset state, as a result resolution being changed. Do not however change + // the current frame rate back to the default. RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + num_pixels_ = num_pixels; usage_->Reset(); + frame_timing_.clear(); last_capture_time_us_ = -1; last_processed_capture_time_us_ = -1; num_process_times_ = 0; metrics_ = rtc::Optional(); + OnTargetFramerateUpdated(max_framerate_); } -void OveruseFrameDetector::FrameCaptured(int width, int height) { +void OveruseFrameDetector::OnTargetFramerateUpdated(int framerate_fps) { RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); - - if (FrameSizeChanged(width * height)) { - ResetAll(); - num_pixels_ = width * height; - } + RTC_DCHECK_GE(framerate_fps, 0); + max_framerate_ = std::min(kMaxFramerate, framerate_fps); + usage_->SetMaxSampleDiffMs((1000 / std::max(kMinFramerate, max_framerate_)) * + kMaxSampleDiffMarginFactor); } -void OveruseFrameDetector::FrameEncoded(int64_t capture_time_us, - int64_t encode_duration_us) { +void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame, + int64_t time_when_first_seen_us) { RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); - if (FrameTimeoutDetected(capture_time_us)) { - ResetAll(); - } else if (last_capture_time_us_ != -1) { - usage_->AddSample(1e-6 * encode_duration_us, - 1e-6 * (capture_time_us - last_capture_time_us_)); - } - last_capture_time_us_ = capture_time_us; - EncodedFrameTimeMeasured(encode_duration_us / rtc::kNumMicrosecsPerMillisec); - if (encoder_timing_) { - // TODO(nisse): Update encoder_timing_ to also use us units. - encoder_timing_->OnEncodeTiming( - capture_time_us / rtc::kNumMicrosecsPerMillisec, - encode_duration_us / rtc::kNumMicrosecsPerMillisec); + if (FrameSizeChanged(frame.width() * frame.height()) || + FrameTimeoutDetected(time_when_first_seen_us)) { + ResetAll(frame.width() * frame.height()); + } + + if (last_capture_time_us_ != -1) + usage_->AddCaptureSample( + 1e-3 * (time_when_first_seen_us - last_capture_time_us_)); + + last_capture_time_us_ = time_when_first_seen_us; + + frame_timing_.push_back(FrameTiming(frame.timestamp_us(), frame.timestamp(), + time_when_first_seen_us)); +} + +void OveruseFrameDetector::FrameSent(uint32_t timestamp, + int64_t time_sent_in_us) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + // Delay before reporting actual encoding time, used to have the ability to + // detect total encoding time when encoding more than one layer. Encoding is + // here assumed to finish within a second (or that we get enough long-time + // samples before one second to trigger an overuse even when this is not the + // case). + static const int64_t kEncodingTimeMeasureWindowMs = 1000; + for (auto& it : frame_timing_) { + if (it.timestamp == timestamp) { + it.last_send_us = time_sent_in_us; + break; + } + } + // TODO(pbos): Handle the case/log errors when not finding the corresponding + // frame (either very slow encoding or incorrect wrong timestamps returned + // from the encoder). + // This is currently the case for all frames on ChromeOS, so logging them + // would be spammy, and triggering overuse would be wrong. + // https://crbug.com/350106 + while (!frame_timing_.empty()) { + FrameTiming timing = frame_timing_.front(); + if (time_sent_in_us - timing.capture_us < + kEncodingTimeMeasureWindowMs * rtc::kNumMicrosecsPerMillisec) { + break; + } + if (timing.last_send_us != -1) { + int encode_duration_us = + static_cast(timing.last_send_us - timing.capture_us); + if (encoder_timing_) { + // TODO(nisse): Update encoder_timing_ to also use us units. + encoder_timing_->OnEncodeTiming(timing.capture_time_us / + rtc::kNumMicrosecsPerMillisec, + encode_duration_us / + rtc::kNumMicrosecsPerMillisec); + } + if (last_processed_capture_time_us_ != -1) { + int64_t diff_us = timing.capture_us - last_processed_capture_time_us_; + usage_->AddSample(1e-3 * encode_duration_us, 1e-3 * diff_us); + } + last_processed_capture_time_us_ = timing.capture_us; + EncodedFrameTimeMeasured(encode_duration_us / + rtc::kNumMicrosecsPerMillisec); + } + frame_timing_.pop_front(); } } diff --git a/video/overuse_frame_detector.h b/video/overuse_frame_detector.h index 2ec52309e0..3cc9262a94 100644 --- a/video/overuse_frame_detector.h +++ b/video/overuse_frame_detector.h @@ -17,6 +17,7 @@ #include "api/optional.h" #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/constructormagic.h" +#include "rtc_base/numerics/exp_filter.h" #include "rtc_base/sequenced_task_checker.h" #include "rtc_base/task_queue.h" #include "rtc_base/thread_annotations.h" @@ -34,12 +35,12 @@ struct CpuOveruseOptions { // General settings. int frame_timeout_interval_ms; // The maximum allowed interval between two // frames before resetting estimations. + int min_frame_samples; // The minimum number of frames required. int min_process_count; // The number of initial process times required before // triggering an overuse/underuse. int high_threshold_consecutive_count; // The number of consecutive checks // above the high threshold before // triggering an overuse. - int filter_time_ms; // Time constant for averaging }; struct CpuOveruseMetrics { @@ -76,11 +77,18 @@ class OveruseFrameDetector { // StartCheckForOveruse has been called. void StopCheckForOveruse(); - // Called for each captured frame. - void FrameCaptured(int width, int height); + // Defines the current maximum framerate targeted by the capturer. This is + // used to make sure the encode usage percent doesn't drop unduly if the + // capturer has quiet periods (for instance caused by screen capturers with + // variable capture rate depending on content updates), otherwise we might + // experience adaptation toggling. + virtual void OnTargetFramerateUpdated(int framerate_fps); - // Called for each encoded frame. - void FrameEncoded(int64_t capture_time_us, int64_t encode_duration_us); + // Called for each captured frame. + void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); + + // Called for each sent frame. + void FrameSent(uint32_t timestamp, int64_t time_sent_in_us); protected: void CheckForOveruse(); // Protected for test purposes. @@ -89,6 +97,17 @@ class OveruseFrameDetector { class OverdoseInjector; class SendProcessingUsage; class CheckOveruseTask; + struct FrameTiming { + FrameTiming(int64_t capture_time_us, uint32_t timestamp, int64_t now) + : capture_time_us(capture_time_us), + timestamp(timestamp), + capture_us(now), + last_send_us(-1) {} + int64_t capture_time_us; + uint32_t timestamp; + int64_t capture_us; + int64_t last_send_us; + }; void EncodedFrameTimeMeasured(int encode_duration_ms); bool IsOverusing(const CpuOveruseMetrics& metrics); @@ -97,7 +116,7 @@ class OveruseFrameDetector { bool FrameTimeoutDetected(int64_t now) const; bool FrameSizeChanged(int num_pixels) const; - void ResetAll(); + void ResetAll(int num_pixels); static std::unique_ptr CreateSendProcessingUsage( const CpuOveruseOptions& options); @@ -123,6 +142,7 @@ class OveruseFrameDetector { // Number of pixels of last captured frame. int num_pixels_ RTC_GUARDED_BY(task_checker_); + int max_framerate_ RTC_GUARDED_BY(task_checker_); int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_); int checks_above_threshold_ RTC_GUARDED_BY(task_checker_); int num_overuse_detections_ RTC_GUARDED_BY(task_checker_); @@ -134,6 +154,7 @@ class OveruseFrameDetector { // allocs)? const std::unique_ptr usage_ RTC_GUARDED_BY(task_checker_); + std::list frame_timing_ RTC_GUARDED_BY(task_checker_); RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector); }; diff --git a/video/overuse_frame_detector_unittest.cc b/video/overuse_frame_detector_unittest.cc index 440d86bf55..0f3dd8643e 100644 --- a/video/overuse_frame_detector_unittest.cc +++ b/video/overuse_frame_detector_unittest.cc @@ -15,8 +15,6 @@ #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/event.h" #include "rtc_base/fakeclock.h" -#include "rtc_base/logging.h" -#include "rtc_base/random.h" #include "test/gmock.h" #include "test/gtest.h" #include "video/overuse_frame_detector.h" @@ -24,8 +22,6 @@ namespace webrtc { using ::testing::InvokeWithoutArgs; -using ::testing::AtLeast; -using ::testing::_; namespace { const int kWidth = 640; @@ -101,20 +97,27 @@ class OveruseFrameDetectorTest : public ::testing::Test, int width, int height, int delay_us) { + VideoFrame frame(I420Buffer::Create(width, height), + webrtc::kVideoRotation_0, 0); + uint32_t timestamp = 0; while (num_frames-- > 0) { - overuse_detector_->FrameCaptured(width, height); - overuse_detector_->FrameEncoded(rtc::TimeMicros(), delay_us); - clock_.AdvanceTimeMicros(interval_us); + frame.set_timestamp(timestamp); + overuse_detector_->FrameCaptured(frame, rtc::TimeMicros()); + clock_.AdvanceTimeMicros(delay_us); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros()); + clock_.AdvanceTimeMicros(interval_us - delay_us); + timestamp += interval_us * 90 / 1000; } } void ForceUpdate(int width, int height) { - // This is mainly to check initial values and whether the overuse - // detector has been reset or not. - InsertAndSendFramesWithInterval(1, rtc::kNumMicrosecsPerSec, width, height, - kFrameIntervalUs); + // Insert one frame, wait a second and then put in another to force update + // the usage. From the tests where these are used, adding another sample + // doesn't affect the expected outcome (this is mainly to check initial + // values and whether the overuse detector has been reset or not). + InsertAndSendFramesWithInterval(2, rtc::kNumMicrosecsPerSec, + width, height, kFrameIntervalUs); } - void TriggerOveruse(int num_times) { const int kDelayUs = 32 * rtc::kNumMicrosecsPerMillisec; for (int i = 0; i < num_times; ++i) { @@ -266,11 +269,76 @@ TEST_F(OveruseFrameDetectorTest, ResetAfterFrameTimeout) { EXPECT_EQ(InitialUsage(), UsagePercent()); } +TEST_F(OveruseFrameDetectorTest, MinFrameSamplesBeforeUpdating) { + options_.min_frame_samples = 40; + ReinitializeOveruseDetector(); + InsertAndSendFramesWithInterval( + 40, kFrameIntervalUs, kWidth, kHeight, kProcessTimeUs); + EXPECT_EQ(InitialUsage(), UsagePercent()); + // Pass time far enough to digest all previous samples. + clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec); + InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + // The last sample has not been processed here. + EXPECT_EQ(InitialUsage(), UsagePercent()); + + // Pass time far enough to digest all previous samples, 41 in total. + clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec); + InsertAndSendFramesWithInterval( + 1, kFrameIntervalUs, kWidth, kHeight, kProcessTimeUs); + EXPECT_NE(InitialUsage(), UsagePercent()); +} + TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) { ForceUpdate(kWidth, kHeight); EXPECT_EQ(InitialUsage(), UsagePercent()); } +TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)) + .Times(testing::AtLeast(1)); + static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; + static const size_t kNumFramesEncodingDelay = 3; + VideoFrame frame(I420Buffer::Create(kWidth, kHeight), + webrtc::kVideoRotation_0, 0); + for (size_t i = 0; i < 1000; ++i) { + // Unique timestamps. + frame.set_timestamp(static_cast(i)); + overuse_detector_->FrameCaptured(frame, rtc::TimeMicros()); + clock_.AdvanceTimeMicros(kIntervalUs); + if (i > kNumFramesEncodingDelay) { + overuse_detector_->FrameSent( + static_cast(i - kNumFramesEncodingDelay), + rtc::TimeMicros()); + } + overuse_detector_->CheckForOveruse(); + } +} + +TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) { + // >85% encoding time should trigger overuse. + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)) + .Times(testing::AtLeast(1)); + static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; + static const int kDelayUs = 30 * rtc::kNumMicrosecsPerMillisec; + VideoFrame frame(I420Buffer::Create(kWidth, kHeight), + webrtc::kVideoRotation_0, 0); + uint32_t timestamp = 0; + for (size_t i = 0; i < 1000; ++i) { + frame.set_timestamp(timestamp); + overuse_detector_->FrameCaptured(frame, rtc::TimeMicros()); + // Encode and send first parts almost instantly. + clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros()); + // Encode heavier part, resulting in >85% usage total. + clock_.AdvanceTimeMicros(kDelayUs - rtc::kNumMicrosecsPerMillisec); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros()); + clock_.AdvanceTimeMicros(kIntervalUs - kDelayUs); + timestamp += kIntervalUs * 90 / 1000; + overuse_detector_->CheckForOveruse(); + } +} + TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { rtc::TaskQueue queue("OveruseFrameDetectorTestQueue"); @@ -301,59 +369,115 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { EXPECT_TRUE(event.Wait(10000)); } -// Models screencast, with irregular arrival of frames which are heavy -// to encode. -TEST_F(OveruseFrameDetectorTest, NoOveruseForLargeRandomFrameInterval) { - EXPECT_CALL(*(observer_.get()), AdaptDown(_)).Times(0); - EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(AtLeast(1)); - - const int kNumFrames = 500; - const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; - - const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; - const int kMaxIntervalUs = 1000 * rtc::kNumMicrosecsPerMillisec; - - webrtc::Random random(17); - - for (int i = 0; i < kNumFrames; i++) { - int interval_us = random.Rand(kMinIntervalUs, kMaxIntervalUs); - overuse_detector_->FrameCaptured(kWidth, kHeight); - overuse_detector_->FrameEncoded(rtc::TimeMicros(), kEncodeTimeUs); +TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) { + const int kCapturerMaxFrameRate = 30; + const int kEncodeMaxFrameRate = 20; // Maximum fps the encoder can sustain. + // Trigger overuse. + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kCapturerMaxFrameRate; + // Processing time just below over use limit given kEncodeMaxFrameRate. + int64_t processing_time_us = + (98 * OveruseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); overuse_detector_->CheckForOveruse(); - clock_.AdvanceTimeMicros(interval_us); } - // Average usage 19%. Check that estimate is in the right ball park. - EXPECT_NEAR(UsagePercent(), 20, 10); + + // Simulate frame rate reduction and normal usage. + frame_interval_us = rtc::kNumMicrosecsPerSec / kEncodeMaxFrameRate; + overuse_detector_->OnTargetFramerateUpdated(kEncodeMaxFrameRate); + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Reduce processing time to trigger underuse. + processing_time_us = + (98 * UnderuseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(1); + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); } -// Models screencast, with irregular arrival of frames, often -// exceeding the timeout interval. -TEST_F(OveruseFrameDetectorTest, NoOveruseForRandomFrameIntervalWithReset) { - EXPECT_CALL(*(observer_.get()), AdaptDown(_)).Times(0); - EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(AtLeast(1)); - - const int kNumFrames = 500; - const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; - - const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; - const int kMaxIntervalUs = 3000 * rtc::kNumMicrosecsPerMillisec; - - webrtc::Random random(17); - - for (int i = 0; i < kNumFrames; i++) { - int interval_us = random.Rand(kMinIntervalUs, kMaxIntervalUs); - overuse_detector_->FrameCaptured(kWidth, kHeight); - overuse_detector_->FrameEncoded(rtc::TimeMicros(), kEncodeTimeUs); +TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) { + const int kMinFrameRate = 7; // Minimum fps allowed by current detector impl. + overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate); + // Normal usage just at the limit. + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMinFrameRate; + // Processing time just below over use limit given kEncodeMaxFrameRate. + int64_t processing_time_us = + (98 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Over the limit to overuse. + processing_time_us = + (102 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Reduce input frame rate. Should still trigger overuse. + overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate - 1); + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } +} + +TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) { + const int kMaxFrameRate = 20; + overuse_detector_->OnTargetFramerateUpdated(kMaxFrameRate); + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMaxFrameRate; + // Maximum frame interval allowed is 35% above ideal. + int64_t max_frame_interval_us = (135 * frame_interval_us) / 100; + // Maximum processing time, without triggering overuse, allowed with the above + // frame interval. + int64_t max_processing_time_us = + (max_frame_interval_us * options_.high_encode_usage_threshold_percent) / + 100; + + // Processing time just below overuse limit given kMaxFrameRate. + int64_t processing_time_us = (98 * max_processing_time_us) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Go above limit, trigger overuse. + processing_time_us = (102 * max_processing_time_us) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Increase frame interval, should still trigger overuse. + max_frame_interval_us *= 2; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); overuse_detector_->CheckForOveruse(); - clock_.AdvanceTimeMicros(interval_us); } - // Average usage 6.6%, but since the frame_timeout_interval_ms is - // only 1500 ms, we often reset the estimate to the initial value. - // Check that estimate is in the right ball park. - EXPECT_GE(UsagePercent(), 1); - EXPECT_LE(UsagePercent(), InitialUsage() + 5); } } // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 1bca22cb98..77945dc0b4 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -508,6 +508,12 @@ void VideoStreamEncoder::SetSource( bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_); initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop; ConfigureQualityScaler(); + if (!IsFramerateScalingEnabled(degradation_preference) && + max_framerate_ != -1) { + // If frame rate scaling is no longer allowed, remove any potential + // allowance for longer frame intervals. + overuse_detector_->OnTargetFramerateUpdated(max_framerate_); + } }); } @@ -622,6 +628,15 @@ void VideoStreamEncoder::ReconfigureEncoder() { sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); + // Get the current target framerate, ie the maximum framerate as specified by + // the current codec configuration, or any limit imposed by cpu adaption in + // maintain-resolution or balanced mode. This is used to make sure overuse + // detection doesn't needlessly trigger in low and/or variable framerate + // scenarios. + int target_framerate = std::min( + max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); + overuse_detector_->OnTargetFramerateUpdated(target_framerate); + ConfigureQualityScaler(); } @@ -806,7 +821,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); - overuse_detector_->FrameCaptured(out_frame.width(), out_frame.height()); + overuse_detector_->FrameCaptured(out_frame, time_when_posted_us); video_sender_.AddVideoFrame(out_frame, nullptr); } @@ -832,21 +847,12 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( EncodedImageCallback::Result result = sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); - int64_t capture_time_us = - encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; - int64_t encode_duration_us; - if (encoded_image.timing_.flags != TimingFrameFlags::kInvalid) { - encode_duration_us = rtc::kNumMicrosecsPerMillisec * - (encoded_image.timing_.encode_finish_ms - - encoded_image.timing_.encode_start_ms); - } else { - encode_duration_us = -1; - } + int64_t time_sent_us = rtc::TimeMicros(); + uint32_t timestamp = encoded_image._timeStamp; const int qp = encoded_image.qp_; - encoder_queue_.PostTask([this, capture_time_us, encode_duration_us, qp] { + encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] { RTC_DCHECK_RUN_ON(&encoder_queue_); - if (encode_duration_us >= 0) - overuse_detector_->FrameEncoded(capture_time_us, encode_duration_us); + overuse_detector_->FrameSent(timestamp, time_sent_us); if (quality_scaler_ && qp >= 0) quality_scaler_->ReportQP(qp); }); @@ -997,6 +1003,8 @@ void VideoStreamEncoder::AdaptDown(AdaptReason reason) { if (requested_framerate == -1) return; RTC_DCHECK_NE(max_framerate_, -1); + overuse_detector_->OnTargetFramerateUpdated( + std::min(max_framerate_, requested_framerate)); GetAdaptCounter().IncrementFramerate(reason); break; } @@ -1080,8 +1088,11 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) { const int requested_framerate = source_proxy_->RequestHigherFramerateThan(fps); if (requested_framerate == -1) { + overuse_detector_->OnTargetFramerateUpdated(max_framerate_); return; } + overuse_detector_->OnTargetFramerateUpdated( + std::min(max_framerate_, requested_framerate)); GetAdaptCounter().DecrementFramerate(reason); break; } diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 38883acafe..26e9d381cb 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -63,11 +63,39 @@ class TestBuffer : public webrtc::I420Buffer { rtc::Event* const event_; }; +class CpuOveruseDetectorProxy : public OveruseFrameDetector { + public: + CpuOveruseDetectorProxy(const CpuOveruseOptions& options, + AdaptationObserverInterface* overuse_observer, + EncodedFrameObserver* encoder_timing_, + CpuOveruseMetricsObserver* metrics_observer) + : OveruseFrameDetector(options, + overuse_observer, + encoder_timing_, + metrics_observer), + last_target_framerate_fps_(-1) {} + virtual ~CpuOveruseDetectorProxy() {} + + void OnTargetFramerateUpdated(int framerate_fps) override { + rtc::CritScope cs(&lock_); + last_target_framerate_fps_ = framerate_fps; + OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps); + } + + int GetLastTargetFramerate() { + rtc::CritScope cs(&lock_); + return last_target_framerate_fps_; + } + + private: + rtc::CriticalSection lock_; + int last_target_framerate_fps_ RTC_GUARDED_BY(lock_); +}; + class VideoStreamEncoderUnderTest : public VideoStreamEncoder { public: - VideoStreamEncoderUnderTest( - SendStatisticsProxy* stats_proxy, - const VideoSendStream::Config::EncoderSettings& settings) + VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy, + const VideoSendStream::Config::EncoderSettings& settings) : VideoStreamEncoder( 1 /* number_of_cores */, stats_proxy, @@ -75,7 +103,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { nullptr /* pre_encode_callback */, nullptr /* encoder_timing */, std::unique_ptr( - overuse_detector_ = new OveruseFrameDetector( + overuse_detector_proxy_ = new CpuOveruseDetectorProxy( GetCpuOveruseOptions(settings.full_overuse_time), this, nullptr, @@ -108,7 +136,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); } - OveruseFrameDetector* overuse_detector_; + CpuOveruseDetectorProxy* overuse_detector_proxy_; }; class VideoStreamFactory @@ -2210,6 +2238,126 @@ TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) { + const int kFrameWidth = 1280; + const int kFrameHeight = 720; + const int kFramerate = 24; + + video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainResolution); + + // Insert a single frame, triggering initial configuration. + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ( + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kDefaultFramerate); + + // Trigger reconfigure encoder (without resetting the entire instance). + VideoEncoderConfig video_encoder_config; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.number_of_streams = 1; + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kFramerate); + video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + // Detector should be updated with fps limit from codec config. + EXPECT_EQ( + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + // Trigger overuse, max framerate should be reduced. + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + stats.input_frame_rate = kFramerate; + stats_proxy_->SetMockStats(stats); + video_stream_encoder_->TriggerCpuOveruse(); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + int adapted_framerate = + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + EXPECT_LT(adapted_framerate, kFramerate); + + // Trigger underuse, max framerate should go back to codec configured fps. + // Set extra low fps, to make sure it's actually reset, not just incremented. + stats = stats_proxy_->GetStats(); + stats.input_frame_rate = adapted_framerate / 2; + stats_proxy_->SetMockStats(stats); + video_stream_encoder_->TriggerCpuNormalUsage(); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + EXPECT_EQ( + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) { + const int kFrameWidth = 1280; + const int kFrameHeight = 720; + const int kLowFramerate = 15; + const int kHighFramerate = 25; + + video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainResolution); + + // Trigger initial configuration. + VideoEncoderConfig video_encoder_config; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.number_of_streams = 1; + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kLowFramerate); + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ( + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kLowFramerate); + + // Trigger overuse, max framerate should be reduced. + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + stats.input_frame_rate = kLowFramerate; + stats_proxy_->SetMockStats(stats); + video_stream_encoder_->TriggerCpuOveruse(); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + int adapted_framerate = + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + EXPECT_LT(adapted_framerate, kLowFramerate); + + // Reconfigure the encoder with a new (higher max framerate), max fps should + // still respect the adaptation. + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kHighFramerate); + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ( + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + adapted_framerate); + + // Trigger underuse, max framerate should go back to codec configured fps. + stats = stats_proxy_->GetStats(); + stats.input_frame_rate = adapted_framerate; + stats_proxy_->SetMockStats(stats); + video_stream_encoder_->TriggerCpuNormalUsage(); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + EXPECT_EQ( + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kHighFramerate); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnDegradationPreferenceChange) { const int kFrameWidth = 1280; @@ -2232,7 +2380,9 @@ TEST_F(VideoStreamEncoderTest, kMaxPayloadLength, false); video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - EXPECT_GT(source.sink_wants().max_framerate_fps, kFramerate); + EXPECT_EQ( + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); // Trigger overuse, max framerate should be reduced. VideoSendStream::Stats stats = stats_proxy_->GetStats(); @@ -2240,8 +2390,8 @@ TEST_F(VideoStreamEncoderTest, stats_proxy_->SetMockStats(stats); video_stream_encoder_->TriggerCpuOveruse(); video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - - int adapted_framerate = source.sink_wants().max_framerate_fps; + int adapted_framerate = + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); EXPECT_LT(adapted_framerate, kFramerate); // Change degradation preference to not enable framerate scaling. Target @@ -2249,7 +2399,9 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->SetSource( &source, VideoSendStream::DegradationPreference::kMaintainFramerate); video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - EXPECT_GT(source.sink_wants().max_framerate_fps, kFramerate); + EXPECT_EQ( + video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); video_stream_encoder_->Stop(); }