diff --git a/video/overuse_frame_detector.cc b/video/overuse_frame_detector.cc index 7b64c10597..8519ce7804 100644 --- a/video/overuse_frame_detector.cc +++ b/video/overuse_frame_detector.cc @@ -23,7 +23,6 @@ #include "common_video/include/frame_callback.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" -#include "rtc_base/numerics/exp_filter.h" #include "rtc_base/timeutils.h" #include "system_wrappers/include/field_trial.h" @@ -48,31 +47,15 @@ const double kRampUpBackoffFactor = 2.0; // Max number of overuses detected before always applying the rampup delay. const int kMaxOverusesBeforeApplyRampupDelay = 4; -// The maximum exponent to use in VCMExpFilter. -const float kMaxExp = 7.0f; -// Default value used before first reconfiguration. -const int kDefaultFrameRate = 30; -// Default sample diff, default frame rate. -const float kDefaultSampleDiffMs = 1000.0f / kDefaultFrameRate; -// A factor applied to the sample diff on OnTargetFramerateUpdated to determine -// a max limit for the sample diff. For instance, with a framerate of 30fps, -// the sample diff is capped to (1000 / 30) * 1.35 = 45ms. This prevents -// triggering too soon if there are individual very large outliers. -const float kMaxSampleDiffMarginFactor = 1.35f; -// Minimum framerate allowed for usage calculation. This prevents crazy long -// encode times from being accepted if the frame rate happens to be low. -const int kMinFramerate = 7; -const int kMaxFramerate = 30; - const auto kScaleReasonCpu = AdaptationObserverInterface::AdaptReason::kCpu; } // namespace CpuOveruseOptions::CpuOveruseOptions() : high_encode_usage_threshold_percent(85), frame_timeout_interval_ms(1500), - min_frame_samples(120), min_process_count(3), - high_threshold_consecutive_count(2) { + high_threshold_consecutive_count(2), + filter_time_ms(5000) { #if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) // This is proof-of-concept code for letting the physical core count affect // the interval into which we attempt to scale. For now, the code is Mac OS @@ -123,72 +106,43 @@ CpuOveruseOptions::CpuOveruseOptions() class OveruseFrameDetector::SendProcessingUsage { public: explicit SendProcessingUsage(const CpuOveruseOptions& options) - : kWeightFactorFrameDiff(0.998f), - kWeightFactorProcessing(0.995f), - kInitialSampleDiffMs(40.0f), - count_(0), - options_(options), - max_sample_diff_ms_(kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor), - filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)), - filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) { + : options_(options) { Reset(); } virtual ~SendProcessingUsage() {} void Reset() { - count_ = 0; - max_sample_diff_ms_ = kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor; - filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff); - filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs); - filtered_processing_ms_->Reset(kWeightFactorProcessing); - filtered_processing_ms_->Apply(1.0f, InitialProcessingMs()); + // Start in between the underuse and overuse threshold. + load_estimate_ = (options_.low_encode_usage_threshold_percent + + options_.high_encode_usage_threshold_percent) / + 200.0; } - void SetMaxSampleDiffMs(float diff_ms) { max_sample_diff_ms_ = diff_ms; } + void AddSample(double encode_time, double diff_time) { + RTC_CHECK_GE(diff_time, 0.0); - void AddCaptureSample(float sample_ms) { - float exp = sample_ms / kDefaultSampleDiffMs; - exp = std::min(exp, kMaxExp); - filtered_frame_diff_ms_->Apply(exp, sample_ms); - } - - void AddSample(float processing_ms, int64_t diff_last_sample_ms) { - ++count_; - float exp = diff_last_sample_ms / kDefaultSampleDiffMs; - exp = std::min(exp, kMaxExp); - filtered_processing_ms_->Apply(exp, processing_ms); - } - - virtual int Value() { - if (count_ < static_cast(options_.min_frame_samples)) { - return static_cast(InitialUsageInPercent() + 0.5f); + // Use the filter update + // + // load <-- x/d (1-exp (-d/T)) + exp (-d/T) load + // + // where we must take care for small d, using the proper limit + // (1 - exp(-d/tau)) / d = 1/tau - d/2tau^2 + O(d^2) + double tau = (1e-3 * options_.filter_time_ms); + double e = diff_time / tau; + double c; + if (e < 0.0001) { + c = (1 - e / 2) / tau; + } else { + c = -expm1(-e) / diff_time; } - float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f); - frame_diff_ms = std::min(frame_diff_ms, max_sample_diff_ms_); - float encode_usage_percent = - 100.0f * filtered_processing_ms_->filtered() / frame_diff_ms; - return static_cast(encode_usage_percent + 0.5); + load_estimate_ = c * encode_time + exp(-e) * load_estimate_; } + virtual int Value() { return static_cast(100.0 * load_estimate_ + 0.5); } + private: - float InitialUsageInPercent() const { - // Start in between the underuse and overuse threshold. - return (options_.low_encode_usage_threshold_percent + - options_.high_encode_usage_threshold_percent) / 2.0f; - } - - float InitialProcessingMs() const { - return InitialUsageInPercent() * kInitialSampleDiffMs / 100; - } - - const float kWeightFactorFrameDiff; - const float kWeightFactorProcessing; - const float kInitialSampleDiffMs; - uint64_t count_; const CpuOveruseOptions options_; - float max_sample_diff_ms_; - std::unique_ptr filtered_processing_ms_; - std::unique_ptr filtered_frame_diff_ms_; + double load_estimate_; }; // Class used for manual testing of overuse, enabled via field trial flag. @@ -218,6 +172,7 @@ class OveruseFrameDetector::OverdoseInjector int64_t now_ms = rtc::TimeMillis(); if (last_toggling_ms_ == -1) { last_toggling_ms_ = now_ms; + } else { switch (state_) { case State::kNormal: @@ -255,7 +210,6 @@ class OveruseFrameDetector::OverdoseInjector overried_usage_value.emplace(5); break; } - return overried_usage_value.value_or(SendProcessingUsage::Value()); } @@ -348,7 +302,6 @@ OveruseFrameDetector::OveruseFrameDetector( last_capture_time_us_(-1), last_processed_capture_time_us_(-1), num_pixels_(0), - max_framerate_(kDefaultFrameRate), last_overuse_time_ms_(-1), checks_above_threshold_(0), num_overuse_detections_(0), @@ -399,93 +352,41 @@ bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now_us) const { options_.frame_timeout_interval_ms * rtc::kNumMicrosecsPerMillisec; } -void OveruseFrameDetector::ResetAll(int num_pixels) { - // Reset state, as a result resolution being changed. Do not however change - // the current frame rate back to the default. +void OveruseFrameDetector::ResetAll() { RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); - num_pixels_ = num_pixels; usage_->Reset(); - frame_timing_.clear(); last_capture_time_us_ = -1; last_processed_capture_time_us_ = -1; num_process_times_ = 0; metrics_ = rtc::Optional(); - OnTargetFramerateUpdated(max_framerate_); } -void OveruseFrameDetector::OnTargetFramerateUpdated(int framerate_fps) { - RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); - RTC_DCHECK_GE(framerate_fps, 0); - max_framerate_ = std::min(kMaxFramerate, framerate_fps); - usage_->SetMaxSampleDiffMs((1000 / std::max(kMinFramerate, max_framerate_)) * - kMaxSampleDiffMarginFactor); -} - -void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame, - int64_t time_when_first_seen_us) { +void OveruseFrameDetector::FrameCaptured(int width, int height) { RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); - if (FrameSizeChanged(frame.width() * frame.height()) || - FrameTimeoutDetected(time_when_first_seen_us)) { - ResetAll(frame.width() * frame.height()); + if (FrameSizeChanged(width * height)) { + ResetAll(); + num_pixels_ = width * height; } - - if (last_capture_time_us_ != -1) - usage_->AddCaptureSample( - 1e-3 * (time_when_first_seen_us - last_capture_time_us_)); - - last_capture_time_us_ = time_when_first_seen_us; - - frame_timing_.push_back(FrameTiming(frame.timestamp_us(), frame.timestamp(), - time_when_first_seen_us)); } -void OveruseFrameDetector::FrameSent(uint32_t timestamp, - int64_t time_sent_in_us) { +void OveruseFrameDetector::FrameEncoded(int64_t capture_time_us, + int64_t encode_duration_us) { RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); - // Delay before reporting actual encoding time, used to have the ability to - // detect total encoding time when encoding more than one layer. Encoding is - // here assumed to finish within a second (or that we get enough long-time - // samples before one second to trigger an overuse even when this is not the - // case). - static const int64_t kEncodingTimeMeasureWindowMs = 1000; - for (auto& it : frame_timing_) { - if (it.timestamp == timestamp) { - it.last_send_us = time_sent_in_us; - break; - } + if (FrameTimeoutDetected(capture_time_us)) { + ResetAll(); + } else if (last_capture_time_us_ != -1) { + usage_->AddSample(1e-6 * encode_duration_us, + 1e-6 * (capture_time_us - last_capture_time_us_)); } - // TODO(pbos): Handle the case/log errors when not finding the corresponding - // frame (either very slow encoding or incorrect wrong timestamps returned - // from the encoder). - // This is currently the case for all frames on ChromeOS, so logging them - // would be spammy, and triggering overuse would be wrong. - // https://crbug.com/350106 - while (!frame_timing_.empty()) { - FrameTiming timing = frame_timing_.front(); - if (time_sent_in_us - timing.capture_us < - kEncodingTimeMeasureWindowMs * rtc::kNumMicrosecsPerMillisec) { - break; - } - if (timing.last_send_us != -1) { - int encode_duration_us = - static_cast(timing.last_send_us - timing.capture_us); - if (encoder_timing_) { - // TODO(nisse): Update encoder_timing_ to also use us units. - encoder_timing_->OnEncodeTiming(timing.capture_time_us / - rtc::kNumMicrosecsPerMillisec, - encode_duration_us / - rtc::kNumMicrosecsPerMillisec); - } - if (last_processed_capture_time_us_ != -1) { - int64_t diff_us = timing.capture_us - last_processed_capture_time_us_; - usage_->AddSample(1e-3 * encode_duration_us, 1e-3 * diff_us); - } - last_processed_capture_time_us_ = timing.capture_us; - EncodedFrameTimeMeasured(encode_duration_us / - rtc::kNumMicrosecsPerMillisec); - } - frame_timing_.pop_front(); + last_capture_time_us_ = capture_time_us; + EncodedFrameTimeMeasured(encode_duration_us / rtc::kNumMicrosecsPerMillisec); + + if (encoder_timing_) { + // TODO(nisse): Update encoder_timing_ to also use us units. + encoder_timing_->OnEncodeTiming( + capture_time_us / rtc::kNumMicrosecsPerMillisec, + encode_duration_us / rtc::kNumMicrosecsPerMillisec); } } diff --git a/video/overuse_frame_detector.h b/video/overuse_frame_detector.h index 3cc9262a94..2ec52309e0 100644 --- a/video/overuse_frame_detector.h +++ b/video/overuse_frame_detector.h @@ -17,7 +17,6 @@ #include "api/optional.h" #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/constructormagic.h" -#include "rtc_base/numerics/exp_filter.h" #include "rtc_base/sequenced_task_checker.h" #include "rtc_base/task_queue.h" #include "rtc_base/thread_annotations.h" @@ -35,12 +34,12 @@ struct CpuOveruseOptions { // General settings. int frame_timeout_interval_ms; // The maximum allowed interval between two // frames before resetting estimations. - int min_frame_samples; // The minimum number of frames required. int min_process_count; // The number of initial process times required before // triggering an overuse/underuse. int high_threshold_consecutive_count; // The number of consecutive checks // above the high threshold before // triggering an overuse. + int filter_time_ms; // Time constant for averaging }; struct CpuOveruseMetrics { @@ -77,18 +76,11 @@ class OveruseFrameDetector { // StartCheckForOveruse has been called. void StopCheckForOveruse(); - // Defines the current maximum framerate targeted by the capturer. This is - // used to make sure the encode usage percent doesn't drop unduly if the - // capturer has quiet periods (for instance caused by screen capturers with - // variable capture rate depending on content updates), otherwise we might - // experience adaptation toggling. - virtual void OnTargetFramerateUpdated(int framerate_fps); - // Called for each captured frame. - void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); + void FrameCaptured(int width, int height); - // Called for each sent frame. - void FrameSent(uint32_t timestamp, int64_t time_sent_in_us); + // Called for each encoded frame. + void FrameEncoded(int64_t capture_time_us, int64_t encode_duration_us); protected: void CheckForOveruse(); // Protected for test purposes. @@ -97,17 +89,6 @@ class OveruseFrameDetector { class OverdoseInjector; class SendProcessingUsage; class CheckOveruseTask; - struct FrameTiming { - FrameTiming(int64_t capture_time_us, uint32_t timestamp, int64_t now) - : capture_time_us(capture_time_us), - timestamp(timestamp), - capture_us(now), - last_send_us(-1) {} - int64_t capture_time_us; - uint32_t timestamp; - int64_t capture_us; - int64_t last_send_us; - }; void EncodedFrameTimeMeasured(int encode_duration_ms); bool IsOverusing(const CpuOveruseMetrics& metrics); @@ -116,7 +97,7 @@ class OveruseFrameDetector { bool FrameTimeoutDetected(int64_t now) const; bool FrameSizeChanged(int num_pixels) const; - void ResetAll(int num_pixels); + void ResetAll(); static std::unique_ptr CreateSendProcessingUsage( const CpuOveruseOptions& options); @@ -142,7 +123,6 @@ class OveruseFrameDetector { // Number of pixels of last captured frame. int num_pixels_ RTC_GUARDED_BY(task_checker_); - int max_framerate_ RTC_GUARDED_BY(task_checker_); int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_); int checks_above_threshold_ RTC_GUARDED_BY(task_checker_); int num_overuse_detections_ RTC_GUARDED_BY(task_checker_); @@ -154,7 +134,6 @@ class OveruseFrameDetector { // allocs)? const std::unique_ptr usage_ RTC_GUARDED_BY(task_checker_); - std::list frame_timing_ RTC_GUARDED_BY(task_checker_); RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector); }; diff --git a/video/overuse_frame_detector_unittest.cc b/video/overuse_frame_detector_unittest.cc index 0f3dd8643e..440d86bf55 100644 --- a/video/overuse_frame_detector_unittest.cc +++ b/video/overuse_frame_detector_unittest.cc @@ -15,6 +15,8 @@ #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/event.h" #include "rtc_base/fakeclock.h" +#include "rtc_base/logging.h" +#include "rtc_base/random.h" #include "test/gmock.h" #include "test/gtest.h" #include "video/overuse_frame_detector.h" @@ -22,6 +24,8 @@ namespace webrtc { using ::testing::InvokeWithoutArgs; +using ::testing::AtLeast; +using ::testing::_; namespace { const int kWidth = 640; @@ -97,27 +101,20 @@ class OveruseFrameDetectorTest : public ::testing::Test, int width, int height, int delay_us) { - VideoFrame frame(I420Buffer::Create(width, height), - webrtc::kVideoRotation_0, 0); - uint32_t timestamp = 0; while (num_frames-- > 0) { - frame.set_timestamp(timestamp); - overuse_detector_->FrameCaptured(frame, rtc::TimeMicros()); - clock_.AdvanceTimeMicros(delay_us); - overuse_detector_->FrameSent(timestamp, rtc::TimeMicros()); - clock_.AdvanceTimeMicros(interval_us - delay_us); - timestamp += interval_us * 90 / 1000; + overuse_detector_->FrameCaptured(width, height); + overuse_detector_->FrameEncoded(rtc::TimeMicros(), delay_us); + clock_.AdvanceTimeMicros(interval_us); } } void ForceUpdate(int width, int height) { - // Insert one frame, wait a second and then put in another to force update - // the usage. From the tests where these are used, adding another sample - // doesn't affect the expected outcome (this is mainly to check initial - // values and whether the overuse detector has been reset or not). - InsertAndSendFramesWithInterval(2, rtc::kNumMicrosecsPerSec, - width, height, kFrameIntervalUs); + // This is mainly to check initial values and whether the overuse + // detector has been reset or not. + InsertAndSendFramesWithInterval(1, rtc::kNumMicrosecsPerSec, width, height, + kFrameIntervalUs); } + void TriggerOveruse(int num_times) { const int kDelayUs = 32 * rtc::kNumMicrosecsPerMillisec; for (int i = 0; i < num_times; ++i) { @@ -269,76 +266,11 @@ TEST_F(OveruseFrameDetectorTest, ResetAfterFrameTimeout) { EXPECT_EQ(InitialUsage(), UsagePercent()); } -TEST_F(OveruseFrameDetectorTest, MinFrameSamplesBeforeUpdating) { - options_.min_frame_samples = 40; - ReinitializeOveruseDetector(); - InsertAndSendFramesWithInterval( - 40, kFrameIntervalUs, kWidth, kHeight, kProcessTimeUs); - EXPECT_EQ(InitialUsage(), UsagePercent()); - // Pass time far enough to digest all previous samples. - clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec); - InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight, - kProcessTimeUs); - // The last sample has not been processed here. - EXPECT_EQ(InitialUsage(), UsagePercent()); - - // Pass time far enough to digest all previous samples, 41 in total. - clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec); - InsertAndSendFramesWithInterval( - 1, kFrameIntervalUs, kWidth, kHeight, kProcessTimeUs); - EXPECT_NE(InitialUsage(), UsagePercent()); -} - TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) { ForceUpdate(kWidth, kHeight); EXPECT_EQ(InitialUsage(), UsagePercent()); } -TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)) - .Times(testing::AtLeast(1)); - static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; - static const size_t kNumFramesEncodingDelay = 3; - VideoFrame frame(I420Buffer::Create(kWidth, kHeight), - webrtc::kVideoRotation_0, 0); - for (size_t i = 0; i < 1000; ++i) { - // Unique timestamps. - frame.set_timestamp(static_cast(i)); - overuse_detector_->FrameCaptured(frame, rtc::TimeMicros()); - clock_.AdvanceTimeMicros(kIntervalUs); - if (i > kNumFramesEncodingDelay) { - overuse_detector_->FrameSent( - static_cast(i - kNumFramesEncodingDelay), - rtc::TimeMicros()); - } - overuse_detector_->CheckForOveruse(); - } -} - -TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) { - // >85% encoding time should trigger overuse. - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)) - .Times(testing::AtLeast(1)); - static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; - static const int kDelayUs = 30 * rtc::kNumMicrosecsPerMillisec; - VideoFrame frame(I420Buffer::Create(kWidth, kHeight), - webrtc::kVideoRotation_0, 0); - uint32_t timestamp = 0; - for (size_t i = 0; i < 1000; ++i) { - frame.set_timestamp(timestamp); - overuse_detector_->FrameCaptured(frame, rtc::TimeMicros()); - // Encode and send first parts almost instantly. - clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec); - overuse_detector_->FrameSent(timestamp, rtc::TimeMicros()); - // Encode heavier part, resulting in >85% usage total. - clock_.AdvanceTimeMicros(kDelayUs - rtc::kNumMicrosecsPerMillisec); - overuse_detector_->FrameSent(timestamp, rtc::TimeMicros()); - clock_.AdvanceTimeMicros(kIntervalUs - kDelayUs); - timestamp += kIntervalUs * 90 / 1000; - overuse_detector_->CheckForOveruse(); - } -} - TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { rtc::TaskQueue queue("OveruseFrameDetectorTestQueue"); @@ -369,115 +301,59 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { EXPECT_TRUE(event.Wait(10000)); } -TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) { - const int kCapturerMaxFrameRate = 30; - const int kEncodeMaxFrameRate = 20; // Maximum fps the encoder can sustain. +// Models screencast, with irregular arrival of frames which are heavy +// to encode. +TEST_F(OveruseFrameDetectorTest, NoOveruseForLargeRandomFrameInterval) { + EXPECT_CALL(*(observer_.get()), AdaptDown(_)).Times(0); + EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(AtLeast(1)); + + const int kNumFrames = 500; + const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; + + const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; + const int kMaxIntervalUs = 1000 * rtc::kNumMicrosecsPerMillisec; + + webrtc::Random random(17); + + for (int i = 0; i < kNumFrames; i++) { + int interval_us = random.Rand(kMinIntervalUs, kMaxIntervalUs); + overuse_detector_->FrameCaptured(kWidth, kHeight); + overuse_detector_->FrameEncoded(rtc::TimeMicros(), kEncodeTimeUs); - // Trigger overuse. - int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kCapturerMaxFrameRate; - // Processing time just below over use limit given kEncodeMaxFrameRate. - int64_t processing_time_us = - (98 * OveruseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); - for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { - InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, - processing_time_us); overuse_detector_->CheckForOveruse(); + clock_.AdvanceTimeMicros(interval_us); } - - // Simulate frame rate reduction and normal usage. - frame_interval_us = rtc::kNumMicrosecsPerSec / kEncodeMaxFrameRate; - overuse_detector_->OnTargetFramerateUpdated(kEncodeMaxFrameRate); - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); - for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { - InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, - processing_time_us); - overuse_detector_->CheckForOveruse(); - } - - // Reduce processing time to trigger underuse. - processing_time_us = - (98 * UnderuseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; - EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(1); - InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, - processing_time_us); - overuse_detector_->CheckForOveruse(); + // Average usage 19%. Check that estimate is in the right ball park. + EXPECT_NEAR(UsagePercent(), 20, 10); } -TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) { - const int kMinFrameRate = 7; // Minimum fps allowed by current detector impl. - overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate); +// Models screencast, with irregular arrival of frames, often +// exceeding the timeout interval. +TEST_F(OveruseFrameDetectorTest, NoOveruseForRandomFrameIntervalWithReset) { + EXPECT_CALL(*(observer_.get()), AdaptDown(_)).Times(0); + EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(AtLeast(1)); + + const int kNumFrames = 500; + const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; + + const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; + const int kMaxIntervalUs = 3000 * rtc::kNumMicrosecsPerMillisec; + + webrtc::Random random(17); + + for (int i = 0; i < kNumFrames; i++) { + int interval_us = random.Rand(kMinIntervalUs, kMaxIntervalUs); + overuse_detector_->FrameCaptured(kWidth, kHeight); + overuse_detector_->FrameEncoded(rtc::TimeMicros(), kEncodeTimeUs); - // Normal usage just at the limit. - int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMinFrameRate; - // Processing time just below over use limit given kEncodeMaxFrameRate. - int64_t processing_time_us = - (98 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); - for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { - InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, - processing_time_us); - overuse_detector_->CheckForOveruse(); - } - - // Over the limit to overuse. - processing_time_us = - (102 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); - for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { - InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, - processing_time_us); - overuse_detector_->CheckForOveruse(); - } - - // Reduce input frame rate. Should still trigger overuse. - overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate - 1); - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); - for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { - InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, - processing_time_us); - overuse_detector_->CheckForOveruse(); - } -} - -TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) { - const int kMaxFrameRate = 20; - overuse_detector_->OnTargetFramerateUpdated(kMaxFrameRate); - int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMaxFrameRate; - // Maximum frame interval allowed is 35% above ideal. - int64_t max_frame_interval_us = (135 * frame_interval_us) / 100; - // Maximum processing time, without triggering overuse, allowed with the above - // frame interval. - int64_t max_processing_time_us = - (max_frame_interval_us * options_.high_encode_usage_threshold_percent) / - 100; - - // Processing time just below overuse limit given kMaxFrameRate. - int64_t processing_time_us = (98 * max_processing_time_us) / 100; - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); - for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { - InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, - kHeight, processing_time_us); - overuse_detector_->CheckForOveruse(); - } - - // Go above limit, trigger overuse. - processing_time_us = (102 * max_processing_time_us) / 100; - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); - for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { - InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, - kHeight, processing_time_us); - overuse_detector_->CheckForOveruse(); - } - - // Increase frame interval, should still trigger overuse. - max_frame_interval_us *= 2; - EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); - for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { - InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, - kHeight, processing_time_us); overuse_detector_->CheckForOveruse(); + clock_.AdvanceTimeMicros(interval_us); } + // Average usage 6.6%, but since the frame_timeout_interval_ms is + // only 1500 ms, we often reset the estimate to the initial value. + // Check that estimate is in the right ball park. + EXPECT_GE(UsagePercent(), 1); + EXPECT_LE(UsagePercent(), InitialUsage() + 5); } } // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 77945dc0b4..1bca22cb98 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -508,12 +508,6 @@ void VideoStreamEncoder::SetSource( bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_); initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop; ConfigureQualityScaler(); - if (!IsFramerateScalingEnabled(degradation_preference) && - max_framerate_ != -1) { - // If frame rate scaling is no longer allowed, remove any potential - // allowance for longer frame intervals. - overuse_detector_->OnTargetFramerateUpdated(max_framerate_); - } }); } @@ -628,15 +622,6 @@ void VideoStreamEncoder::ReconfigureEncoder() { sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); - // Get the current target framerate, ie the maximum framerate as specified by - // the current codec configuration, or any limit imposed by cpu adaption in - // maintain-resolution or balanced mode. This is used to make sure overuse - // detection doesn't needlessly trigger in low and/or variable framerate - // scenarios. - int target_framerate = std::min( - max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); - overuse_detector_->OnTargetFramerateUpdated(target_framerate); - ConfigureQualityScaler(); } @@ -821,7 +806,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); - overuse_detector_->FrameCaptured(out_frame, time_when_posted_us); + overuse_detector_->FrameCaptured(out_frame.width(), out_frame.height()); video_sender_.AddVideoFrame(out_frame, nullptr); } @@ -847,12 +832,21 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( EncodedImageCallback::Result result = sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); - int64_t time_sent_us = rtc::TimeMicros(); - uint32_t timestamp = encoded_image._timeStamp; + int64_t capture_time_us = + encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; + int64_t encode_duration_us; + if (encoded_image.timing_.flags != TimingFrameFlags::kInvalid) { + encode_duration_us = rtc::kNumMicrosecsPerMillisec * + (encoded_image.timing_.encode_finish_ms - + encoded_image.timing_.encode_start_ms); + } else { + encode_duration_us = -1; + } const int qp = encoded_image.qp_; - encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] { + encoder_queue_.PostTask([this, capture_time_us, encode_duration_us, qp] { RTC_DCHECK_RUN_ON(&encoder_queue_); - overuse_detector_->FrameSent(timestamp, time_sent_us); + if (encode_duration_us >= 0) + overuse_detector_->FrameEncoded(capture_time_us, encode_duration_us); if (quality_scaler_ && qp >= 0) quality_scaler_->ReportQP(qp); }); @@ -1003,8 +997,6 @@ void VideoStreamEncoder::AdaptDown(AdaptReason reason) { if (requested_framerate == -1) return; RTC_DCHECK_NE(max_framerate_, -1); - overuse_detector_->OnTargetFramerateUpdated( - std::min(max_framerate_, requested_framerate)); GetAdaptCounter().IncrementFramerate(reason); break; } @@ -1088,11 +1080,8 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) { const int requested_framerate = source_proxy_->RequestHigherFramerateThan(fps); if (requested_framerate == -1) { - overuse_detector_->OnTargetFramerateUpdated(max_framerate_); return; } - overuse_detector_->OnTargetFramerateUpdated( - std::min(max_framerate_, requested_framerate)); GetAdaptCounter().DecrementFramerate(reason); break; } diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 5afbe27d2b..02d6d3ec12 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -63,39 +63,11 @@ class TestBuffer : public webrtc::I420Buffer { rtc::Event* const event_; }; -class CpuOveruseDetectorProxy : public OveruseFrameDetector { - public: - CpuOveruseDetectorProxy(const CpuOveruseOptions& options, - AdaptationObserverInterface* overuse_observer, - EncodedFrameObserver* encoder_timing_, - CpuOveruseMetricsObserver* metrics_observer) - : OveruseFrameDetector(options, - overuse_observer, - encoder_timing_, - metrics_observer), - last_target_framerate_fps_(-1) {} - virtual ~CpuOveruseDetectorProxy() {} - - void OnTargetFramerateUpdated(int framerate_fps) override { - rtc::CritScope cs(&lock_); - last_target_framerate_fps_ = framerate_fps; - OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps); - } - - int GetLastTargetFramerate() { - rtc::CritScope cs(&lock_); - return last_target_framerate_fps_; - } - - private: - rtc::CriticalSection lock_; - int last_target_framerate_fps_ RTC_GUARDED_BY(lock_); -}; - class VideoStreamEncoderUnderTest : public VideoStreamEncoder { public: - VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy, - const VideoSendStream::Config::EncoderSettings& settings) + VideoStreamEncoderUnderTest( + SendStatisticsProxy* stats_proxy, + const VideoSendStream::Config::EncoderSettings& settings) : VideoStreamEncoder( 1 /* number_of_cores */, stats_proxy, @@ -103,7 +75,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { nullptr /* pre_encode_callback */, nullptr /* encoder_timing */, std::unique_ptr( - overuse_detector_proxy_ = new CpuOveruseDetectorProxy( + overuse_detector_ = new OveruseFrameDetector( GetCpuOveruseOptions(settings.full_overuse_time), this, nullptr, @@ -136,7 +108,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); } - CpuOveruseDetectorProxy* overuse_detector_proxy_; + OveruseFrameDetector* overuse_detector_; }; class VideoStreamFactory @@ -2238,126 +2210,6 @@ TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) { video_stream_encoder_->Stop(); } -TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) { - const int kFrameWidth = 1280; - const int kFrameHeight = 720; - const int kFramerate = 24; - - video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); - test::FrameForwarder source; - video_stream_encoder_->SetSource( - &source, VideoSendStream::DegradationPreference::kMaintainResolution); - - // Insert a single frame, triggering initial configuration. - source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - - EXPECT_EQ( - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), - kDefaultFramerate); - - // Trigger reconfigure encoder (without resetting the entire instance). - VideoEncoderConfig video_encoder_config; - video_encoder_config.max_bitrate_bps = kTargetBitrateBps; - video_encoder_config.number_of_streams = 1; - video_encoder_config.video_stream_factory = - new rtc::RefCountedObject(1, kFramerate); - video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), - kMaxPayloadLength, false); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - - // Detector should be updated with fps limit from codec config. - EXPECT_EQ( - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), - kFramerate); - - // Trigger overuse, max framerate should be reduced. - VideoSendStream::Stats stats = stats_proxy_->GetStats(); - stats.input_frame_rate = kFramerate; - stats_proxy_->SetMockStats(stats); - video_stream_encoder_->TriggerCpuOveruse(); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - int adapted_framerate = - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); - EXPECT_LT(adapted_framerate, kFramerate); - - // Trigger underuse, max framerate should go back to codec configured fps. - // Set extra low fps, to make sure it's actually reset, not just incremented. - stats = stats_proxy_->GetStats(); - stats.input_frame_rate = adapted_framerate / 2; - stats_proxy_->SetMockStats(stats); - video_stream_encoder_->TriggerCpuNormalUsage(); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - EXPECT_EQ( - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), - kFramerate); - - video_stream_encoder_->Stop(); -} - -TEST_F(VideoStreamEncoderTest, - OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) { - const int kFrameWidth = 1280; - const int kFrameHeight = 720; - const int kLowFramerate = 15; - const int kHighFramerate = 25; - - video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); - test::FrameForwarder source; - video_stream_encoder_->SetSource( - &source, VideoSendStream::DegradationPreference::kMaintainResolution); - - // Trigger initial configuration. - VideoEncoderConfig video_encoder_config; - video_encoder_config.max_bitrate_bps = kTargetBitrateBps; - video_encoder_config.number_of_streams = 1; - video_encoder_config.video_stream_factory = - new rtc::RefCountedObject(1, kLowFramerate); - source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); - video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), - kMaxPayloadLength, false); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - - EXPECT_EQ( - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), - kLowFramerate); - - // Trigger overuse, max framerate should be reduced. - VideoSendStream::Stats stats = stats_proxy_->GetStats(); - stats.input_frame_rate = kLowFramerate; - stats_proxy_->SetMockStats(stats); - video_stream_encoder_->TriggerCpuOveruse(); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - int adapted_framerate = - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); - EXPECT_LT(adapted_framerate, kLowFramerate); - - // Reconfigure the encoder with a new (higher max framerate), max fps should - // still respect the adaptation. - video_encoder_config.video_stream_factory = - new rtc::RefCountedObject(1, kHighFramerate); - source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); - video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), - kMaxPayloadLength, false); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - - EXPECT_EQ( - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), - adapted_framerate); - - // Trigger underuse, max framerate should go back to codec configured fps. - stats = stats_proxy_->GetStats(); - stats.input_frame_rate = adapted_framerate; - stats_proxy_->SetMockStats(stats); - video_stream_encoder_->TriggerCpuNormalUsage(); - video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - EXPECT_EQ( - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), - kHighFramerate); - - video_stream_encoder_->Stop(); -} - TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnDegradationPreferenceChange) { const int kFrameWidth = 1280; @@ -2380,9 +2232,7 @@ TEST_F(VideoStreamEncoderTest, kMaxPayloadLength, false); video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - EXPECT_EQ( - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), - kFramerate); + EXPECT_GT(source.sink_wants().max_framerate_fps, kFramerate); // Trigger overuse, max framerate should be reduced. VideoSendStream::Stats stats = stats_proxy_->GetStats(); @@ -2390,8 +2240,8 @@ TEST_F(VideoStreamEncoderTest, stats_proxy_->SetMockStats(stats); video_stream_encoder_->TriggerCpuOveruse(); video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - int adapted_framerate = - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + + int adapted_framerate = source.sink_wants().max_framerate_fps; EXPECT_LT(adapted_framerate, kFramerate); // Change degradation preference to not enable framerate scaling. Target @@ -2399,9 +2249,7 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->SetSource( &source, VideoSendStream::DegradationPreference::kMaintainFramerate); video_stream_encoder_->WaitUntilTaskQueueIsIdle(); - EXPECT_EQ( - video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), - kFramerate); + EXPECT_GT(source.sink_wants().max_framerate_fps, kFramerate); video_stream_encoder_->Stop(); }