From fda496a31e5c4d0aac98d3309eb288495748ba8f Mon Sep 17 00:00:00 2001 From: sprang Date: Thu, 15 Jun 2017 04:21:07 -0700 Subject: [PATCH] Set overuse detector max frame interval based on target frame rate. Currently there is a hard limit for the estimated captured frame interval of 45ms. As the encoder utilization is calculated as (input frame interval)/(encode time), overuse signals can be triggered even though there is plenty of time to go around if the fps is low. However, in order to avoid falsly estimating low encode usage in case the capturer has a dynamic frame rate, set the frame interval based on the actual current max framerate. BUG=webrtc:4172 Review-Url: https://codereview.webrtc.org/2918143003 Cr-Commit-Position: refs/heads/master@{#18610} --- .../video/encoder_rtcp_feedback_unittest.cc | 5 +- webrtc/video/overuse_frame_detector.cc | 39 +++- webrtc/video/overuse_frame_detector.h | 10 +- .../video/overuse_frame_detector_unittest.cc | 125 +++++++++++ webrtc/video/video_send_stream.cc | 7 +- webrtc/video/vie_encoder.cc | 173 +++++++++------ webrtc/video/vie_encoder.h | 10 +- webrtc/video/vie_encoder_unittest.cc | 209 +++++++++++++++++- 8 files changed, 499 insertions(+), 79 deletions(-) diff --git a/webrtc/video/encoder_rtcp_feedback_unittest.cc b/webrtc/video/encoder_rtcp_feedback_unittest.cc index 1606a59357..12c0951ea0 100644 --- a/webrtc/video/encoder_rtcp_feedback_unittest.cc +++ b/webrtc/video/encoder_rtcp_feedback_unittest.cc @@ -10,6 +10,8 @@ #include "webrtc/video/encoder_rtcp_feedback.h" +#include + #include "webrtc/modules/utility/include/mock/mock_process_thread.h" #include "webrtc/test/gmock.h" #include "webrtc/test/gtest.h" @@ -27,7 +29,8 @@ class MockVieEncoder : public ViEEncoder { send_stats_proxy, VideoSendStream::Config::EncoderSettings("fake", 0, nullptr), nullptr, - nullptr) {} + nullptr, + std::unique_ptr()) {} ~MockVieEncoder() { Stop(); } MOCK_METHOD1(OnReceivedIntraFrameRequest, void(size_t)); diff --git a/webrtc/video/overuse_frame_detector.cc b/webrtc/video/overuse_frame_detector.cc index 9f650579ea..9711ddb039 100644 --- a/webrtc/video/overuse_frame_detector.cc +++ b/webrtc/video/overuse_frame_detector.cc @@ -49,8 +49,20 @@ const double kRampUpBackoffFactor = 2.0; const int kMaxOverusesBeforeApplyRampupDelay = 4; // The maximum exponent to use in VCMExpFilter. -const float kSampleDiffMs = 33.0f; const float kMaxExp = 7.0f; +// Default value used before first reconfiguration. +const int kDefaultFrameRate = 30; +// Default sample diff, default frame rate. +const float kDefaultSampleDiffMs = 1000.0f / kDefaultFrameRate; +// A factor applied to the sample diff on OnTargetFramerateUpdated to determine +// a max limit for the sample diff. For instance, with a framerate of 30fps, +// the sample diff is capped to (1000 / 30) * 1.35 = 45ms. This prevents +// triggering too soon if there are individual very large outliers. +const float kMaxSampleDiffMarginFactor = 1.35f; +// Minimum framerate allowed for usage calculation. This prevents crazy long +// encode times from being accepted if the frame rate happens to be low. +const int kMinFramerate = 7; +const int kMaxFramerate = 30; const auto kScaleReasonCpu = AdaptationObserverInterface::AdaptReason::kCpu; } // namespace @@ -113,9 +125,9 @@ class OveruseFrameDetector::SendProcessingUsage { : kWeightFactorFrameDiff(0.998f), kWeightFactorProcessing(0.995f), kInitialSampleDiffMs(40.0f), - kMaxSampleDiffMs(45.0f), count_(0), options_(options), + max_sample_diff_ms_(kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor), filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)), filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) { Reset(); @@ -124,21 +136,24 @@ class OveruseFrameDetector::SendProcessingUsage { void Reset() { count_ = 0; + max_sample_diff_ms_ = kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor; filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff); filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs); filtered_processing_ms_->Reset(kWeightFactorProcessing); filtered_processing_ms_->Apply(1.0f, InitialProcessingMs()); } + void SetMaxSampleDiffMs(float diff_ms) { max_sample_diff_ms_ = diff_ms; } + void AddCaptureSample(float sample_ms) { - float exp = sample_ms / kSampleDiffMs; + float exp = sample_ms / kDefaultSampleDiffMs; exp = std::min(exp, kMaxExp); filtered_frame_diff_ms_->Apply(exp, sample_ms); } void AddSample(float processing_ms, int64_t diff_last_sample_ms) { ++count_; - float exp = diff_last_sample_ms / kSampleDiffMs; + float exp = diff_last_sample_ms / kDefaultSampleDiffMs; exp = std::min(exp, kMaxExp); filtered_processing_ms_->Apply(exp, processing_ms); } @@ -148,7 +163,7 @@ class OveruseFrameDetector::SendProcessingUsage { return static_cast(InitialUsageInPercent() + 0.5f); } float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f); - frame_diff_ms = std::min(frame_diff_ms, kMaxSampleDiffMs); + frame_diff_ms = std::min(frame_diff_ms, max_sample_diff_ms_); float encode_usage_percent = 100.0f * filtered_processing_ms_->filtered() / frame_diff_ms; return static_cast(encode_usage_percent + 0.5); @@ -168,9 +183,9 @@ class OveruseFrameDetector::SendProcessingUsage { const float kWeightFactorFrameDiff; const float kWeightFactorProcessing; const float kInitialSampleDiffMs; - const float kMaxSampleDiffMs; uint64_t count_; const CpuOveruseOptions options_; + float max_sample_diff_ms_; std::unique_ptr filtered_processing_ms_; std::unique_ptr filtered_frame_diff_ms_; }; @@ -331,6 +346,7 @@ OveruseFrameDetector::OveruseFrameDetector( last_capture_time_us_(-1), last_processed_capture_time_us_(-1), num_pixels_(0), + max_framerate_(kDefaultFrameRate), last_overuse_time_ms_(-1), checks_above_threshold_(0), num_overuse_detections_(0), @@ -382,6 +398,8 @@ bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now_us) const { } void OveruseFrameDetector::ResetAll(int num_pixels) { + // Reset state, as a result resolution being changed. Do not however change + // the current frame rate back to the default. RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); num_pixels_ = num_pixels; usage_->Reset(); @@ -390,6 +408,15 @@ void OveruseFrameDetector::ResetAll(int num_pixels) { last_processed_capture_time_us_ = -1; num_process_times_ = 0; metrics_ = rtc::Optional(); + OnTargetFramerateUpdated(max_framerate_); +} + +void OveruseFrameDetector::OnTargetFramerateUpdated(int framerate_fps) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + RTC_DCHECK_GE(framerate_fps, 0); + max_framerate_ = std::min(kMaxFramerate, framerate_fps); + usage_->SetMaxSampleDiffMs((1000 / std::max(kMinFramerate, max_framerate_)) * + kMaxSampleDiffMarginFactor); } void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame, diff --git a/webrtc/video/overuse_frame_detector.h b/webrtc/video/overuse_frame_detector.h index 2fb3104d53..29efa1609d 100644 --- a/webrtc/video/overuse_frame_detector.h +++ b/webrtc/video/overuse_frame_detector.h @@ -68,7 +68,7 @@ class OveruseFrameDetector { AdaptationObserverInterface* overuse_observer, EncodedFrameObserver* encoder_timing_, CpuOveruseMetricsObserver* metrics_observer); - ~OveruseFrameDetector(); + virtual ~OveruseFrameDetector(); // Start to periodically check for overuse. void StartCheckForOveruse(); @@ -77,6 +77,13 @@ class OveruseFrameDetector { // StartCheckForOveruse has been called. void StopCheckForOveruse(); + // Defines the current maximum framerate targeted by the capturer. This is + // used to make sure the encode usage percent doesn't drop unduly if the + // capturer has quiet periods (for instance caused by screen capturers with + // variable capture rate depending on content updates), otherwise we might + // experience adaptation toggling. + virtual void OnTargetFramerateUpdated(int framerate_fps); + // Called for each captured frame. void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); @@ -135,6 +142,7 @@ class OveruseFrameDetector { // Number of pixels of last captured frame. int num_pixels_ GUARDED_BY(task_checker_); + int max_framerate_ GUARDED_BY(task_checker_); int64_t last_overuse_time_ms_ GUARDED_BY(task_checker_); int checks_above_threshold_ GUARDED_BY(task_checker_); int num_overuse_detections_ GUARDED_BY(task_checker_); diff --git a/webrtc/video/overuse_frame_detector_unittest.cc b/webrtc/video/overuse_frame_detector_unittest.cc index bf1c8b4aca..ae3b687acc 100644 --- a/webrtc/video/overuse_frame_detector_unittest.cc +++ b/webrtc/video/overuse_frame_detector_unittest.cc @@ -139,6 +139,20 @@ class OveruseFrameDetectorTest : public ::testing::Test, int UsagePercent() { return metrics_.encode_usage_percent; } + int64_t OveruseProcessingTimeLimitForFramerate(int fps) const { + int64_t frame_interval = rtc::kNumMicrosecsPerSec / fps; + int64_t max_processing_time_us = + (frame_interval * options_.high_encode_usage_threshold_percent) / 100; + return max_processing_time_us; + } + + int64_t UnderuseProcessingTimeLimitForFramerate(int fps) const { + int64_t frame_interval = rtc::kNumMicrosecsPerSec / fps; + int64_t max_processing_time_us = + (frame_interval * options_.low_encode_usage_threshold_percent) / 100; + return max_processing_time_us; + } + CpuOveruseOptions options_; rtc::ScopedFakeClock clock_; std::unique_ptr observer_; @@ -355,4 +369,115 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { EXPECT_TRUE(event.Wait(10000)); } +TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) { + const int kCapturerMaxFrameRate = 30; + const int kEncodeMaxFrameRate = 20; // Maximum fps the encoder can sustain. + + // Trigger overuse. + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kCapturerMaxFrameRate; + // Processing time just below over use limit given kEncodeMaxFrameRate. + int64_t processing_time_us = + (98 * OveruseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Simulate frame rate reduction and normal usage. + frame_interval_us = rtc::kNumMicrosecsPerSec / kEncodeMaxFrameRate; + overuse_detector_->OnTargetFramerateUpdated(kEncodeMaxFrameRate); + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Reduce processing time to trigger underuse. + processing_time_us = + (98 * UnderuseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(1); + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); +} + +TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) { + const int kMinFrameRate = 7; // Minimum fps allowed by current detector impl. + overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate); + + // Normal usage just at the limit. + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMinFrameRate; + // Processing time just below over use limit given kEncodeMaxFrameRate. + int64_t processing_time_us = + (98 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Over the limit to overuse. + processing_time_us = + (102 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Reduce input frame rate. Should still trigger overuse. + overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate - 1); + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } +} + +TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) { + const int kMaxFrameRate = 20; + overuse_detector_->OnTargetFramerateUpdated(kMaxFrameRate); + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMaxFrameRate; + // Maximum frame interval allowed is 35% above ideal. + int64_t max_frame_interval_us = (135 * frame_interval_us) / 100; + // Maximum processing time, without triggering overuse, allowed with the above + // frame interval. + int64_t max_processing_time_us = + (max_frame_interval_us * options_.high_encode_usage_threshold_percent) / + 100; + + // Processing time just below overuse limit given kMaxFrameRate. + int64_t processing_time_us = (98 * max_processing_time_us) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Go above limit, trigger overuse. + processing_time_us = (102 * max_processing_time_us) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Increase frame interval, should still trigger overuse. + max_frame_interval_us *= 2; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(); + } +} + } // namespace webrtc diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc index 7bf32c6dba..285b7ef033 100644 --- a/webrtc/video/video_send_stream.cc +++ b/webrtc/video/video_send_stream.cc @@ -632,9 +632,10 @@ VideoSendStream::VideoSendStream( encoder_config.content_type), config_(std::move(config)), content_type_(encoder_config.content_type) { - vie_encoder_.reset(new ViEEncoder( - num_cpu_cores, &stats_proxy_, config_.encoder_settings, - config_.pre_encode_callback, config_.post_encode_callback)); + vie_encoder_.reset( + new ViEEncoder(num_cpu_cores, &stats_proxy_, config_.encoder_settings, + config_.pre_encode_callback, config_.post_encode_callback, + std::unique_ptr())); worker_queue_->PostTask(std::unique_ptr(new ConstructionTask( &send_stream_, &thread_sync_event_, &stats_proxy_, vie_encoder_.get(), module_process_thread, call_stats, transport, bitrate_allocator, diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 63b1c80fef..d82e4fc7a2 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -44,24 +44,12 @@ const int64_t kFrameLogIntervalMs = 60000; // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206 const int kMinPixelsPerFrame = 320 * 180; const int kMinFramerateFps = 2; +const int kMaxFramerateFps = 120; // The maximum number of frames to drop at beginning of stream // to try and achieve desired bitrate. const int kMaxInitialFramedrop = 4; -// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle -// pipelining encoders better (multiple input frames before something comes -// out). This should effectively turn off CPU adaptations for systems that -// remotely cope with the load right now. -CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { - CpuOveruseOptions options; - if (full_overuse_time) { - options.low_encode_usage_threshold_percent = 150; - options.high_encode_usage_threshold_percent = 200; - } - return options; -} - uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) { if (kbps > 0) { if (kbps < 300 /* qvga */) { @@ -207,7 +195,7 @@ class ViEEncoder::VideoSourceProxy { degradation_preference_ = degradation_preference; old_source = source_; source_ = source; - wants = GetActiveSinkWants(); + wants = GetActiveSinkWantsInternal(); } if (old_source != source && old_source != nullptr) { @@ -228,26 +216,9 @@ class ViEEncoder::VideoSourceProxy { source_->AddOrUpdateSink(vie_encoder_, sink_wants_); } - rtc::VideoSinkWants GetActiveSinkWants() EXCLUSIVE_LOCKS_REQUIRED(&crit_) { - rtc::VideoSinkWants wants = sink_wants_; - // Clear any constraints from the current sink wants that don't apply to - // the used degradation_preference. - switch (degradation_preference_) { - case VideoSendStream::DegradationPreference::kBalanced: - break; - case VideoSendStream::DegradationPreference::kMaintainFramerate: - wants.max_framerate_fps = std::numeric_limits::max(); - break; - case VideoSendStream::DegradationPreference::kMaintainResolution: - wants.max_pixel_count = std::numeric_limits::max(); - wants.target_pixel_count.reset(); - break; - case VideoSendStream::DegradationPreference::kDegradationDisabled: - wants.max_pixel_count = std::numeric_limits::max(); - wants.target_pixel_count.reset(); - wants.max_framerate_fps = std::numeric_limits::max(); - } - return wants; + rtc::VideoSinkWants GetActiveSinkWants() { + rtc::CritScope lock(&crit_); + return GetActiveSinkWantsInternal(); } void ResetPixelFpsCount() { @@ -277,14 +248,15 @@ class ViEEncoder::VideoSourceProxy { LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted; sink_wants_.max_pixel_count = pixels_wanted; sink_wants_.target_pixel_count = rtc::Optional(); - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); return true; } - bool RequestFramerateLowerThan(int fps) { + int RequestFramerateLowerThan(int fps) { // Called on the encoder task queue. // The input video frame rate will be scaled down to 2/3, rounding down. - return RestrictFramerate((fps * 2) / 3); + int framerate_wanted = (fps * 2) / 3; + return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; } bool RequestHigherResolutionThan(int pixel_count) { @@ -317,18 +289,22 @@ class ViEEncoder::VideoSourceProxy { rtc::Optional((pixel_count * 5) / 3); } LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); return true; } - bool RequestHigherFramerateThan(int fps) { + // Request upgrade in framerate. Returns the new requested frame, or -1 if + // no change requested. Note that maxint may be returned if limits due to + // adaptation requests are removed completely. In that case, consider + // |max_framerate_| to be the current limit (assuming the capturer complies). + int RequestHigherFramerateThan(int fps) { // Called on the encoder task queue. // The input frame rate will be scaled up to the last step, with rounding. int framerate_wanted = fps; if (fps != std::numeric_limits::max()) framerate_wanted = (fps * 3) / 2; - return IncreaseFramerate(framerate_wanted); + return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; } bool RestrictFramerate(int fps) { @@ -343,7 +319,7 @@ class ViEEncoder::VideoSourceProxy { LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; sink_wants_.max_framerate_fps = fps_wanted; - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); return true; } @@ -359,11 +335,34 @@ class ViEEncoder::VideoSourceProxy { LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; sink_wants_.max_framerate_fps = fps_wanted; - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); return true; } private: + rtc::VideoSinkWants GetActiveSinkWantsInternal() + EXCLUSIVE_LOCKS_REQUIRED(&crit_) { + rtc::VideoSinkWants wants = sink_wants_; + // Clear any constraints from the current sink wants that don't apply to + // the used degradation_preference. + switch (degradation_preference_) { + case VideoSendStream::DegradationPreference::kBalanced: + break; + case VideoSendStream::DegradationPreference::kMaintainFramerate: + wants.max_framerate_fps = std::numeric_limits::max(); + break; + case VideoSendStream::DegradationPreference::kMaintainResolution: + wants.max_pixel_count = std::numeric_limits::max(); + wants.target_pixel_count.reset(); + break; + case VideoSendStream::DegradationPreference::kDegradationDisabled: + wants.max_pixel_count = std::numeric_limits::max(); + wants.target_pixel_count.reset(); + wants.max_framerate_fps = std::numeric_limits::max(); + } + return wants; + } + rtc::CriticalSection crit_; rtc::SequencedTaskChecker main_checker_; ViEEncoder* const vie_encoder_; @@ -379,7 +378,8 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, SendStatisticsProxy* stats_proxy, const VideoSendStream::Config::EncoderSettings& settings, rtc::VideoSinkInterface* pre_encode_callback, - EncodedFrameObserver* encoder_timing) + EncodedFrameObserver* encoder_timing, + std::unique_ptr overuse_detector) : shutdown_event_(true /* manual_reset */, false), number_of_cores_(number_of_cores), initial_rampup_(0), @@ -389,13 +389,18 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, codec_type_(PayloadNameToCodecType(settings.payload_name) .value_or(VideoCodecType::kVideoCodecUnknown)), video_sender_(Clock::GetRealTimeClock(), this, this), - overuse_detector_(GetCpuOveruseOptions(settings.full_overuse_time), - this, - encoder_timing, - stats_proxy), + overuse_detector_( + overuse_detector.get() + ? overuse_detector.release() + : new OveruseFrameDetector( + GetCpuOveruseOptions(settings.full_overuse_time), + this, + encoder_timing, + stats_proxy)), stats_proxy_(stats_proxy), pre_encode_callback_(pre_encode_callback), module_process_thread_(nullptr), + max_framerate_(-1), pending_encoder_reconfiguration_(false), encoder_start_bitrate_bps_(0), max_data_payload_length_(0), @@ -416,7 +421,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, RTC_DCHECK(stats_proxy); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); - overuse_detector_.StartCheckForOveruse(); + overuse_detector_->StartCheckForOveruse(); video_sender_.RegisterExternalEncoder( settings_.encoder, settings_.payload_type, settings_.internal_source); }); @@ -428,12 +433,25 @@ ViEEncoder::~ViEEncoder() { << "Must call ::Stop() before destruction."; } +// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle +// pipelining encoders better (multiple input frames before something comes +// out). This should effectively turn off CPU adaptations for systems that +// remotely cope with the load right now. +CpuOveruseOptions ViEEncoder::GetCpuOveruseOptions(bool full_overuse_time) { + CpuOveruseOptions options; + if (full_overuse_time) { + options.low_encode_usage_threshold_percent = 150; + options.high_encode_usage_threshold_percent = 200; + } + return options; +} + void ViEEncoder::Stop() { RTC_DCHECK_RUN_ON(&thread_checker_); source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); - overuse_detector_.StopCheckForOveruse(); + overuse_detector_->StopCheckForOveruse(); rate_allocator_.reset(); bitrate_observer_ = nullptr; video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, @@ -493,6 +511,12 @@ void ViEEncoder::SetSource( bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_); initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop; ConfigureQualityScaler(); + if (!IsFramerateScalingEnabled(degradation_preference) && + max_framerate_ != -1) { + // If frame rate scaling is no longer allowed, remove any potential + // allowance for longer frame intervals. + overuse_detector_->OnTargetFramerateUpdated(max_framerate_); + } }); } @@ -562,6 +586,8 @@ void ViEEncoder::ReconfigureEncoder() { std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate); codec.expect_encode_from_texture = last_frame_info_->is_texture; + max_framerate_ = codec.maxFramerate; + RTC_DCHECK_LE(max_framerate_, kMaxFramerateFps); bool success = video_sender_.RegisterSendCodec( &codec, number_of_cores_, @@ -574,19 +600,31 @@ void ViEEncoder::ReconfigureEncoder() { video_sender_.UpdateChannelParemeters(rate_allocator_.get(), bitrate_observer_); - int framerate = stats_proxy_->GetSendFrameRate(); - if (framerate == 0) - framerate = codec.maxFramerate; + // Get the current actual framerate, as measured by the stats proxy. This is + // used to get the correct bitrate layer allocation. + int current_framerate = stats_proxy_->GetSendFrameRate(); + if (current_framerate == 0) + current_framerate = codec.maxFramerate; stats_proxy_->OnEncoderReconfigured( - encoder_config_, rate_allocator_.get() - ? rate_allocator_->GetPreferredBitrateBps(framerate) - : codec.maxBitrate); + encoder_config_, + rate_allocator_.get() + ? rate_allocator_->GetPreferredBitrateBps(current_framerate) + : codec.maxBitrate); pending_encoder_reconfiguration_ = false; sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); + // Get the current target framerate, ie the maximum framerate as specified by + // the current codec configuration, or any limit imposed by cpu adaption in + // maintain-resolution or balanced mode. This is used to make sure overuse + // detection doesn't needlessly trigger in low and/or variable framerate + // scenarios. + int target_framerate = std::min( + max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); + overuse_detector_->OnTargetFramerateUpdated(target_framerate); + ConfigureQualityScaler(); } @@ -735,7 +773,7 @@ void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); - overuse_detector_.FrameCaptured(video_frame, time_when_posted_us); + overuse_detector_->FrameCaptured(video_frame, time_when_posted_us); video_sender_.AddVideoFrame(video_frame, nullptr); } @@ -766,7 +804,7 @@ EncodedImageCallback::Result ViEEncoder::OnEncodedImage( const int qp = encoded_image.qp_; encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] { RTC_DCHECK_RUN_ON(&encoder_queue_); - overuse_detector_.FrameSent(timestamp, time_sent_us); + overuse_detector_->FrameSent(timestamp, time_sent_us); if (quality_scaler_ && qp >= 0) quality_scaler_->ReportQP(qp); }); @@ -901,14 +939,18 @@ void ViEEncoder::AdaptDown(AdaptReason reason) { } GetAdaptCounter().IncrementResolution(reason); break; - case VideoSendStream::DegradationPreference::kMaintainResolution: + case VideoSendStream::DegradationPreference::kMaintainResolution: { // Scale down framerate. - if (!source_proxy_->RequestFramerateLowerThan( - adaptation_request.framerate_fps_)) { + const int requested_framerate = source_proxy_->RequestFramerateLowerThan( + adaptation_request.framerate_fps_); + if (requested_framerate == -1) return; - } + RTC_DCHECK_NE(max_framerate_, -1); + overuse_detector_->OnTargetFramerateUpdated( + std::min(max_framerate_, requested_framerate)); GetAdaptCounter().IncrementFramerate(reason); break; + } case VideoSendStream::DegradationPreference::kDegradationDisabled: RTC_NOTREACHED(); } @@ -985,8 +1027,15 @@ void ViEEncoder::AdaptUp(AdaptReason reason) { LOG(LS_INFO) << "Removing framerate down-scaling setting."; fps = std::numeric_limits::max(); } - if (!source_proxy_->RequestHigherFramerateThan(fps)) + + const int requested_framerate = + source_proxy_->RequestHigherFramerateThan(fps); + if (requested_framerate == -1) { + overuse_detector_->OnTargetFramerateUpdated(max_framerate_); return; + } + overuse_detector_->OnTargetFramerateUpdated( + std::min(max_framerate_, requested_framerate)); GetAdaptCounter().DecrementFramerate(reason); break; } diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 460abf128e..77a6bc417d 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -77,7 +77,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, SendStatisticsProxy* stats_proxy, const VideoSendStream::Config::EncoderSettings& settings, rtc::VideoSinkInterface* pre_encode_callback, - EncodedFrameObserver* encoder_timing); + EncodedFrameObserver* encoder_timing, + std::unique_ptr overuse_detector); ~ViEEncoder(); // RegisterProcessThread register |module_process_thread| with those objects // that use it. Registration has to happen on the thread where @@ -129,6 +130,7 @@ class ViEEncoder : public rtc::VideoSinkInterface, // These methods are protected for easier testing. void AdaptUp(AdaptReason reason) override; void AdaptDown(AdaptReason reason) override; + static CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time); private: class ConfigureEncoderTask; @@ -232,7 +234,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, const VideoCodecType codec_type_; vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); - OveruseFrameDetector overuse_detector_ ACCESS_ON(&encoder_queue_); + std::unique_ptr overuse_detector_ + ACCESS_ON(&encoder_queue_); std::unique_ptr quality_scaler_ ACCESS_ON(&encoder_queue_); SendStatisticsProxy* const stats_proxy_; @@ -246,6 +249,9 @@ class ViEEncoder : public rtc::VideoSinkInterface, VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_); std::unique_ptr rate_allocator_ ACCESS_ON(&encoder_queue_); + // The maximum frame rate of the current codec configuration, as determined + // at the last ReconfigureEncoder() call. + int max_framerate_ ACCESS_ON(&encoder_queue_); // Set when ConfigureEncoder has been called in order to lazy reconfigure the // encoder on the next frame. diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc index a016fa9313..b143e4d546 100644 --- a/webrtc/video/vie_encoder_unittest.cc +++ b/webrtc/video/vie_encoder_unittest.cc @@ -48,6 +48,7 @@ const size_t kMaxPayloadLength = 1440; const int kTargetBitrateBps = 1000000; const int kLowTargetBitrateBps = kTargetBitrateBps / 10; const int kMaxInitialFramedrop = 4; +const int kDefaultFramerate = 30; class TestBuffer : public webrtc::I420Buffer { public: @@ -63,6 +64,35 @@ class TestBuffer : public webrtc::I420Buffer { rtc::Event* const event_; }; +class CpuOveruseDetectorProxy : public OveruseFrameDetector { + public: + CpuOveruseDetectorProxy(const CpuOveruseOptions& options, + AdaptationObserverInterface* overuse_observer, + EncodedFrameObserver* encoder_timing_, + CpuOveruseMetricsObserver* metrics_observer) + : OveruseFrameDetector(options, + overuse_observer, + encoder_timing_, + metrics_observer), + last_target_framerate_fps_(-1) {} + virtual ~CpuOveruseDetectorProxy() {} + + void OnTargetFramerateUpdated(int framerate_fps) override { + rtc::CritScope cs(&lock_); + last_target_framerate_fps_ = framerate_fps; + OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps); + } + + int GetLastTargetFramerate() { + rtc::CritScope cs(&lock_); + return last_target_framerate_fps_; + } + + private: + rtc::CriticalSection lock_; + int last_target_framerate_fps_ GUARDED_BY(lock_); +}; + class ViEEncoderUnderTest : public ViEEncoder { public: ViEEncoderUnderTest(SendStatisticsProxy* stats_proxy, @@ -71,7 +101,13 @@ class ViEEncoderUnderTest : public ViEEncoder { stats_proxy, settings, nullptr /* pre_encode_callback */, - nullptr /* encoder_timing */) {} + nullptr /* encoder_timing */, + std::unique_ptr( + overuse_detector_proxy_ = new CpuOveruseDetectorProxy( + GetCpuOveruseOptions(settings.full_overuse_time), + this, + nullptr, + stats_proxy))) {} void PostTaskAndWait(bool down, AdaptReason reason) { rtc::Event event(false, false); @@ -99,14 +135,17 @@ class ViEEncoderUnderTest : public ViEEncoder { void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); } void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); } + + CpuOveruseDetectorProxy* overuse_detector_proxy_; }; class VideoStreamFactory : public VideoEncoderConfig::VideoStreamFactoryInterface { public: - explicit VideoStreamFactory(size_t num_temporal_layers) - : num_temporal_layers_(num_temporal_layers) { + explicit VideoStreamFactory(size_t num_temporal_layers, int framerate) + : num_temporal_layers_(num_temporal_layers), framerate_(framerate) { EXPECT_GT(num_temporal_layers, 0u); + EXPECT_GT(framerate, 0); } private: @@ -118,10 +157,13 @@ class VideoStreamFactory test::CreateVideoStreams(width, height, encoder_config); for (VideoStream& stream : streams) { stream.temporal_layer_thresholds_bps.resize(num_temporal_layers_ - 1); + stream.max_framerate = framerate_; } return streams; } + const size_t num_temporal_layers_; + const int framerate_; }; class AdaptingFrameForwarder : public test::FrameForwarder { @@ -264,7 +306,8 @@ class ViEEncoderTest : public ::testing::Test { video_encoder_config.number_of_streams = num_streams; video_encoder_config.max_bitrate_bps = kTargetBitrateBps; video_encoder_config.video_stream_factory = - new rtc::RefCountedObject(num_temporal_layers); + new rtc::RefCountedObject(num_temporal_layers, + kDefaultFramerate); ConfigureEncoder(std::move(video_encoder_config), nack_enabled); } @@ -2027,6 +2070,164 @@ TEST_F(ViEEncoderTest, CallsBitrateObserver) { vie_encoder_->Stop(); } +TEST_F(ViEEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) { + const int kFrameWidth = 1280; + const int kFrameHeight = 720; + const int kFramerate = 24; + + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + test::FrameForwarder source; + vie_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainResolution); + + // Insert a single frame, triggering initial configuration. + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kDefaultFramerate); + + // Trigger reconfigure encoder (without resetting the entire instance). + VideoEncoderConfig video_encoder_config; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.number_of_streams = 1; + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kFramerate); + vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + // Detector should be updated with fps limit from codec config. + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + // Trigger overuse, max framerate should be reduced. + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + stats.input_frame_rate = kFramerate; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuOveruse(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + int adapted_framerate = + vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + EXPECT_LT(adapted_framerate, kFramerate); + + // Trigger underuse, max framerate should go back to codec configured fps. + // Set extra low fps, to make sure it's actually reset, not just incremented. + stats = stats_proxy_->GetStats(); + stats.input_frame_rate = adapted_framerate / 2; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuNormalUsage(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) { + const int kFrameWidth = 1280; + const int kFrameHeight = 720; + const int kLowFramerate = 15; + const int kHighFramerate = 25; + + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + test::FrameForwarder source; + vie_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainResolution); + + // Trigger initial configuration. + VideoEncoderConfig video_encoder_config; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.number_of_streams = 1; + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kLowFramerate); + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kLowFramerate); + + // Trigger overuse, max framerate should be reduced. + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + stats.input_frame_rate = kLowFramerate; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuOveruse(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + int adapted_framerate = + vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + EXPECT_LT(adapted_framerate, kLowFramerate); + + // Reconfigure the encoder with a new (higher max framerate), max fps should + // still respect the adaptation. + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kHighFramerate); + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + adapted_framerate); + + // Trigger underuse, max framerate should go back to codec configured fps. + stats = stats_proxy_->GetStats(); + stats.input_frame_rate = adapted_framerate; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuNormalUsage(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kHighFramerate); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, OveruseDetectorUpdatedOnDegradationPreferenceChange) { + const int kFrameWidth = 1280; + const int kFrameHeight = 720; + const int kFramerate = 24; + + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + test::FrameForwarder source; + vie_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainResolution); + + // Trigger initial configuration. + VideoEncoderConfig video_encoder_config; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.number_of_streams = 1; + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kFramerate); + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + // Trigger overuse, max framerate should be reduced. + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + stats.input_frame_rate = kFramerate; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuOveruse(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + int adapted_framerate = + vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + EXPECT_LT(adapted_framerate, kFramerate); + + // Change degradation preference to not enable framerate scaling. Target + // framerate should be changed to codec defined limit. + vie_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainFramerate); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + vie_encoder_->Stop(); +} + TEST_F(ViEEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) { const int kTooLowBitrateForFrameSizeBps = 10000; vie_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);