diff --git a/webrtc/video/encoder_rtcp_feedback_unittest.cc b/webrtc/video/encoder_rtcp_feedback_unittest.cc index 1606a59357..12c0951ea0 100644 --- a/webrtc/video/encoder_rtcp_feedback_unittest.cc +++ b/webrtc/video/encoder_rtcp_feedback_unittest.cc @@ -10,6 +10,8 @@ #include "webrtc/video/encoder_rtcp_feedback.h" +#include + #include "webrtc/modules/utility/include/mock/mock_process_thread.h" #include "webrtc/test/gmock.h" #include "webrtc/test/gtest.h" @@ -27,7 +29,8 @@ class MockVieEncoder : public ViEEncoder { send_stats_proxy, VideoSendStream::Config::EncoderSettings("fake", 0, nullptr), nullptr, - nullptr) {} + nullptr, + std::unique_ptr()) {} ~MockVieEncoder() { Stop(); } MOCK_METHOD1(OnReceivedIntraFrameRequest, void(size_t)); diff --git a/webrtc/video/overuse_frame_detector.cc b/webrtc/video/overuse_frame_detector.cc index 9f650579ea..9711ddb039 100644 --- a/webrtc/video/overuse_frame_detector.cc +++ b/webrtc/video/overuse_frame_detector.cc @@ -49,8 +49,20 @@ const double kRampUpBackoffFactor = 2.0; const int kMaxOverusesBeforeApplyRampupDelay = 4; // The maximum exponent to use in VCMExpFilter. -const float kSampleDiffMs = 33.0f; const float kMaxExp = 7.0f; +// Default value used before first reconfiguration. +const int kDefaultFrameRate = 30; +// Default sample diff, default frame rate. +const float kDefaultSampleDiffMs = 1000.0f / kDefaultFrameRate; +// A factor applied to the sample diff on OnTargetFramerateUpdated to determine +// a max limit for the sample diff. For instance, with a framerate of 30fps, +// the sample diff is capped to (1000 / 30) * 1.35 = 45ms. This prevents +// triggering too soon if there are individual very large outliers. +const float kMaxSampleDiffMarginFactor = 1.35f; +// Minimum framerate allowed for usage calculation. This prevents crazy long +// encode times from being accepted if the frame rate happens to be low. +const int kMinFramerate = 7; +const int kMaxFramerate = 30; const auto kScaleReasonCpu = AdaptationObserverInterface::AdaptReason::kCpu; } // namespace @@ -113,9 +125,9 @@ class OveruseFrameDetector::SendProcessingUsage { : kWeightFactorFrameDiff(0.998f), kWeightFactorProcessing(0.995f), kInitialSampleDiffMs(40.0f), - kMaxSampleDiffMs(45.0f), count_(0), options_(options), + max_sample_diff_ms_(kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor), filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)), filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) { Reset(); @@ -124,21 +136,24 @@ class OveruseFrameDetector::SendProcessingUsage { void Reset() { count_ = 0; + max_sample_diff_ms_ = kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor; filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff); filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs); filtered_processing_ms_->Reset(kWeightFactorProcessing); filtered_processing_ms_->Apply(1.0f, InitialProcessingMs()); } + void SetMaxSampleDiffMs(float diff_ms) { max_sample_diff_ms_ = diff_ms; } + void AddCaptureSample(float sample_ms) { - float exp = sample_ms / kSampleDiffMs; + float exp = sample_ms / kDefaultSampleDiffMs; exp = std::min(exp, kMaxExp); filtered_frame_diff_ms_->Apply(exp, sample_ms); } void AddSample(float processing_ms, int64_t diff_last_sample_ms) { ++count_; - float exp = diff_last_sample_ms / kSampleDiffMs; + float exp = diff_last_sample_ms / kDefaultSampleDiffMs; exp = std::min(exp, kMaxExp); filtered_processing_ms_->Apply(exp, processing_ms); } @@ -148,7 +163,7 @@ class OveruseFrameDetector::SendProcessingUsage { return static_cast(InitialUsageInPercent() + 0.5f); } float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f); - frame_diff_ms = std::min(frame_diff_ms, kMaxSampleDiffMs); + frame_diff_ms = std::min(frame_diff_ms, max_sample_diff_ms_); float encode_usage_percent = 100.0f * filtered_processing_ms_->filtered() / frame_diff_ms; return static_cast(encode_usage_percent + 0.5); @@ -168,9 +183,9 @@ class OveruseFrameDetector::SendProcessingUsage { const float kWeightFactorFrameDiff; const float kWeightFactorProcessing; const float kInitialSampleDiffMs; - const float kMaxSampleDiffMs; uint64_t count_; const CpuOveruseOptions options_; + float max_sample_diff_ms_; std::unique_ptr filtered_processing_ms_; std::unique_ptr filtered_frame_diff_ms_; }; @@ -331,6 +346,7 @@ OveruseFrameDetector::OveruseFrameDetector( last_capture_time_us_(-1), last_processed_capture_time_us_(-1), num_pixels_(0), + max_framerate_(kDefaultFrameRate), last_overuse_time_ms_(-1), checks_above_threshold_(0), num_overuse_detections_(0), @@ -382,6 +398,8 @@ bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now_us) const { } void OveruseFrameDetector::ResetAll(int num_pixels) { + // Reset state, as a result resolution being changed. Do not however change + // the current frame rate back to the default. RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); num_pixels_ = num_pixels; usage_->Reset(); @@ -390,6 +408,15 @@ void OveruseFrameDetector::ResetAll(int num_pixels) { last_processed_capture_time_us_ = -1; num_process_times_ = 0; metrics_ = rtc::Optional(); + OnTargetFramerateUpdated(max_framerate_); +} + +void OveruseFrameDetector::OnTargetFramerateUpdated(int framerate_fps) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + RTC_DCHECK_GE(framerate_fps, 0); + max_framerate_ = std::min(kMaxFramerate, framerate_fps); + usage_->SetMaxSampleDiffMs((1000 / std::max(kMinFramerate, max_framerate_)) * + kMaxSampleDiffMarginFactor); } void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame, diff --git a/webrtc/video/overuse_frame_detector.h b/webrtc/video/overuse_frame_detector.h index 2fb3104d53..29efa1609d 100644 --- a/webrtc/video/overuse_frame_detector.h +++ b/webrtc/video/overuse_frame_detector.h @@ -68,7 +68,7 @@ class OveruseFrameDetector { AdaptationObserverInterface* overuse_observer, EncodedFrameObserver* encoder_timing_, CpuOveruseMetricsObserver* metrics_observer); - ~OveruseFrameDetector(); + virtual ~OveruseFrameDetector(); // Start to periodically check for overuse. void StartCheckForOveruse(); @@ -77,6 +77,13 @@ class OveruseFrameDetector { // StartCheckForOveruse has been called. void StopCheckForOveruse(); + // Defines the current maximum framerate targeted by the capturer. This is + // used to make sure the encode usage percent doesn't drop unduly if the + // capturer has quiet periods (for instance caused by screen capturers with + // variable capture rate depending on content updates), otherwise we might + // experience adaptation toggling. + virtual void OnTargetFramerateUpdated(int framerate_fps); + // Called for each captured frame. void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); @@ -135,6 +142,7 @@ class OveruseFrameDetector { // Number of pixels of last captured frame. int num_pixels_ GUARDED_BY(task_checker_); + int max_framerate_ GUARDED_BY(task_checker_); int64_t last_overuse_time_ms_ GUARDED_BY(task_checker_); int checks_above_threshold_ GUARDED_BY(task_checker_); int num_overuse_detections_ GUARDED_BY(task_checker_); diff --git a/webrtc/video/overuse_frame_detector_unittest.cc b/webrtc/video/overuse_frame_detector_unittest.cc index bf1c8b4aca..ae3b687acc 100644 --- a/webrtc/video/overuse_frame_detector_unittest.cc +++ b/webrtc/video/overuse_frame_detector_unittest.cc @@ -139,6 +139,20 @@ class OveruseFrameDetectorTest : public ::testing::Test, int UsagePercent() { return metrics_.encode_usage_percent; } + int64_t OveruseProcessingTimeLimitForFramerate(int fps) const { + int64_t frame_interval = rtc::kNumMicrosecsPerSec / fps; + int64_t max_processing_time_us = + (frame_interval * options_.high_encode_usage_threshold_percent) / 100; + return max_processing_time_us; + } + + int64_t UnderuseProcessingTimeLimitForFramerate(int fps) const { + int64_t frame_interval = rtc::kNumMicrosecsPerSec / fps; + int64_t max_processing_time_us = + (frame_interval * options_.low_encode_usage_threshold_percent) / 100; + return max_processing_time_us; + } + CpuOveruseOptions options_; rtc::ScopedFakeClock clock_; std::unique_ptr observer_; @@ -355,4 +369,115 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { EXPECT_TRUE(event.Wait(10000)); } +TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) { + const int kCapturerMaxFrameRate = 30; + const int kEncodeMaxFrameRate = 20; // Maximum fps the encoder can sustain. + + // Trigger overuse. + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kCapturerMaxFrameRate; + // Processing time just below over use limit given kEncodeMaxFrameRate. + int64_t processing_time_us = + (98 * OveruseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Simulate frame rate reduction and normal usage. + frame_interval_us = rtc::kNumMicrosecsPerSec / kEncodeMaxFrameRate; + overuse_detector_->OnTargetFramerateUpdated(kEncodeMaxFrameRate); + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Reduce processing time to trigger underuse. + processing_time_us = + (98 * UnderuseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(1); + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); +} + +TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) { + const int kMinFrameRate = 7; // Minimum fps allowed by current detector impl. + overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate); + + // Normal usage just at the limit. + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMinFrameRate; + // Processing time just below over use limit given kEncodeMaxFrameRate. + int64_t processing_time_us = + (98 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Over the limit to overuse. + processing_time_us = + (102 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Reduce input frame rate. Should still trigger overuse. + overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate - 1); + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(); + } +} + +TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) { + const int kMaxFrameRate = 20; + overuse_detector_->OnTargetFramerateUpdated(kMaxFrameRate); + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMaxFrameRate; + // Maximum frame interval allowed is 35% above ideal. + int64_t max_frame_interval_us = (135 * frame_interval_us) / 100; + // Maximum processing time, without triggering overuse, allowed with the above + // frame interval. + int64_t max_processing_time_us = + (max_frame_interval_us * options_.high_encode_usage_threshold_percent) / + 100; + + // Processing time just below overuse limit given kMaxFrameRate. + int64_t processing_time_us = (98 * max_processing_time_us) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Go above limit, trigger overuse. + processing_time_us = (102 * max_processing_time_us) / 100; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(); + } + + // Increase frame interval, should still trigger overuse. + max_frame_interval_us *= 2; + EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(); + } +} + } // namespace webrtc diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc index 7bf32c6dba..285b7ef033 100644 --- a/webrtc/video/video_send_stream.cc +++ b/webrtc/video/video_send_stream.cc @@ -632,9 +632,10 @@ VideoSendStream::VideoSendStream( encoder_config.content_type), config_(std::move(config)), content_type_(encoder_config.content_type) { - vie_encoder_.reset(new ViEEncoder( - num_cpu_cores, &stats_proxy_, config_.encoder_settings, - config_.pre_encode_callback, config_.post_encode_callback)); + vie_encoder_.reset( + new ViEEncoder(num_cpu_cores, &stats_proxy_, config_.encoder_settings, + config_.pre_encode_callback, config_.post_encode_callback, + std::unique_ptr())); worker_queue_->PostTask(std::unique_ptr(new ConstructionTask( &send_stream_, &thread_sync_event_, &stats_proxy_, vie_encoder_.get(), module_process_thread, call_stats, transport, bitrate_allocator, diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 63b1c80fef..d82e4fc7a2 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -44,24 +44,12 @@ const int64_t kFrameLogIntervalMs = 60000; // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206 const int kMinPixelsPerFrame = 320 * 180; const int kMinFramerateFps = 2; +const int kMaxFramerateFps = 120; // The maximum number of frames to drop at beginning of stream // to try and achieve desired bitrate. const int kMaxInitialFramedrop = 4; -// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle -// pipelining encoders better (multiple input frames before something comes -// out). This should effectively turn off CPU adaptations for systems that -// remotely cope with the load right now. -CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { - CpuOveruseOptions options; - if (full_overuse_time) { - options.low_encode_usage_threshold_percent = 150; - options.high_encode_usage_threshold_percent = 200; - } - return options; -} - uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) { if (kbps > 0) { if (kbps < 300 /* qvga */) { @@ -207,7 +195,7 @@ class ViEEncoder::VideoSourceProxy { degradation_preference_ = degradation_preference; old_source = source_; source_ = source; - wants = GetActiveSinkWants(); + wants = GetActiveSinkWantsInternal(); } if (old_source != source && old_source != nullptr) { @@ -228,26 +216,9 @@ class ViEEncoder::VideoSourceProxy { source_->AddOrUpdateSink(vie_encoder_, sink_wants_); } - rtc::VideoSinkWants GetActiveSinkWants() EXCLUSIVE_LOCKS_REQUIRED(&crit_) { - rtc::VideoSinkWants wants = sink_wants_; - // Clear any constraints from the current sink wants that don't apply to - // the used degradation_preference. - switch (degradation_preference_) { - case VideoSendStream::DegradationPreference::kBalanced: - break; - case VideoSendStream::DegradationPreference::kMaintainFramerate: - wants.max_framerate_fps = std::numeric_limits::max(); - break; - case VideoSendStream::DegradationPreference::kMaintainResolution: - wants.max_pixel_count = std::numeric_limits::max(); - wants.target_pixel_count.reset(); - break; - case VideoSendStream::DegradationPreference::kDegradationDisabled: - wants.max_pixel_count = std::numeric_limits::max(); - wants.target_pixel_count.reset(); - wants.max_framerate_fps = std::numeric_limits::max(); - } - return wants; + rtc::VideoSinkWants GetActiveSinkWants() { + rtc::CritScope lock(&crit_); + return GetActiveSinkWantsInternal(); } void ResetPixelFpsCount() { @@ -277,14 +248,15 @@ class ViEEncoder::VideoSourceProxy { LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted; sink_wants_.max_pixel_count = pixels_wanted; sink_wants_.target_pixel_count = rtc::Optional(); - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); return true; } - bool RequestFramerateLowerThan(int fps) { + int RequestFramerateLowerThan(int fps) { // Called on the encoder task queue. // The input video frame rate will be scaled down to 2/3, rounding down. - return RestrictFramerate((fps * 2) / 3); + int framerate_wanted = (fps * 2) / 3; + return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; } bool RequestHigherResolutionThan(int pixel_count) { @@ -317,18 +289,22 @@ class ViEEncoder::VideoSourceProxy { rtc::Optional((pixel_count * 5) / 3); } LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); return true; } - bool RequestHigherFramerateThan(int fps) { + // Request upgrade in framerate. Returns the new requested frame, or -1 if + // no change requested. Note that maxint may be returned if limits due to + // adaptation requests are removed completely. In that case, consider + // |max_framerate_| to be the current limit (assuming the capturer complies). + int RequestHigherFramerateThan(int fps) { // Called on the encoder task queue. // The input frame rate will be scaled up to the last step, with rounding. int framerate_wanted = fps; if (fps != std::numeric_limits::max()) framerate_wanted = (fps * 3) / 2; - return IncreaseFramerate(framerate_wanted); + return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; } bool RestrictFramerate(int fps) { @@ -343,7 +319,7 @@ class ViEEncoder::VideoSourceProxy { LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; sink_wants_.max_framerate_fps = fps_wanted; - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); return true; } @@ -359,11 +335,34 @@ class ViEEncoder::VideoSourceProxy { LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; sink_wants_.max_framerate_fps = fps_wanted; - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); return true; } private: + rtc::VideoSinkWants GetActiveSinkWantsInternal() + EXCLUSIVE_LOCKS_REQUIRED(&crit_) { + rtc::VideoSinkWants wants = sink_wants_; + // Clear any constraints from the current sink wants that don't apply to + // the used degradation_preference. + switch (degradation_preference_) { + case VideoSendStream::DegradationPreference::kBalanced: + break; + case VideoSendStream::DegradationPreference::kMaintainFramerate: + wants.max_framerate_fps = std::numeric_limits::max(); + break; + case VideoSendStream::DegradationPreference::kMaintainResolution: + wants.max_pixel_count = std::numeric_limits::max(); + wants.target_pixel_count.reset(); + break; + case VideoSendStream::DegradationPreference::kDegradationDisabled: + wants.max_pixel_count = std::numeric_limits::max(); + wants.target_pixel_count.reset(); + wants.max_framerate_fps = std::numeric_limits::max(); + } + return wants; + } + rtc::CriticalSection crit_; rtc::SequencedTaskChecker main_checker_; ViEEncoder* const vie_encoder_; @@ -379,7 +378,8 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, SendStatisticsProxy* stats_proxy, const VideoSendStream::Config::EncoderSettings& settings, rtc::VideoSinkInterface* pre_encode_callback, - EncodedFrameObserver* encoder_timing) + EncodedFrameObserver* encoder_timing, + std::unique_ptr overuse_detector) : shutdown_event_(true /* manual_reset */, false), number_of_cores_(number_of_cores), initial_rampup_(0), @@ -389,13 +389,18 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, codec_type_(PayloadNameToCodecType(settings.payload_name) .value_or(VideoCodecType::kVideoCodecUnknown)), video_sender_(Clock::GetRealTimeClock(), this, this), - overuse_detector_(GetCpuOveruseOptions(settings.full_overuse_time), - this, - encoder_timing, - stats_proxy), + overuse_detector_( + overuse_detector.get() + ? overuse_detector.release() + : new OveruseFrameDetector( + GetCpuOveruseOptions(settings.full_overuse_time), + this, + encoder_timing, + stats_proxy)), stats_proxy_(stats_proxy), pre_encode_callback_(pre_encode_callback), module_process_thread_(nullptr), + max_framerate_(-1), pending_encoder_reconfiguration_(false), encoder_start_bitrate_bps_(0), max_data_payload_length_(0), @@ -416,7 +421,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, RTC_DCHECK(stats_proxy); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); - overuse_detector_.StartCheckForOveruse(); + overuse_detector_->StartCheckForOveruse(); video_sender_.RegisterExternalEncoder( settings_.encoder, settings_.payload_type, settings_.internal_source); }); @@ -428,12 +433,25 @@ ViEEncoder::~ViEEncoder() { << "Must call ::Stop() before destruction."; } +// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle +// pipelining encoders better (multiple input frames before something comes +// out). This should effectively turn off CPU adaptations for systems that +// remotely cope with the load right now. +CpuOveruseOptions ViEEncoder::GetCpuOveruseOptions(bool full_overuse_time) { + CpuOveruseOptions options; + if (full_overuse_time) { + options.low_encode_usage_threshold_percent = 150; + options.high_encode_usage_threshold_percent = 200; + } + return options; +} + void ViEEncoder::Stop() { RTC_DCHECK_RUN_ON(&thread_checker_); source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); - overuse_detector_.StopCheckForOveruse(); + overuse_detector_->StopCheckForOveruse(); rate_allocator_.reset(); bitrate_observer_ = nullptr; video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, @@ -493,6 +511,12 @@ void ViEEncoder::SetSource( bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_); initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop; ConfigureQualityScaler(); + if (!IsFramerateScalingEnabled(degradation_preference) && + max_framerate_ != -1) { + // If frame rate scaling is no longer allowed, remove any potential + // allowance for longer frame intervals. + overuse_detector_->OnTargetFramerateUpdated(max_framerate_); + } }); } @@ -562,6 +586,8 @@ void ViEEncoder::ReconfigureEncoder() { std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate); codec.expect_encode_from_texture = last_frame_info_->is_texture; + max_framerate_ = codec.maxFramerate; + RTC_DCHECK_LE(max_framerate_, kMaxFramerateFps); bool success = video_sender_.RegisterSendCodec( &codec, number_of_cores_, @@ -574,19 +600,31 @@ void ViEEncoder::ReconfigureEncoder() { video_sender_.UpdateChannelParemeters(rate_allocator_.get(), bitrate_observer_); - int framerate = stats_proxy_->GetSendFrameRate(); - if (framerate == 0) - framerate = codec.maxFramerate; + // Get the current actual framerate, as measured by the stats proxy. This is + // used to get the correct bitrate layer allocation. + int current_framerate = stats_proxy_->GetSendFrameRate(); + if (current_framerate == 0) + current_framerate = codec.maxFramerate; stats_proxy_->OnEncoderReconfigured( - encoder_config_, rate_allocator_.get() - ? rate_allocator_->GetPreferredBitrateBps(framerate) - : codec.maxBitrate); + encoder_config_, + rate_allocator_.get() + ? rate_allocator_->GetPreferredBitrateBps(current_framerate) + : codec.maxBitrate); pending_encoder_reconfiguration_ = false; sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); + // Get the current target framerate, ie the maximum framerate as specified by + // the current codec configuration, or any limit imposed by cpu adaption in + // maintain-resolution or balanced mode. This is used to make sure overuse + // detection doesn't needlessly trigger in low and/or variable framerate + // scenarios. + int target_framerate = std::min( + max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); + overuse_detector_->OnTargetFramerateUpdated(target_framerate); + ConfigureQualityScaler(); } @@ -735,7 +773,7 @@ void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); - overuse_detector_.FrameCaptured(video_frame, time_when_posted_us); + overuse_detector_->FrameCaptured(video_frame, time_when_posted_us); video_sender_.AddVideoFrame(video_frame, nullptr); } @@ -766,7 +804,7 @@ EncodedImageCallback::Result ViEEncoder::OnEncodedImage( const int qp = encoded_image.qp_; encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] { RTC_DCHECK_RUN_ON(&encoder_queue_); - overuse_detector_.FrameSent(timestamp, time_sent_us); + overuse_detector_->FrameSent(timestamp, time_sent_us); if (quality_scaler_ && qp >= 0) quality_scaler_->ReportQP(qp); }); @@ -901,14 +939,18 @@ void ViEEncoder::AdaptDown(AdaptReason reason) { } GetAdaptCounter().IncrementResolution(reason); break; - case VideoSendStream::DegradationPreference::kMaintainResolution: + case VideoSendStream::DegradationPreference::kMaintainResolution: { // Scale down framerate. - if (!source_proxy_->RequestFramerateLowerThan( - adaptation_request.framerate_fps_)) { + const int requested_framerate = source_proxy_->RequestFramerateLowerThan( + adaptation_request.framerate_fps_); + if (requested_framerate == -1) return; - } + RTC_DCHECK_NE(max_framerate_, -1); + overuse_detector_->OnTargetFramerateUpdated( + std::min(max_framerate_, requested_framerate)); GetAdaptCounter().IncrementFramerate(reason); break; + } case VideoSendStream::DegradationPreference::kDegradationDisabled: RTC_NOTREACHED(); } @@ -985,8 +1027,15 @@ void ViEEncoder::AdaptUp(AdaptReason reason) { LOG(LS_INFO) << "Removing framerate down-scaling setting."; fps = std::numeric_limits::max(); } - if (!source_proxy_->RequestHigherFramerateThan(fps)) + + const int requested_framerate = + source_proxy_->RequestHigherFramerateThan(fps); + if (requested_framerate == -1) { + overuse_detector_->OnTargetFramerateUpdated(max_framerate_); return; + } + overuse_detector_->OnTargetFramerateUpdated( + std::min(max_framerate_, requested_framerate)); GetAdaptCounter().DecrementFramerate(reason); break; } diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 460abf128e..77a6bc417d 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -77,7 +77,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, SendStatisticsProxy* stats_proxy, const VideoSendStream::Config::EncoderSettings& settings, rtc::VideoSinkInterface* pre_encode_callback, - EncodedFrameObserver* encoder_timing); + EncodedFrameObserver* encoder_timing, + std::unique_ptr overuse_detector); ~ViEEncoder(); // RegisterProcessThread register |module_process_thread| with those objects // that use it. Registration has to happen on the thread where @@ -129,6 +130,7 @@ class ViEEncoder : public rtc::VideoSinkInterface, // These methods are protected for easier testing. void AdaptUp(AdaptReason reason) override; void AdaptDown(AdaptReason reason) override; + static CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time); private: class ConfigureEncoderTask; @@ -232,7 +234,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, const VideoCodecType codec_type_; vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); - OveruseFrameDetector overuse_detector_ ACCESS_ON(&encoder_queue_); + std::unique_ptr overuse_detector_ + ACCESS_ON(&encoder_queue_); std::unique_ptr quality_scaler_ ACCESS_ON(&encoder_queue_); SendStatisticsProxy* const stats_proxy_; @@ -246,6 +249,9 @@ class ViEEncoder : public rtc::VideoSinkInterface, VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_); std::unique_ptr rate_allocator_ ACCESS_ON(&encoder_queue_); + // The maximum frame rate of the current codec configuration, as determined + // at the last ReconfigureEncoder() call. + int max_framerate_ ACCESS_ON(&encoder_queue_); // Set when ConfigureEncoder has been called in order to lazy reconfigure the // encoder on the next frame. diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc index a016fa9313..b143e4d546 100644 --- a/webrtc/video/vie_encoder_unittest.cc +++ b/webrtc/video/vie_encoder_unittest.cc @@ -48,6 +48,7 @@ const size_t kMaxPayloadLength = 1440; const int kTargetBitrateBps = 1000000; const int kLowTargetBitrateBps = kTargetBitrateBps / 10; const int kMaxInitialFramedrop = 4; +const int kDefaultFramerate = 30; class TestBuffer : public webrtc::I420Buffer { public: @@ -63,6 +64,35 @@ class TestBuffer : public webrtc::I420Buffer { rtc::Event* const event_; }; +class CpuOveruseDetectorProxy : public OveruseFrameDetector { + public: + CpuOveruseDetectorProxy(const CpuOveruseOptions& options, + AdaptationObserverInterface* overuse_observer, + EncodedFrameObserver* encoder_timing_, + CpuOveruseMetricsObserver* metrics_observer) + : OveruseFrameDetector(options, + overuse_observer, + encoder_timing_, + metrics_observer), + last_target_framerate_fps_(-1) {} + virtual ~CpuOveruseDetectorProxy() {} + + void OnTargetFramerateUpdated(int framerate_fps) override { + rtc::CritScope cs(&lock_); + last_target_framerate_fps_ = framerate_fps; + OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps); + } + + int GetLastTargetFramerate() { + rtc::CritScope cs(&lock_); + return last_target_framerate_fps_; + } + + private: + rtc::CriticalSection lock_; + int last_target_framerate_fps_ GUARDED_BY(lock_); +}; + class ViEEncoderUnderTest : public ViEEncoder { public: ViEEncoderUnderTest(SendStatisticsProxy* stats_proxy, @@ -71,7 +101,13 @@ class ViEEncoderUnderTest : public ViEEncoder { stats_proxy, settings, nullptr /* pre_encode_callback */, - nullptr /* encoder_timing */) {} + nullptr /* encoder_timing */, + std::unique_ptr( + overuse_detector_proxy_ = new CpuOveruseDetectorProxy( + GetCpuOveruseOptions(settings.full_overuse_time), + this, + nullptr, + stats_proxy))) {} void PostTaskAndWait(bool down, AdaptReason reason) { rtc::Event event(false, false); @@ -99,14 +135,17 @@ class ViEEncoderUnderTest : public ViEEncoder { void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); } void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); } + + CpuOveruseDetectorProxy* overuse_detector_proxy_; }; class VideoStreamFactory : public VideoEncoderConfig::VideoStreamFactoryInterface { public: - explicit VideoStreamFactory(size_t num_temporal_layers) - : num_temporal_layers_(num_temporal_layers) { + explicit VideoStreamFactory(size_t num_temporal_layers, int framerate) + : num_temporal_layers_(num_temporal_layers), framerate_(framerate) { EXPECT_GT(num_temporal_layers, 0u); + EXPECT_GT(framerate, 0); } private: @@ -118,10 +157,13 @@ class VideoStreamFactory test::CreateVideoStreams(width, height, encoder_config); for (VideoStream& stream : streams) { stream.temporal_layer_thresholds_bps.resize(num_temporal_layers_ - 1); + stream.max_framerate = framerate_; } return streams; } + const size_t num_temporal_layers_; + const int framerate_; }; class AdaptingFrameForwarder : public test::FrameForwarder { @@ -264,7 +306,8 @@ class ViEEncoderTest : public ::testing::Test { video_encoder_config.number_of_streams = num_streams; video_encoder_config.max_bitrate_bps = kTargetBitrateBps; video_encoder_config.video_stream_factory = - new rtc::RefCountedObject(num_temporal_layers); + new rtc::RefCountedObject(num_temporal_layers, + kDefaultFramerate); ConfigureEncoder(std::move(video_encoder_config), nack_enabled); } @@ -2027,6 +2070,164 @@ TEST_F(ViEEncoderTest, CallsBitrateObserver) { vie_encoder_->Stop(); } +TEST_F(ViEEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) { + const int kFrameWidth = 1280; + const int kFrameHeight = 720; + const int kFramerate = 24; + + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + test::FrameForwarder source; + vie_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainResolution); + + // Insert a single frame, triggering initial configuration. + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kDefaultFramerate); + + // Trigger reconfigure encoder (without resetting the entire instance). + VideoEncoderConfig video_encoder_config; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.number_of_streams = 1; + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kFramerate); + vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + // Detector should be updated with fps limit from codec config. + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + // Trigger overuse, max framerate should be reduced. + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + stats.input_frame_rate = kFramerate; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuOveruse(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + int adapted_framerate = + vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + EXPECT_LT(adapted_framerate, kFramerate); + + // Trigger underuse, max framerate should go back to codec configured fps. + // Set extra low fps, to make sure it's actually reset, not just incremented. + stats = stats_proxy_->GetStats(); + stats.input_frame_rate = adapted_framerate / 2; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuNormalUsage(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) { + const int kFrameWidth = 1280; + const int kFrameHeight = 720; + const int kLowFramerate = 15; + const int kHighFramerate = 25; + + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + test::FrameForwarder source; + vie_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainResolution); + + // Trigger initial configuration. + VideoEncoderConfig video_encoder_config; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.number_of_streams = 1; + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kLowFramerate); + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kLowFramerate); + + // Trigger overuse, max framerate should be reduced. + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + stats.input_frame_rate = kLowFramerate; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuOveruse(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + int adapted_framerate = + vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + EXPECT_LT(adapted_framerate, kLowFramerate); + + // Reconfigure the encoder with a new (higher max framerate), max fps should + // still respect the adaptation. + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kHighFramerate); + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + adapted_framerate); + + // Trigger underuse, max framerate should go back to codec configured fps. + stats = stats_proxy_->GetStats(); + stats.input_frame_rate = adapted_framerate; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuNormalUsage(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kHighFramerate); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, OveruseDetectorUpdatedOnDegradationPreferenceChange) { + const int kFrameWidth = 1280; + const int kFrameHeight = 720; + const int kFramerate = 24; + + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + test::FrameForwarder source; + vie_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainResolution); + + // Trigger initial configuration. + VideoEncoderConfig video_encoder_config; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.number_of_streams = 1; + video_encoder_config.video_stream_factory = + new rtc::RefCountedObject(1, kFramerate); + source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); + vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), + kMaxPayloadLength, false); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + // Trigger overuse, max framerate should be reduced. + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + stats.input_frame_rate = kFramerate; + stats_proxy_->SetMockStats(stats); + vie_encoder_->TriggerCpuOveruse(); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + int adapted_framerate = + vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(); + EXPECT_LT(adapted_framerate, kFramerate); + + // Change degradation preference to not enable framerate scaling. Target + // framerate should be changed to codec defined limit. + vie_encoder_->SetSource( + &source, VideoSendStream::DegradationPreference::kMaintainFramerate); + vie_encoder_->WaitUntilTaskQueueIsIdle(); + EXPECT_EQ(vie_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(), + kFramerate); + + vie_encoder_->Stop(); +} + TEST_F(ViEEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) { const int kTooLowBitrateForFrameSizeBps = 10000; vie_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);