diff --git a/webrtc/video/overuse_frame_detector.cc b/webrtc/video/overuse_frame_detector.cc index cc4c000a73..bb89864bb0 100644 --- a/webrtc/video/overuse_frame_detector.cc +++ b/webrtc/video/overuse_frame_detector.cc @@ -31,7 +31,8 @@ namespace webrtc { namespace { -const int64_t kProcessIntervalMs = 5000; +const int64_t kCheckForOveruseIntervalMs = 5000; +const int64_t kTimeToFirstCheckForOveruseMs = 100; // Delay between consecutive rampups. (Used for quick recovery.) const int kQuickRampUpDelayMs = 10 * 1000; @@ -170,13 +171,44 @@ class OveruseFrameDetector::SendProcessingUsage { std::unique_ptr filtered_frame_diff_ms_; }; +class OveruseFrameDetector::CheckOveruseTask : public rtc::QueuedTask { + public: + explicit CheckOveruseTask(OveruseFrameDetector* overuse_detector) + : overuse_detector_(overuse_detector) { + rtc::TaskQueue::Current()->PostDelayedTask( + std::unique_ptr(this), kTimeToFirstCheckForOveruseMs); + } + + void Stop() { + RTC_CHECK(task_checker_.CalledSequentially()); + overuse_detector_ = nullptr; + } + + private: + bool Run() override { + RTC_CHECK(task_checker_.CalledSequentially()); + if (!overuse_detector_) + return true; // This will make the task queue delete this task. + overuse_detector_->CheckForOveruse(); + + rtc::TaskQueue::Current()->PostDelayedTask( + std::unique_ptr(this), kCheckForOveruseIntervalMs); + // Return false to prevent this task from being deleted. Ownership has been + // transferred to the task queue when PostDelayedTask was called. + return false; + } + rtc::SequencedTaskChecker task_checker_; + OveruseFrameDetector* overuse_detector_; +}; + OveruseFrameDetector::OveruseFrameDetector( Clock* clock, const CpuOveruseOptions& options, CpuOveruseObserver* observer, EncodedFrameObserver* encoder_timing, CpuOveruseMetricsObserver* metrics_observer) - : options_(options), + : check_overuse_task_(nullptr), + options_(options), observer_(observer), encoder_timing_(encoder_timing), metrics_observer_(metrics_observer), @@ -185,7 +217,6 @@ OveruseFrameDetector::OveruseFrameDetector( last_capture_time_ms_(-1), last_processed_capture_time_ms_(-1), num_pixels_(0), - next_process_time_ms_(clock_->TimeInMilliseconds()), last_overuse_time_ms_(-1), checks_above_threshold_(0), num_overuse_detections_(0), @@ -193,13 +224,26 @@ OveruseFrameDetector::OveruseFrameDetector( in_quick_rampup_(false), current_rampup_delay_ms_(kStandardRampUpDelayMs), usage_(new SendProcessingUsage(options)) { - processing_thread_.DetachFromThread(); + task_checker_.Detach(); } OveruseFrameDetector::~OveruseFrameDetector() { + RTC_DCHECK(!check_overuse_task_) << "StopCheckForOverUse must be called."; +} + +void OveruseFrameDetector::StartCheckForOveruse() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + RTC_DCHECK(!check_overuse_task_); + check_overuse_task_ = new CheckOveruseTask(this); +} +void OveruseFrameDetector::StopCheckForOveruse() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + check_overuse_task_->Stop(); + check_overuse_task_ = nullptr; } void OveruseFrameDetector::EncodedFrameTimeMeasured(int encode_duration_ms) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); if (!metrics_) metrics_ = rtc::Optional(CpuOveruseMetrics()); metrics_->encode_usage_percent = usage_->Value(); @@ -207,12 +251,8 @@ void OveruseFrameDetector::EncodedFrameTimeMeasured(int encode_duration_ms) { metrics_observer_->OnEncodedFrameTimeMeasured(encode_duration_ms, *metrics_); } -int64_t OveruseFrameDetector::TimeUntilNextProcess() { - RTC_DCHECK(processing_thread_.CalledOnValidThread()); - return next_process_time_ms_ - clock_->TimeInMilliseconds(); -} - bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); if (num_pixels != num_pixels_) { return true; } @@ -220,12 +260,14 @@ bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const { } bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now) const { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); if (last_capture_time_ms_ == -1) return false; return (now - last_capture_time_ms_) > options_.frame_timeout_interval_ms; } void OveruseFrameDetector::ResetAll(int num_pixels) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); num_pixels_ = num_pixels; usage_->Reset(); frame_timing_.clear(); @@ -235,36 +277,36 @@ void OveruseFrameDetector::ResetAll(int num_pixels) { metrics_ = rtc::Optional(); } -void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame) { - rtc::CritScope cs(&crit_); +void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame, + int64_t time_when_first_seen_ms) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); - int64_t now = clock_->TimeInMilliseconds(); if (FrameSizeChanged(frame.width() * frame.height()) || - FrameTimeoutDetected(now)) { + FrameTimeoutDetected(time_when_first_seen_ms)) { ResetAll(frame.width() * frame.height()); } if (last_capture_time_ms_ != -1) - usage_->AddCaptureSample(now - last_capture_time_ms_); + usage_->AddCaptureSample(time_when_first_seen_ms - last_capture_time_ms_); - last_capture_time_ms_ = now; + last_capture_time_ms_ = time_when_first_seen_ms; - frame_timing_.push_back( - FrameTiming(frame.ntp_time_ms(), frame.timestamp(), now)); + frame_timing_.push_back(FrameTiming(frame.ntp_time_ms(), frame.timestamp(), + time_when_first_seen_ms)); } -void OveruseFrameDetector::FrameSent(uint32_t timestamp) { - rtc::CritScope cs(&crit_); +void OveruseFrameDetector::FrameSent(uint32_t timestamp, + int64_t time_sent_in_ms) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); // Delay before reporting actual encoding time, used to have the ability to // detect total encoding time when encoding more than one layer. Encoding is // here assumed to finish within a second (or that we get enough long-time // samples before one second to trigger an overuse even when this is not the // case). static const int64_t kEncodingTimeMeasureWindowMs = 1000; - int64_t now = clock_->TimeInMilliseconds(); for (auto& it : frame_timing_) { if (it.timestamp == timestamp) { - it.last_send_ms = now; + it.last_send_ms = time_sent_in_ms; break; } } @@ -276,7 +318,7 @@ void OveruseFrameDetector::FrameSent(uint32_t timestamp) { // https://crbug.com/350106 while (!frame_timing_.empty()) { FrameTiming timing = frame_timing_.front(); - if (now - timing.capture_ms < kEncodingTimeMeasureWindowMs) + if (time_sent_in_ms - timing.capture_ms < kEncodingTimeMeasureWindowMs) break; if (timing.last_send_ms != -1) { int encode_duration_ms = @@ -296,28 +338,15 @@ void OveruseFrameDetector::FrameSent(uint32_t timestamp) { } } -void OveruseFrameDetector::Process() { - RTC_DCHECK(processing_thread_.CalledOnValidThread()); +void OveruseFrameDetector::CheckForOveruse() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + ++num_process_times_; + if (num_process_times_ <= options_.min_process_count || !metrics_) + return; int64_t now = clock_->TimeInMilliseconds(); - // Used to protect against Process() being called too often. - if (now < next_process_time_ms_) - return; - - next_process_time_ms_ = now + kProcessIntervalMs; - - CpuOveruseMetrics current_metrics; - { - rtc::CritScope cs(&crit_); - ++num_process_times_; - if (num_process_times_ <= options_.min_process_count || !metrics_) - return; - - current_metrics = *metrics_; - } - - if (IsOverusing(current_metrics)) { + if (IsOverusing(*metrics_)) { // If the last thing we did was going up, and now have to back down, we need // to check if this peak was short. If so we should back off to avoid going // back and forth between this load, the system doesn't seem to handle it. @@ -342,7 +371,7 @@ void OveruseFrameDetector::Process() { if (observer_) observer_->OveruseDetected(); - } else if (IsUnderusing(current_metrics, now)) { + } else if (IsUnderusing(*metrics_, now)) { last_rampup_time_ms_ = now; in_quick_rampup_ = true; @@ -354,12 +383,13 @@ void OveruseFrameDetector::Process() { in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_; LOG(LS_VERBOSE) << " Frame stats: " - << " encode usage " << current_metrics.encode_usage_percent + << " encode usage " << metrics_->encode_usage_percent << " overuse detections " << num_overuse_detections_ << " rampup delay " << rampup_delay; } bool OveruseFrameDetector::IsOverusing(const CpuOveruseMetrics& metrics) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); if (metrics.encode_usage_percent >= options_.high_encode_usage_threshold_percent) { ++checks_above_threshold_; @@ -371,6 +401,7 @@ bool OveruseFrameDetector::IsOverusing(const CpuOveruseMetrics& metrics) { bool OveruseFrameDetector::IsUnderusing(const CpuOveruseMetrics& metrics, int64_t time_now) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); int delay = in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_; if (time_now < last_rampup_time_ms_ + delay) return false; diff --git a/webrtc/video/overuse_frame_detector.h b/webrtc/video/overuse_frame_detector.h index 9f78c6c0ff..3cd1fd5cdf 100644 --- a/webrtc/video/overuse_frame_detector.h +++ b/webrtc/video/overuse_frame_detector.h @@ -15,12 +15,11 @@ #include #include "webrtc/base/constructormagic.h" -#include "webrtc/base/criticalsection.h" #include "webrtc/base/optional.h" #include "webrtc/base/exp_filter.h" +#include "webrtc/base/sequenced_task_checker.h" +#include "webrtc/base/task_queue.h" #include "webrtc/base/thread_annotations.h" -#include "webrtc/base/thread_checker.h" -#include "webrtc/modules/include/module.h" namespace webrtc { @@ -72,8 +71,11 @@ class CpuOveruseMetricsObserver { }; // Use to detect system overuse based on the send-side processing time of -// incoming frames. -class OveruseFrameDetector : public Module { +// incoming frames. All methods must be called on a single task queue but it can +// be created and destroyed on an arbitrary thread. +// OveruseFrameDetector::StartCheckForOveruse must be called to periodically +// check for overuse. +class OveruseFrameDetector { public: OveruseFrameDetector(Clock* clock, const CpuOveruseOptions& options, @@ -82,18 +84,25 @@ class OveruseFrameDetector : public Module { CpuOveruseMetricsObserver* metrics_observer); ~OveruseFrameDetector(); + // Start to periodically check for overuse. + void StartCheckForOveruse(); + + // StopCheckForOveruse must be called before destruction if + // StartCheckForOveruse has been called. + void StopCheckForOveruse(); + // Called for each captured frame. - void FrameCaptured(const VideoFrame& frame); + void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_ms); // Called for each sent frame. - void FrameSent(uint32_t timestamp); + void FrameSent(uint32_t timestamp, int64_t time_sent_in_ms); - // Implements Module. - int64_t TimeUntilNextProcess() override; - void Process() override; + protected: + void CheckForOveruse(); // Protected for test purposes. private: class SendProcessingUsage; + class CheckOveruseTask; struct FrameTiming { FrameTiming(int64_t capture_ntp_ms, uint32_t timestamp, int64_t now) : capture_ntp_ms(capture_ntp_ms), @@ -106,23 +115,18 @@ class OveruseFrameDetector : public Module { int64_t last_send_ms; }; - void EncodedFrameTimeMeasured(int encode_duration_ms) - EXCLUSIVE_LOCKS_REQUIRED(crit_); - - // Only called on the processing thread. + void EncodedFrameTimeMeasured(int encode_duration_ms); bool IsOverusing(const CpuOveruseMetrics& metrics); bool IsUnderusing(const CpuOveruseMetrics& metrics, int64_t time_now); - bool FrameTimeoutDetected(int64_t now) const EXCLUSIVE_LOCKS_REQUIRED(crit_); - bool FrameSizeChanged(int num_pixels) const EXCLUSIVE_LOCKS_REQUIRED(crit_); + bool FrameTimeoutDetected(int64_t now) const; + bool FrameSizeChanged(int num_pixels) const; - void ResetAll(int num_pixels) EXCLUSIVE_LOCKS_REQUIRED(crit_); + void ResetAll(int num_pixels); - // Protecting all members except const and those that are only accessed on the - // processing thread. - // TODO(asapersson): See if we can reduce locking. As is, video frame - // processing contends with reading stats and the processing thread. - rtc::CriticalSection crit_; + rtc::SequencedTaskChecker task_checker_; + // Owned by the task queue from where StartCheckForOveruse is called. + CheckOveruseTask* check_overuse_task_; const CpuOveruseOptions options_; @@ -132,32 +136,27 @@ class OveruseFrameDetector : public Module { // Stats metrics. CpuOveruseMetricsObserver* const metrics_observer_; - rtc::Optional metrics_ GUARDED_BY(crit_); - + rtc::Optional metrics_ GUARDED_BY(task_checker_); Clock* const clock_; - int64_t num_process_times_ GUARDED_BY(crit_); - int64_t last_capture_time_ms_ GUARDED_BY(crit_); - int64_t last_processed_capture_time_ms_ GUARDED_BY(crit_); + int64_t num_process_times_ GUARDED_BY(task_checker_); + + int64_t last_capture_time_ms_ GUARDED_BY(task_checker_); + int64_t last_processed_capture_time_ms_ GUARDED_BY(task_checker_); // Number of pixels of last captured frame. - int num_pixels_ GUARDED_BY(crit_); - - // These seven members are only accessed on the processing thread. - int64_t next_process_time_ms_; - int64_t last_overuse_time_ms_; - int checks_above_threshold_; - int num_overuse_detections_; - int64_t last_rampup_time_ms_; - bool in_quick_rampup_; - int current_rampup_delay_ms_; + int num_pixels_ GUARDED_BY(task_checker_); + int64_t last_overuse_time_ms_ GUARDED_BY(task_checker_); + int checks_above_threshold_ GUARDED_BY(task_checker_); + int num_overuse_detections_ GUARDED_BY(task_checker_); + int64_t last_rampup_time_ms_ GUARDED_BY(task_checker_); + bool in_quick_rampup_ GUARDED_BY(task_checker_); + int current_rampup_delay_ms_ GUARDED_BY(task_checker_); // TODO(asapersson): Can these be regular members (avoid separate heap // allocs)? - const std::unique_ptr usage_ GUARDED_BY(crit_); - std::list frame_timing_ GUARDED_BY(crit_); - - rtc::ThreadChecker processing_thread_; + const std::unique_ptr usage_ GUARDED_BY(task_checker_); + std::list frame_timing_ GUARDED_BY(task_checker_); RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector); }; diff --git a/webrtc/video/overuse_frame_detector_unittest.cc b/webrtc/video/overuse_frame_detector_unittest.cc index 67d05325e3..d6700e248d 100644 --- a/webrtc/video/overuse_frame_detector_unittest.cc +++ b/webrtc/video/overuse_frame_detector_unittest.cc @@ -15,10 +15,14 @@ #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/event.h" #include "webrtc/system_wrappers/include/clock.h" #include "webrtc/video_frame.h" namespace webrtc { + +using ::testing::Invoke; + namespace { const int kWidth = 640; const int kHeight = 480; @@ -50,6 +54,23 @@ class CpuOveruseObserverImpl : public CpuOveruseObserver { int normaluse_; }; +class OveruseFrameDetectorUnderTest : public OveruseFrameDetector { + public: + OveruseFrameDetectorUnderTest(Clock* clock, + const CpuOveruseOptions& options, + CpuOveruseObserver* overuse_observer, + EncodedFrameObserver* encoder_timing, + CpuOveruseMetricsObserver* metrics_observer) + : OveruseFrameDetector(clock, + options, + overuse_observer, + encoder_timing, + metrics_observer) {} + ~OveruseFrameDetectorUnderTest() {} + + using OveruseFrameDetector::CheckForOveruse; +}; + class OveruseFrameDetectorTest : public ::testing::Test, public CpuOveruseMetricsObserver { protected: @@ -61,7 +82,7 @@ class OveruseFrameDetectorTest : public ::testing::Test, } void ReinitializeOveruseDetector() { - overuse_detector_.reset(new OveruseFrameDetector( + overuse_detector_.reset(new OveruseFrameDetectorUnderTest( clock_.get(), options_, observer_.get(), nullptr, this)); } @@ -85,9 +106,9 @@ class OveruseFrameDetectorTest : public ::testing::Test, uint32_t timestamp = 0; while (num_frames-- > 0) { frame.set_timestamp(timestamp); - overuse_detector_->FrameCaptured(frame); + overuse_detector_->FrameCaptured(frame, clock_->TimeInMilliseconds()); clock_->AdvanceTimeMilliseconds(delay_ms); - overuse_detector_->FrameSent(timestamp); + overuse_detector_->FrameSent(timestamp, clock_->TimeInMilliseconds()); clock_->AdvanceTimeMilliseconds(interval_ms - delay_ms); timestamp += interval_ms * 90; } @@ -105,7 +126,7 @@ class OveruseFrameDetectorTest : public ::testing::Test, for (int i = 0; i < num_times; ++i) { InsertAndSendFramesWithInterval( 1000, kFrameInterval33ms, kWidth, kHeight, kDelayMs); - overuse_detector_->Process(); + overuse_detector_->CheckForOveruse(); } } @@ -116,7 +137,7 @@ class OveruseFrameDetectorTest : public ::testing::Test, 1300, kFrameInterval33ms, kWidth, kHeight, kDelayMs1); InsertAndSendFramesWithInterval( 1, kFrameInterval33ms, kWidth, kHeight, kDelayMs2); - overuse_detector_->Process(); + overuse_detector_->CheckForOveruse(); } int UsagePercent() { return metrics_.encode_usage_percent; } @@ -124,7 +145,7 @@ class OveruseFrameDetectorTest : public ::testing::Test, CpuOveruseOptions options_; std::unique_ptr clock_; std::unique_ptr observer_; - std::unique_ptr overuse_detector_; + std::unique_ptr overuse_detector_; CpuOveruseMetrics metrics_; }; @@ -147,8 +168,8 @@ TEST_F(OveruseFrameDetectorTest, OveruseAndRecover) { } TEST_F(OveruseFrameDetectorTest, OveruseAndRecoverWithNoObserver) { - overuse_detector_.reset( - new OveruseFrameDetector(clock_.get(), options_, nullptr, nullptr, this)); + overuse_detector_.reset(new OveruseFrameDetectorUnderTest( + clock_.get(), options_, nullptr, nullptr, this)); EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(0); TriggerOveruse(options_.high_threshold_consecutive_count); EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(0); @@ -166,14 +187,14 @@ TEST_F(OveruseFrameDetectorTest, DoubleOveruseAndRecover) { TEST_F(OveruseFrameDetectorTest, TriggerUnderuseWithMinProcessCount) { options_.min_process_count = 1; CpuOveruseObserverImpl overuse_observer; - overuse_detector_.reset(new OveruseFrameDetector( + overuse_detector_.reset(new OveruseFrameDetectorUnderTest( clock_.get(), options_, &overuse_observer, nullptr, this)); InsertAndSendFramesWithInterval( 1200, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms); - overuse_detector_->Process(); + overuse_detector_->CheckForOveruse(); EXPECT_EQ(0, overuse_observer.normaluse_); clock_->AdvanceTimeMilliseconds(kProcessIntervalMs); - overuse_detector_->Process(); + overuse_detector_->CheckForOveruse(); EXPECT_EQ(1, overuse_observer.normaluse_); } @@ -267,13 +288,14 @@ TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { for (size_t i = 0; i < 1000; ++i) { // Unique timestamps. frame.set_timestamp(static_cast(i)); - overuse_detector_->FrameCaptured(frame); + overuse_detector_->FrameCaptured(frame, clock_->TimeInMilliseconds()); clock_->AdvanceTimeMilliseconds(kIntervalMs); if (i > kNumFramesEncodingDelay) { overuse_detector_->FrameSent( - static_cast(i - kNumFramesEncodingDelay)); + static_cast(i - kNumFramesEncodingDelay), + clock_->TimeInMilliseconds()); } - overuse_detector_->Process(); + overuse_detector_->CheckForOveruse(); } } @@ -287,17 +309,47 @@ TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) { uint32_t timestamp = 0; for (size_t i = 0; i < 1000; ++i) { frame.set_timestamp(timestamp); - overuse_detector_->FrameCaptured(frame); + overuse_detector_->FrameCaptured(frame, clock_->TimeInMilliseconds()); // Encode and send first parts almost instantly. clock_->AdvanceTimeMilliseconds(1); - overuse_detector_->FrameSent(timestamp); + overuse_detector_->FrameSent(timestamp, clock_->TimeInMilliseconds()); // Encode heavier part, resulting in >85% usage total. clock_->AdvanceTimeMilliseconds(kDelayMs - 1); - overuse_detector_->FrameSent(timestamp); + overuse_detector_->FrameSent(timestamp, clock_->TimeInMilliseconds()); clock_->AdvanceTimeMilliseconds(kIntervalMs - kDelayMs); timestamp += kIntervalMs * 90; - overuse_detector_->Process(); + overuse_detector_->CheckForOveruse(); } } +TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { + rtc::TaskQueue queue("OveruseFrameDetectorTestQueue"); + + rtc::Event event(false, false); + queue.PostTask([this, &event] { + overuse_detector_->StartCheckForOveruse(); + event.Set(); + }); + event.Wait(rtc::Event::kForever); + + // Expect NormalUsage(). When called, stop the |overuse_detector_| and then + // set |event| to end the test. + EXPECT_CALL(*(observer_.get()), NormalUsage()) + .WillOnce(Invoke([this, &event] { + overuse_detector_->StopCheckForOveruse(); + event.Set(); + })); + + queue.PostTask([this, &event] { + const int kDelayMs1 = 5; + const int kDelayMs2 = 6; + InsertAndSendFramesWithInterval(1300, kFrameInterval33ms, kWidth, kHeight, + kDelayMs1); + InsertAndSendFramesWithInterval(1, kFrameInterval33ms, kWidth, kHeight, + kDelayMs2); + }); + + EXPECT_TRUE(event.Wait(10000)); +} + } // namespace webrtc diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index f654543129..118959ebec 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -199,8 +199,11 @@ CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { class ViEEncoder::EncodeTask : public rtc::QueuedTask { public: - EncodeTask(const VideoFrame& frame, ViEEncoder* vie_encoder) - : vie_encoder_(vie_encoder) { + EncodeTask(const VideoFrame& frame, + ViEEncoder* vie_encoder, + int64_t time_when_posted_in_ms) + : vie_encoder_(vie_encoder), + time_when_posted_ms_(time_when_posted_in_ms) { frame_.ShallowCopy(frame); ++vie_encoder_->posted_frames_waiting_for_encode_; } @@ -209,7 +212,7 @@ class ViEEncoder::EncodeTask : public rtc::QueuedTask { bool Run() override { RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0); if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) { - vie_encoder_->EncodeVideoFrame(frame_); + vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_ms_); } else { // There is a newer frame in flight. Do not encode this frame. LOG(LS_VERBOSE) @@ -218,7 +221,8 @@ class ViEEncoder::EncodeTask : public rtc::QueuedTask { return true; } VideoFrame frame_; - ViEEncoder* vie_encoder_; + ViEEncoder* const vie_encoder_; + const int64_t time_when_posted_ms_; }; ViEEncoder::ViEEncoder(uint32_t number_of_cores, @@ -256,10 +260,11 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, encoder_queue_("EncoderQueue") { vp_->EnableTemporalDecimation(false); - encoder_queue_.PostTask([this] { + encoder_queue_.PostTask([this, encoder_timing] { RTC_DCHECK_RUN_ON(&encoder_queue_); video_sender_.RegisterExternalEncoder( settings_.encoder, settings_.payload_type, settings_.internal_source); + overuse_detector_.StartCheckForOveruse(); }); } @@ -276,19 +281,18 @@ void ViEEncoder::Stop() { } RTC_DCHECK_RUN_ON(&encoder_queue_); video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); + overuse_detector_.StopCheckForOveruse(); shutdown_event_.Set(); } void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { RTC_DCHECK(!module_process_thread_); module_process_thread_ = module_process_thread; - module_process_thread_->RegisterModule(&overuse_detector_); module_process_thread_->RegisterModule(&video_sender_); module_process_thread_checker_.DetachFromThread(); } void ViEEncoder::DeRegisterProcessThread() { - module_process_thread_->DeRegisterModule(&overuse_detector_); module_process_thread_->DeRegisterModule(&video_sender_); } @@ -397,9 +401,8 @@ void ViEEncoder::IncomingCapturedFrame(const VideoFrame& video_frame) { } last_captured_timestamp_ = incoming_frame.ntp_time_ms(); - overuse_detector_.FrameCaptured(incoming_frame); - encoder_queue_.PostTask( - std::unique_ptr(new EncodeTask(incoming_frame, this))); + encoder_queue_.PostTask(std::unique_ptr( + new EncodeTask(incoming_frame, this, clock_->TimeInMilliseconds()))); } bool ViEEncoder::EncoderPaused() const { @@ -430,7 +433,8 @@ void ViEEncoder::TraceFrameDropEnd() { encoder_paused_and_dropped_frame_ = false; } -void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { +void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, + int64_t time_when_posted_in_ms) { RTC_DCHECK_RUN_ON(&encoder_queue_); if (pre_encode_callback_) pre_encode_callback_->OnFrame(video_frame); @@ -454,6 +458,8 @@ void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { } } + overuse_detector_.FrameCaptured(video_frame, time_when_posted_in_ms); + if (encoder_config_.codecType == webrtc::kVideoCodecVP8) { webrtc::CodecSpecificInfo codec_specific_info; codec_specific_info.codecType = webrtc::kVideoCodecVP8; @@ -498,7 +504,12 @@ EncodedImageCallback::Result ViEEncoder::OnEncodedImage( EncodedImageCallback::Result result = sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); - overuse_detector_.FrameSent(encoded_image._timeStamp); + int64_t time_sent = clock_->TimeInMilliseconds(); + uint32_t timestamp = encoded_image._timeStamp; + encoder_queue_.PostTask([this, timestamp, time_sent] { + RTC_DCHECK_RUN_ON(&encoder_queue_); + overuse_detector_.FrameSent(timestamp, time_sent); + }); return result; } @@ -575,7 +586,7 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, } void ViEEncoder::OveruseDetected() { - RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + RTC_DCHECK_RUN_ON(&encoder_queue_); // TODO(perkj): When ViEEncoder inherit rtc::VideoSink instead of // VideoCaptureInput |load_observer_| should be removed and overuse be // expressed as rtc::VideoSinkWants instead. @@ -584,7 +595,7 @@ void ViEEncoder::OveruseDetected() { } void ViEEncoder::NormalUsage() { - RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + RTC_DCHECK_RUN_ON(&encoder_queue_); if (load_observer_) load_observer_->OnLoadUpdate(LoadObserver::kUnderuse); } diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 7408a53c1f..dd74b20409 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -101,7 +101,8 @@ class ViEEncoder : public VideoCaptureInput, void SendStatistics(uint32_t bit_rate, uint32_t frame_rate) override; - void EncodeVideoFrame(const VideoFrame& frame); + void EncodeVideoFrame(const VideoFrame& frame, + int64_t time_when_posted_in_ms); // Implements EncodedImageCallback. EncodedImageCallback::Result OnEncodedImage( @@ -125,8 +126,9 @@ class ViEEncoder : public VideoCaptureInput, const std::unique_ptr vp_; vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); - OveruseFrameDetector overuse_detector_; - LoadObserver* const load_observer_ ACCESS_ON(&module_process_thread_checker_); + + OveruseFrameDetector overuse_detector_ ACCESS_ON(&encoder_queue_); + LoadObserver* const load_observer_ ACCESS_ON(&encoder_queue_); SendStatisticsProxy* const stats_proxy_; rtc::VideoSinkInterface* const pre_encode_callback_;