diff --git a/webrtc/frame_callback.h b/webrtc/frame_callback.h index b7f2210334..2bae25036d 100644 --- a/webrtc/frame_callback.h +++ b/webrtc/frame_callback.h @@ -12,6 +12,7 @@ #define WEBRTC_FRAME_CALLBACK_H_ #include +#include #include "webrtc/common_types.h" @@ -43,6 +44,7 @@ class I420FrameCallback { class EncodedFrameObserver { public: virtual void EncodedFrameCallback(const EncodedFrame& encoded_frame) = 0; + virtual void OnEncodeTiming(int64_t capture_ntp_ms, int encode_duration_ms) {} protected: virtual ~EncodedFrameObserver() {} diff --git a/webrtc/media/webrtc/webrtcvideoengine2.cc b/webrtc/media/webrtc/webrtcvideoengine2.cc index 265aa12108..0489198fac 100644 --- a/webrtc/media/webrtc/webrtcvideoengine2.cc +++ b/webrtc/media/webrtc/webrtcvideoengine2.cc @@ -1768,6 +1768,7 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions( AllocatedEncoder new_encoder = CreateVideoEncoder(codec_settings.codec); parameters_.config.encoder_settings.encoder = new_encoder.encoder; + parameters_.config.encoder_settings.full_overuse_time = new_encoder.external; parameters_.config.encoder_settings.payload_name = codec_settings.codec.name; parameters_.config.encoder_settings.payload_type = codec_settings.codec.id; if (new_encoder.external) { diff --git a/webrtc/media/webrtc/webrtcvideoengine2_unittest.cc b/webrtc/media/webrtc/webrtcvideoengine2_unittest.cc index 7a7c81535c..663f40dcde 100644 --- a/webrtc/media/webrtc/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/webrtc/webrtcvideoengine2_unittest.cc @@ -153,6 +153,8 @@ class WebRtcVideoEngine2Test : public ::testing::Test { cricket::WebRtcVideoDecoderFactory* decoder_factory, const std::vector& codecs); + void TestExtendedEncoderOveruse(bool use_external_encoder); + webrtc::test::ScopedFieldTrials override_field_trials_; // Used in WebRtcVideoEngine2VoiceTest, but defined here so it's properly // initialized when the constructor is called. @@ -356,6 +358,42 @@ TEST_F(WebRtcVideoEngine2Test, UseExternalFactoryForVp8WhenSupported) { EXPECT_EQ(0u, encoder_factory.encoders().size()); } +void WebRtcVideoEngine2Test::TestExtendedEncoderOveruse( + bool use_external_encoder) { + cricket::FakeWebRtcVideoEncoderFactory encoder_factory; + encoder_factory.AddSupportedVideoCodecType(webrtc::kVideoCodecVP8, "VP8"); + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(kVp8Codec); + rtc::scoped_ptr channel; + FakeCall* fake_call = new FakeCall(webrtc::Call::Config()); + call_.reset(fake_call); + if (use_external_encoder) { + channel.reset( + SetUpForExternalEncoderFactory(&encoder_factory, parameters.codecs)); + } else { + engine_.Init(); + channel.reset(engine_.CreateChannel(call_.get(), cricket::VideoOptions())); + } + ASSERT_TRUE( + channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + EXPECT_TRUE(channel->SetSendParameters(parameters)); + EXPECT_TRUE(channel->SetSend(true)); + FakeVideoSendStream* stream = fake_call->GetVideoSendStreams()[0]; + + EXPECT_EQ(use_external_encoder, + stream->GetConfig().encoder_settings.full_overuse_time); + // Remove stream previously added to free the external encoder instance. + EXPECT_TRUE(channel->RemoveSendStream(kSsrc)); +} + +TEST_F(WebRtcVideoEngine2Test, EnablesFullEncoderTimeForExternalEncoders) { + TestExtendedEncoderOveruse(true); +} + +TEST_F(WebRtcVideoEngine2Test, DisablesFullEncoderTimeForNonExternalEncoders) { + TestExtendedEncoderOveruse(false); +} + TEST_F(WebRtcVideoEngine2Test, CanConstructDecoderForVp9EncoderFactory) { cricket::FakeWebRtcVideoEncoderFactory encoder_factory; encoder_factory.AddSupportedVideoCodecType(webrtc::kVideoCodecVP9, "VP9"); diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h index 2786d8b82a..581ab0eda5 100644 --- a/webrtc/modules/video_coding/video_coding_impl.h +++ b/webrtc/modules/video_coding/video_coding_impl.h @@ -31,8 +31,6 @@ namespace webrtc { -class EncodedFrameObserver; - namespace vcm { class VCMProcessTimer { diff --git a/webrtc/video/encoder_state_feedback_unittest.cc b/webrtc/video/encoder_state_feedback_unittest.cc index b9199ddeb3..8fa1c12f76 100644 --- a/webrtc/video/encoder_state_feedback_unittest.cc +++ b/webrtc/video/encoder_state_feedback_unittest.cc @@ -31,7 +31,13 @@ namespace webrtc { class MockVieEncoder : public ViEEncoder { public: explicit MockVieEncoder(ProcessThread* process_thread, PacedSender* pacer) - : ViEEncoder(1, process_thread, nullptr, nullptr, pacer, nullptr) {} + : ViEEncoder(1, + process_thread, + nullptr, + nullptr, + nullptr, + pacer, + nullptr) {} ~MockVieEncoder() {} MOCK_METHOD1(OnReceivedIntraFrameRequest, diff --git a/webrtc/video/end_to_end_tests.cc b/webrtc/video/end_to_end_tests.cc index ce3255c413..5385450298 100644 --- a/webrtc/video/end_to_end_tests.cc +++ b/webrtc/video/end_to_end_tests.cc @@ -2653,7 +2653,7 @@ TEST_F(EndToEndTest, GetStats) { stats.substreams.size() == expected_send_ssrcs_.size(); send_stats_filled_["CpuOveruseMetrics"] |= - stats.avg_encode_time_ms != 0 || stats.encode_usage_percent != 0; + stats.avg_encode_time_ms != 0 && stats.encode_usage_percent != 0; send_stats_filled_["EncoderImplementationName"] |= stats.encoder_implementation_name == diff --git a/webrtc/video/overuse_frame_detector.cc b/webrtc/video/overuse_frame_detector.cc index d971ad9d3e..e589c191f4 100644 --- a/webrtc/video/overuse_frame_detector.cc +++ b/webrtc/video/overuse_frame_detector.cc @@ -20,7 +20,9 @@ #include "webrtc/base/checks.h" #include "webrtc/base/exp_filter.h" #include "webrtc/base/logging.h" +#include "webrtc/frame_callback.h" #include "webrtc/system_wrappers/include/clock.h" +#include "webrtc/video_frame.h" namespace webrtc { @@ -115,106 +117,47 @@ class OveruseFrameDetector::SendProcessingUsage { rtc::scoped_ptr filtered_frame_diff_ms_; }; -// Class for calculating the processing time of frames. -class OveruseFrameDetector::FrameQueue { - public: - FrameQueue() : last_processing_time_ms_(-1) {} - ~FrameQueue() {} - - // Called when a frame is captured. - // Starts the measuring of the processing time of the frame. - void Start(int64_t capture_time, int64_t now) { - const size_t kMaxSize = 90; // Allows for processing time of 1.5s at 60fps. - if (frame_times_.size() > kMaxSize) { - LOG(LS_WARNING) << "Max size reached, removed oldest frame."; - frame_times_.erase(frame_times_.begin()); - } - if (frame_times_.find(capture_time) != frame_times_.end()) { - // Frame should not exist. - assert(false); - return; - } - frame_times_[capture_time] = now; - } - - // Called when the processing of a frame has finished. - // Returns the processing time of the frame. - int End(int64_t capture_time, int64_t now) { - std::map::iterator it = frame_times_.find(capture_time); - if (it == frame_times_.end()) { - return -1; - } - // Remove any old frames up to current. - // Old frames have been skipped by the capture process thread. - // TODO(asapersson): Consider measuring time from first frame in list. - last_processing_time_ms_ = now - (*it).second; - frame_times_.erase(frame_times_.begin(), ++it); - return last_processing_time_ms_; - } - - void Reset() { frame_times_.clear(); } - int NumFrames() const { return static_cast(frame_times_.size()); } - int last_processing_time_ms() const { return last_processing_time_ms_; } - - private: - // Captured frames mapped by the capture time. - std::map frame_times_; - int last_processing_time_ms_; -}; - - OveruseFrameDetector::OveruseFrameDetector( Clock* clock, const CpuOveruseOptions& options, CpuOveruseObserver* observer, + EncodedFrameObserver* encoder_timing, CpuOveruseMetricsObserver* metrics_observer) : options_(options), observer_(observer), + encoder_timing_(encoder_timing), metrics_observer_(metrics_observer), clock_(clock), num_process_times_(0), - last_capture_time_(0), + last_capture_time_ms_(-1), + last_processed_capture_time_ms_(-1), num_pixels_(0), - next_process_time_(clock_->TimeInMilliseconds()), - last_overuse_time_(0), + next_process_time_ms_(clock_->TimeInMilliseconds()), + last_overuse_time_ms_(-1), checks_above_threshold_(0), num_overuse_detections_(0), - last_rampup_time_(0), + last_rampup_time_ms_(-1), in_quick_rampup_(false), current_rampup_delay_ms_(kStandardRampUpDelayMs), - last_sample_time_ms_(0), - usage_(new SendProcessingUsage(options)), - frame_queue_(new FrameQueue()) { + usage_(new SendProcessingUsage(options)) { RTC_DCHECK(metrics_observer != nullptr); - // Make sure stats are initially up-to-date. This simplifies unit testing - // since we don't have to trigger an update using one of the methods which - // would also alter the overuse state. - UpdateCpuOveruseMetrics(); processing_thread_.DetachFromThread(); } OveruseFrameDetector::~OveruseFrameDetector() { } -int OveruseFrameDetector::LastProcessingTimeMs() const { - rtc::CritScope cs(&crit_); - return frame_queue_->last_processing_time_ms(); -} +void OveruseFrameDetector::EncodedFrameTimeMeasured(int encode_duration_ms) { + if (!metrics_) + metrics_ = rtc::Optional(CpuOveruseMetrics()); + metrics_->encode_usage_percent = usage_->Value(); -int OveruseFrameDetector::FramesInQueue() const { - rtc::CritScope cs(&crit_); - return frame_queue_->NumFrames(); -} - -void OveruseFrameDetector::UpdateCpuOveruseMetrics() { - metrics_.encode_usage_percent = usage_->Value(); - - metrics_observer_->CpuOveruseMetricsUpdated(metrics_); + metrics_observer_->OnEncodedFrameTimeMeasured(encode_duration_ms, *metrics_); } int64_t OveruseFrameDetector::TimeUntilNextProcess() { RTC_DCHECK(processing_thread_.CalledOnValidThread()); - return next_process_time_ - clock_->TimeInMilliseconds(); + return next_process_time_ms_ - clock_->TimeInMilliseconds(); } bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const { @@ -225,56 +168,80 @@ bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const { } bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now) const { - if (last_capture_time_ == 0) { + if (last_capture_time_ms_ == -1) return false; - } - return (now - last_capture_time_) > options_.frame_timeout_interval_ms; + return (now - last_capture_time_ms_) > options_.frame_timeout_interval_ms; } void OveruseFrameDetector::ResetAll(int num_pixels) { num_pixels_ = num_pixels; usage_->Reset(); - frame_queue_->Reset(); - last_capture_time_ = 0; + frame_timing_.clear(); + last_capture_time_ms_ = -1; + last_processed_capture_time_ms_ = -1; num_process_times_ = 0; - UpdateCpuOveruseMetrics(); + metrics_ = rtc::Optional(); } -void OveruseFrameDetector::FrameCaptured(int width, - int height, - int64_t capture_time_ms) { +void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame) { rtc::CritScope cs(&crit_); int64_t now = clock_->TimeInMilliseconds(); - if (FrameSizeChanged(width * height) || FrameTimeoutDetected(now)) { - ResetAll(width * height); + if (FrameSizeChanged(frame.width() * frame.height()) || + FrameTimeoutDetected(now)) { + ResetAll(frame.width() * frame.height()); } - if (last_capture_time_ != 0) - usage_->AddCaptureSample(now - last_capture_time_); + if (last_capture_time_ms_ != -1) + usage_->AddCaptureSample(now - last_capture_time_ms_); - last_capture_time_ = now; + last_capture_time_ms_ = now; - frame_queue_->Start(capture_time_ms, now); + frame_timing_.push_back( + FrameTiming(frame.ntp_time_ms(), frame.timestamp(), now)); } -void OveruseFrameDetector::FrameSent(int64_t capture_time_ms) { +void OveruseFrameDetector::FrameSent(uint32_t timestamp) { rtc::CritScope cs(&crit_); - int delay_ms = frame_queue_->End(capture_time_ms, - clock_->TimeInMilliseconds()); - if (delay_ms > 0) { - AddProcessingTime(delay_ms); - } -} - -void OveruseFrameDetector::AddProcessingTime(int elapsed_ms) { + // Delay before reporting actual encoding time, used to have the ability to + // detect total encoding time when encoding more than one layer. Encoding is + // here assumed to finish within a second (or that we get enough long-time + // samples before one second to trigger an overuse even when this is not the + // case). + static const int64_t kEncodingTimeMeasureWindowMs = 1000; int64_t now = clock_->TimeInMilliseconds(); - if (last_sample_time_ms_ != 0) { - int64_t diff_ms = now - last_sample_time_ms_; - usage_->AddSample(elapsed_ms, diff_ms); + for (auto& it : frame_timing_) { + if (it.timestamp == timestamp) { + it.last_send_ms = now; + break; + } + } + // TODO(pbos): Handle the case/log errors when not finding the corresponding + // frame (either very slow encoding or incorrect wrong timestamps returned + // from the encoder). + // This is currently the case for all frames on ChromeOS, so logging them + // would be spammy, and triggering overuse would be wrong. + // https://crbug.com/350106 + while (!frame_timing_.empty()) { + FrameTiming timing = frame_timing_.front(); + if (now - timing.capture_ms < kEncodingTimeMeasureWindowMs) + break; + if (timing.last_send_ms != -1) { + int encode_duration_ms = + static_cast(timing.last_send_ms - timing.capture_ms); + if (encoder_timing_) { + encoder_timing_->OnEncodeTiming(timing.capture_ntp_ms, + encode_duration_ms); + } + if (last_processed_capture_time_ms_ != -1) { + int64_t diff_ms = timing.capture_ms - last_processed_capture_time_ms_; + usage_->AddSample(encode_duration_ms, diff_ms); + } + last_processed_capture_time_ms_ = timing.capture_ms; + EncodedFrameTimeMeasured(encode_duration_ms); + } + frame_timing_.pop_front(); } - last_sample_time_ms_ = now; - UpdateCpuOveruseMetrics(); } int32_t OveruseFrameDetector::Process() { @@ -283,28 +250,28 @@ int32_t OveruseFrameDetector::Process() { int64_t now = clock_->TimeInMilliseconds(); // Used to protect against Process() being called too often. - if (now < next_process_time_) + if (now < next_process_time_ms_) return 0; - next_process_time_ = now + kProcessIntervalMs; + next_process_time_ms_ = now + kProcessIntervalMs; CpuOveruseMetrics current_metrics; { rtc::CritScope cs(&crit_); ++num_process_times_; - - current_metrics = metrics_; - if (num_process_times_ <= options_.min_process_count) + if (num_process_times_ <= options_.min_process_count || !metrics_) return 0; + + current_metrics = *metrics_; } if (IsOverusing(current_metrics)) { // If the last thing we did was going up, and now have to back down, we need // to check if this peak was short. If so we should back off to avoid going // back and forth between this load, the system doesn't seem to handle it. - bool check_for_backoff = last_rampup_time_ > last_overuse_time_; + bool check_for_backoff = last_rampup_time_ms_ > last_overuse_time_ms_; if (check_for_backoff) { - if (now - last_rampup_time_ < kStandardRampUpDelayMs || + if (now - last_rampup_time_ms_ < kStandardRampUpDelayMs || num_overuse_detections_ > kMaxOverusesBeforeApplyRampupDelay) { // Going up was not ok for very long, back off. current_rampup_delay_ms_ *= kRampUpBackoffFactor; @@ -316,7 +283,7 @@ int32_t OveruseFrameDetector::Process() { } } - last_overuse_time_ = now; + last_overuse_time_ms_ = now; in_quick_rampup_ = false; checks_above_threshold_ = 0; ++num_overuse_detections_; @@ -324,7 +291,7 @@ int32_t OveruseFrameDetector::Process() { if (observer_ != NULL) observer_->OveruseDetected(); } else if (IsUnderusing(current_metrics, now)) { - last_rampup_time_ = now; + last_rampup_time_ms_ = now; in_quick_rampup_ = true; if (observer_ != NULL) @@ -355,7 +322,7 @@ bool OveruseFrameDetector::IsOverusing(const CpuOveruseMetrics& metrics) { bool OveruseFrameDetector::IsUnderusing(const CpuOveruseMetrics& metrics, int64_t time_now) { int delay = in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_; - if (time_now < last_rampup_time_ + delay) + if (time_now < last_rampup_time_ms_ + delay) return false; return metrics.encode_usage_percent < diff --git a/webrtc/video/overuse_frame_detector.h b/webrtc/video/overuse_frame_detector.h index 0ef2e43857..8184aaf369 100644 --- a/webrtc/video/overuse_frame_detector.h +++ b/webrtc/video/overuse_frame_detector.h @@ -11,8 +11,11 @@ #ifndef WEBRTC_VIDEO_OVERUSE_FRAME_DETECTOR_H_ #define WEBRTC_VIDEO_OVERUSE_FRAME_DETECTOR_H_ +#include + #include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" +#include "webrtc/base/optional.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/base/exp_filter.h" #include "webrtc/base/thread_annotations.h" @@ -22,6 +25,8 @@ namespace webrtc { class Clock; +class EncodedFrameObserver; +class VideoFrame; // CpuOveruseObserver is called when a system overuse is detected and // VideoEngine cannot keep up the encoding frequency. @@ -68,10 +73,10 @@ struct CpuOveruseMetrics { class CpuOveruseMetricsObserver { public: virtual ~CpuOveruseMetricsObserver() {} - virtual void CpuOveruseMetricsUpdated(const CpuOveruseMetrics& metrics) = 0; + virtual void OnEncodedFrameTimeMeasured(int encode_duration_ms, + const CpuOveruseMetrics& metrics) = 0; }; - // Use to detect system overuse based on the send-side processing time of // incoming frames. class OveruseFrameDetector : public Module { @@ -79,18 +84,15 @@ class OveruseFrameDetector : public Module { OveruseFrameDetector(Clock* clock, const CpuOveruseOptions& options, CpuOveruseObserver* overuse_observer, + EncodedFrameObserver* encoder_timing_, CpuOveruseMetricsObserver* metrics_observer); ~OveruseFrameDetector(); // Called for each captured frame. - void FrameCaptured(int width, int height, int64_t capture_time_ms); + void FrameCaptured(const VideoFrame& frame); // Called for each sent frame. - void FrameSent(int64_t capture_time_ms); - - // Only public for testing. - int LastProcessingTimeMs() const; - int FramesInQueue() const; + void FrameSent(uint32_t timestamp); // Implements Module. int64_t TimeUntilNextProcess() override; @@ -98,13 +100,20 @@ class OveruseFrameDetector : public Module { private: class SendProcessingUsage; - class FrameQueue; + struct FrameTiming { + FrameTiming(int64_t capture_ntp_ms, uint32_t timestamp, int64_t now) + : capture_ntp_ms(capture_ntp_ms), + timestamp(timestamp), + capture_ms(now), + last_send_ms(-1) {} + int64_t capture_ntp_ms; + uint32_t timestamp; + int64_t capture_ms; + int64_t last_send_ms; + }; - void UpdateCpuOveruseMetrics() EXCLUSIVE_LOCKS_REQUIRED(crit_); - - // TODO(asapersson): This method is only used on one thread, so it shouldn't - // need a guard. - void AddProcessingTime(int elapsed_ms) EXCLUSIVE_LOCKS_REQUIRED(crit_); + void EncodedFrameTimeMeasured(int encode_duration_ms) + EXCLUSIVE_LOCKS_REQUIRED(crit_); // Only called on the processing thread. bool IsOverusing(const CpuOveruseMetrics& metrics); @@ -125,34 +134,34 @@ class OveruseFrameDetector : public Module { // Observer getting overuse reports. CpuOveruseObserver* const observer_; + EncodedFrameObserver* const encoder_timing_; // Stats metrics. CpuOveruseMetricsObserver* const metrics_observer_; - CpuOveruseMetrics metrics_ GUARDED_BY(crit_); + rtc::Optional metrics_ GUARDED_BY(crit_); Clock* const clock_; int64_t num_process_times_ GUARDED_BY(crit_); - int64_t last_capture_time_ GUARDED_BY(crit_); + int64_t last_capture_time_ms_ GUARDED_BY(crit_); + int64_t last_processed_capture_time_ms_ GUARDED_BY(crit_); // Number of pixels of last captured frame. int num_pixels_ GUARDED_BY(crit_); // These seven members are only accessed on the processing thread. - int64_t next_process_time_; - int64_t last_overuse_time_; + int64_t next_process_time_ms_; + int64_t last_overuse_time_ms_; int checks_above_threshold_; int num_overuse_detections_; - int64_t last_rampup_time_; + int64_t last_rampup_time_ms_; bool in_quick_rampup_; int current_rampup_delay_ms_; - int64_t last_sample_time_ms_; // Only accessed by one thread. - // TODO(asapersson): Can these be regular members (avoid separate heap // allocs)? const rtc::scoped_ptr usage_ GUARDED_BY(crit_); - const rtc::scoped_ptr frame_queue_ GUARDED_BY(crit_); + std::list frame_timing_ GUARDED_BY(crit_); rtc::ThreadChecker processing_thread_; diff --git a/webrtc/video/overuse_frame_detector_unittest.cc b/webrtc/video/overuse_frame_detector_unittest.cc index 65e006b485..1a6384ca8a 100644 --- a/webrtc/video/overuse_frame_detector_unittest.cc +++ b/webrtc/video/overuse_frame_detector_unittest.cc @@ -15,6 +15,7 @@ #include "webrtc/base/scoped_ptr.h" #include "webrtc/system_wrappers/include/clock.h" +#include "webrtc/video_frame.h" namespace webrtc { namespace { @@ -59,11 +60,12 @@ class OveruseFrameDetectorTest : public ::testing::Test, } void ReinitializeOveruseDetector() { - overuse_detector_.reset(new OveruseFrameDetector(clock_.get(), options_, - observer_.get(), this)); + overuse_detector_.reset(new OveruseFrameDetector( + clock_.get(), options_, observer_.get(), nullptr, this)); } - void CpuOveruseMetricsUpdated(const CpuOveruseMetrics& metrics) override { + void OnEncodedFrameTimeMeasured(int encode_time_ms, + const CpuOveruseMetrics& metrics) override { metrics_ = metrics; } @@ -72,17 +74,31 @@ class OveruseFrameDetectorTest : public ::testing::Test, options_.high_encode_usage_threshold_percent) / 2.0f) + 0.5; } - void InsertAndSendFramesWithInterval( - int num_frames, int interval_ms, int width, int height, int delay_ms) { + void InsertAndSendFramesWithInterval(int num_frames, + int interval_ms, + int width, + int height, + int delay_ms) { + VideoFrame frame; + frame.CreateEmptyFrame(width, height, width, width / 2, width / 2); + uint32_t timestamp = 0; while (num_frames-- > 0) { - int64_t capture_time_ms = clock_->TimeInMilliseconds(); - overuse_detector_->FrameCaptured(width, height, capture_time_ms); + frame.set_timestamp(timestamp); + overuse_detector_->FrameCaptured(frame); clock_->AdvanceTimeMilliseconds(delay_ms); - overuse_detector_->FrameSent(capture_time_ms); + overuse_detector_->FrameSent(timestamp); clock_->AdvanceTimeMilliseconds(interval_ms - delay_ms); + timestamp += interval_ms * 90; } } + void ForceUpdate(int width, int height) { + // Insert one frame, wait a second and then put in another to force update + // the usage. From the tests where these are used, adding another sample + // doesn't affect the expected outcome (this is mainly to check initial + // values and whether the overuse detector has been reset or not). + InsertAndSendFramesWithInterval(2, 1000, width, height, kFrameInterval33ms); + } void TriggerOveruse(int num_times) { const int kDelayMs = 32; for (int i = 0; i < num_times; ++i) { @@ -131,7 +147,7 @@ TEST_F(OveruseFrameDetectorTest, OveruseAndRecover) { TEST_F(OveruseFrameDetectorTest, OveruseAndRecoverWithNoObserver) { overuse_detector_.reset( - new OveruseFrameDetector(clock_.get(), options_, nullptr, this)); + new OveruseFrameDetector(clock_.get(), options_, nullptr, nullptr, this)); EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(0); TriggerOveruse(options_.high_threshold_consecutive_count); EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(0); @@ -149,8 +165,8 @@ TEST_F(OveruseFrameDetectorTest, DoubleOveruseAndRecover) { TEST_F(OveruseFrameDetectorTest, TriggerUnderuseWithMinProcessCount) { options_.min_process_count = 1; CpuOveruseObserverImpl overuse_observer; - overuse_detector_.reset(new OveruseFrameDetector(clock_.get(), options_, - &overuse_observer, this)); + overuse_detector_.reset(new OveruseFrameDetector( + clock_.get(), options_, &overuse_observer, nullptr, this)); InsertAndSendFramesWithInterval( 1200, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms); overuse_detector_->Process(); @@ -189,17 +205,18 @@ TEST_F(OveruseFrameDetectorTest, ProcessingUsage) { } TEST_F(OveruseFrameDetectorTest, ResetAfterResolutionChange) { + ForceUpdate(kWidth, kHeight); EXPECT_EQ(InitialUsage(), UsagePercent()); InsertAndSendFramesWithInterval( 1000, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms); EXPECT_NE(InitialUsage(), UsagePercent()); - // Verify reset. - InsertAndSendFramesWithInterval( - 1, kFrameInterval33ms, kWidth, kHeight + 1, kProcessTime5ms); + // Verify reset (with new width/height). + ForceUpdate(kWidth, kHeight + 1); EXPECT_EQ(InitialUsage(), UsagePercent()); } TEST_F(OveruseFrameDetectorTest, ResetAfterFrameTimeout) { + ForceUpdate(kWidth, kHeight); EXPECT_EQ(InitialUsage(), UsagePercent()); InsertAndSendFramesWithInterval( 1000, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms); @@ -211,6 +228,7 @@ TEST_F(OveruseFrameDetectorTest, ResetAfterFrameTimeout) { InsertAndSendFramesWithInterval( 2, options_.frame_timeout_interval_ms + 1, kWidth, kHeight, kProcessTime5ms); + ForceUpdate(kWidth, kHeight); EXPECT_EQ(InitialUsage(), UsagePercent()); } @@ -220,91 +238,65 @@ TEST_F(OveruseFrameDetectorTest, MinFrameSamplesBeforeUpdating) { InsertAndSendFramesWithInterval( 40, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms); EXPECT_EQ(InitialUsage(), UsagePercent()); + // Pass time far enough to digest all previous samples. + clock_->AdvanceTimeMilliseconds(1000); + InsertAndSendFramesWithInterval(1, kFrameInterval33ms, kWidth, kHeight, + kProcessTime5ms); + // The last sample has not been processed here. + EXPECT_EQ(InitialUsage(), UsagePercent()); + + // Pass time far enough to digest all previous samples, 41 in total. + clock_->AdvanceTimeMilliseconds(1000); InsertAndSendFramesWithInterval( 1, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms); EXPECT_NE(InitialUsage(), UsagePercent()); } TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) { + ForceUpdate(kWidth, kHeight); EXPECT_EQ(InitialUsage(), UsagePercent()); } -TEST_F(OveruseFrameDetectorTest, FrameDelay_OneFrame) { - const int kProcessingTimeMs = 100; - overuse_detector_->FrameCaptured(kWidth, kHeight, 33); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs); - EXPECT_EQ(-1, overuse_detector_->LastProcessingTimeMs()); - overuse_detector_->FrameSent(33); - EXPECT_EQ(kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs()); - EXPECT_EQ(0, overuse_detector_->FramesInQueue()); -} - -TEST_F(OveruseFrameDetectorTest, FrameDelay_TwoFrames) { - const int kProcessingTimeMs1 = 100; - const int kProcessingTimeMs2 = 50; - const int kTimeBetweenFramesMs = 200; - overuse_detector_->FrameCaptured(kWidth, kHeight, 33); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs1); - overuse_detector_->FrameSent(33); - EXPECT_EQ(kProcessingTimeMs1, overuse_detector_->LastProcessingTimeMs()); - clock_->AdvanceTimeMilliseconds(kTimeBetweenFramesMs); - overuse_detector_->FrameCaptured(kWidth, kHeight, 66); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs2); - overuse_detector_->FrameSent(66); - EXPECT_EQ(kProcessingTimeMs2, overuse_detector_->LastProcessingTimeMs()); -} - -TEST_F(OveruseFrameDetectorTest, FrameDelay_MaxQueueSize) { - const int kMaxQueueSize = 91; - for (int i = 0; i < kMaxQueueSize * 2; ++i) { - overuse_detector_->FrameCaptured(kWidth, kHeight, i); +TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { + EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(testing::AtLeast(1)); + static const int kIntervalMs = 33; + static const size_t kNumFramesEncodingDelay = 3; + VideoFrame frame; + frame.CreateEmptyFrame(kWidth, kHeight, kWidth, kWidth / 2, kWidth / 2); + for (size_t i = 0; i < 1000; ++i) { + // Unique timestamps. + frame.set_timestamp(static_cast(i)); + overuse_detector_->FrameCaptured(frame); + clock_->AdvanceTimeMilliseconds(kIntervalMs); + if (i > kNumFramesEncodingDelay) { + overuse_detector_->FrameSent( + static_cast(i - kNumFramesEncodingDelay)); + } + overuse_detector_->Process(); } - EXPECT_EQ(kMaxQueueSize, overuse_detector_->FramesInQueue()); } -TEST_F(OveruseFrameDetectorTest, FrameDelay_NonProcessedFramesRemoved) { - const int kProcessingTimeMs = 100; - overuse_detector_->FrameCaptured(kWidth, kHeight, 33); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs); - overuse_detector_->FrameCaptured(kWidth, kHeight, 35); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs); - overuse_detector_->FrameCaptured(kWidth, kHeight, 66); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs); - overuse_detector_->FrameCaptured(kWidth, kHeight, 99); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs); - EXPECT_EQ(-1, overuse_detector_->LastProcessingTimeMs()); - EXPECT_EQ(4, overuse_detector_->FramesInQueue()); - overuse_detector_->FrameSent(66); - // Frame 33, 35 removed, 66 processed, 99 not processed. - EXPECT_EQ(2 * kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs()); - EXPECT_EQ(1, overuse_detector_->FramesInQueue()); - overuse_detector_->FrameSent(99); - EXPECT_EQ(kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs()); - EXPECT_EQ(0, overuse_detector_->FramesInQueue()); -} - -TEST_F(OveruseFrameDetectorTest, FrameDelay_ResetClearsFrames) { - const int kProcessingTimeMs = 100; - overuse_detector_->FrameCaptured(kWidth, kHeight, 33); - EXPECT_EQ(1, overuse_detector_->FramesInQueue()); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs); - // Verify reset (resolution changed). - overuse_detector_->FrameCaptured(kWidth, kHeight + 1, 66); - EXPECT_EQ(1, overuse_detector_->FramesInQueue()); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs); - overuse_detector_->FrameSent(66); - EXPECT_EQ(kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs()); - EXPECT_EQ(0, overuse_detector_->FramesInQueue()); -} - -TEST_F(OveruseFrameDetectorTest, FrameDelay_NonMatchingSendFrameIgnored) { - const int kProcessingTimeMs = 100; - overuse_detector_->FrameCaptured(kWidth, kHeight, 33); - clock_->AdvanceTimeMilliseconds(kProcessingTimeMs); - overuse_detector_->FrameSent(34); - EXPECT_EQ(-1, overuse_detector_->LastProcessingTimeMs()); - overuse_detector_->FrameSent(33); - EXPECT_EQ(kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs()); +TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) { + // >85% encoding time should trigger overuse. + EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(testing::AtLeast(1)); + static const int kIntervalMs = 33; + static const int kDelayMs = 30; + VideoFrame frame; + frame.CreateEmptyFrame(kWidth, kHeight, kWidth, kWidth / 2, kWidth / 2); + uint32_t timestamp = 0; + for (size_t i = 0; i < 1000; ++i) { + frame.set_timestamp(timestamp); + overuse_detector_->FrameCaptured(frame); + // Encode and send first parts almost instantly. + clock_->AdvanceTimeMilliseconds(1); + overuse_detector_->FrameSent(timestamp); + // Encode heavier part, resulting in >85% usage total. + clock_->AdvanceTimeMilliseconds(kDelayMs - 1); + overuse_detector_->FrameSent(timestamp); + clock_->AdvanceTimeMilliseconds(kIntervalMs - kDelayMs); + timestamp += kIntervalMs * 90; + overuse_detector_->Process(); + } } } // namespace webrtc diff --git a/webrtc/video/send_statistics_proxy.cc b/webrtc/video/send_statistics_proxy.cc index 7371111066..8102b9310e 100644 --- a/webrtc/video/send_statistics_proxy.cc +++ b/webrtc/video/send_statistics_proxy.cc @@ -189,9 +189,13 @@ void SendStatisticsProxy::OnOutgoingRate(uint32_t framerate, uint32_t bitrate) { stats_.media_bitrate_bps = bitrate; } -void SendStatisticsProxy::CpuOveruseMetricsUpdated( +void SendStatisticsProxy::OnEncodedFrameTimeMeasured( + int encode_time_ms, const CpuOveruseMetrics& metrics) { rtc::CritScope lock(&crit_); + uma_container_->encode_time_counter_.Add(encode_time_ms); + encode_time_.Apply(1.0f, encode_time_ms); + stats_.avg_encode_time_ms = round(encode_time_.filtered()); stats_.encode_usage_percent = metrics.encode_usage_percent; } @@ -331,13 +335,6 @@ void SendStatisticsProxy::OnIncomingFrame(int width, int height) { uma_container_->input_height_counter_.Add(height); } -void SendStatisticsProxy::OnEncodedFrame(int encode_time_ms) { - rtc::CritScope lock(&crit_); - uma_container_->encode_time_counter_.Add(encode_time_ms); - encode_time_.Apply(1.0f, encode_time_ms); - stats_.avg_encode_time_ms = round(encode_time_.filtered()); -} - void SendStatisticsProxy::RtcpPacketTypesCounterUpdated( uint32_t ssrc, const RtcpPacketTypeCounter& packet_counter) { diff --git a/webrtc/video/send_statistics_proxy.h b/webrtc/video/send_statistics_proxy.h index f4c3f5ae00..d2b9b565b3 100644 --- a/webrtc/video/send_statistics_proxy.h +++ b/webrtc/video/send_statistics_proxy.h @@ -52,12 +52,6 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver, // Used to update incoming frame rate. void OnIncomingFrame(int width, int height); - // Used to update encode time of frames. - void OnEncodedFrame(int encode_time_ms); - - // From VideoEncoderRateObserver. - void OnSetRates(uint32_t bitrate_bps, int framerate) override; - void OnEncoderImplementationName(const char* implementation_name); void OnOutgoingRate(uint32_t framerate, uint32_t bitrate); void OnSuspendChange(bool is_suspended); @@ -67,9 +61,14 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver, // how stats are collected. void SetContentType(VideoEncoderConfig::ContentType content_type); + // Implements VideoEncoderRateObserver. + void OnSetRates(uint32_t bitrate_bps, int framerate) override; + + // Implements CpuOveruseMetricsObserver. + void OnEncodedFrameTimeMeasured(int encode_time_ms, + const CpuOveruseMetrics& metrics) override; + protected: - // From CpuOveruseMetricsObserver. - void CpuOveruseMetricsUpdated(const CpuOveruseMetrics& metrics) override; // From RtcpStatisticsCallback. void StatisticsUpdated(const RtcpStatistics& statistics, uint32_t ssrc) override; diff --git a/webrtc/video/send_statistics_proxy_unittest.cc b/webrtc/video/send_statistics_proxy_unittest.cc index fc1f3fdbde..cf54190d13 100644 --- a/webrtc/video/send_statistics_proxy_unittest.cc +++ b/webrtc/video/send_statistics_proxy_unittest.cc @@ -289,12 +289,15 @@ TEST_F(SendStatisticsProxyTest, SendSideDelay) { ExpectEqual(expected_, stats); } -TEST_F(SendStatisticsProxyTest, OnEncodedFrame) { +TEST_F(SendStatisticsProxyTest, OnEncodedFrameTimeMeasured) { const int kEncodeTimeMs = 11; - statistics_proxy_->OnEncodedFrame(kEncodeTimeMs); + CpuOveruseMetrics metrics; + metrics.encode_usage_percent = 80; + statistics_proxy_->OnEncodedFrameTimeMeasured(kEncodeTimeMs, metrics); VideoSendStream::Stats stats = statistics_proxy_->GetStats(); EXPECT_EQ(kEncodeTimeMs, stats.avg_encode_time_ms); + EXPECT_EQ(metrics.encode_usage_percent, stats.encode_usage_percent); } TEST_F(SendStatisticsProxyTest, SwitchContentTypeUpdatesHistograms) { diff --git a/webrtc/video/video_capture_input.cc b/webrtc/video/video_capture_input.cc index 2b5d638c69..dfdf5ae482 100644 --- a/webrtc/video/video_capture_input.cc +++ b/webrtc/video/video_capture_input.cc @@ -14,7 +14,6 @@ #include "webrtc/base/logging.h" #include "webrtc/base/trace_event.h" #include "webrtc/modules/include/module_common_types.h" -#include "webrtc/modules/utility/include/process_thread.h" #include "webrtc/modules/video_capture/video_capture_factory.h" #include "webrtc/modules/video_processing/include/video_processing.h" #include "webrtc/modules/video_render/video_render_defines.h" @@ -27,15 +26,11 @@ namespace webrtc { namespace internal { -VideoCaptureInput::VideoCaptureInput( - ProcessThread* module_process_thread, - VideoCaptureCallback* frame_callback, - VideoRenderer* local_renderer, - SendStatisticsProxy* stats_proxy, - CpuOveruseObserver* overuse_observer, - EncodingTimeObserver* encoding_time_observer) - : module_process_thread_(module_process_thread), - frame_callback_(frame_callback), +VideoCaptureInput::VideoCaptureInput(VideoCaptureCallback* frame_callback, + VideoRenderer* local_renderer, + SendStatisticsProxy* stats_proxy, + OveruseFrameDetector* overuse_detector) + : frame_callback_(frame_callback), local_renderer_(local_renderer), stats_proxy_(stats_proxy), encoder_thread_(EncoderThreadFunction, this, "EncoderThread"), @@ -45,19 +40,12 @@ VideoCaptureInput::VideoCaptureInput( delta_ntp_internal_ms_( Clock::GetRealTimeClock()->CurrentNtpInMilliseconds() - TickTime::MillisecondTimestamp()), - overuse_detector_(new OveruseFrameDetector(Clock::GetRealTimeClock(), - CpuOveruseOptions(), - overuse_observer, - stats_proxy)), - encoding_time_observer_(encoding_time_observer) { + overuse_detector_(overuse_detector) { encoder_thread_.Start(); encoder_thread_.SetPriority(rtc::kHighPriority); - module_process_thread_->RegisterModule(overuse_detector_.get()); } VideoCaptureInput::~VideoCaptureInput() { - module_process_thread_->DeRegisterModule(overuse_detector_.get()); - // Stop the thread. rtc::AtomicOps::ReleaseStore(&stop_, 1); capture_event_.Set(); @@ -105,9 +93,7 @@ void VideoCaptureInput::IncomingCapturedFrame(const VideoFrame& video_frame) { captured_frame_.ShallowCopy(incoming_frame); last_captured_timestamp_ = incoming_frame.ntp_time_ms(); - overuse_detector_->FrameCaptured(captured_frame_.width(), - captured_frame_.height(), - captured_frame_.render_time_ms()); + overuse_detector_->FrameCaptured(captured_frame_); TRACE_EVENT_ASYNC_BEGIN1("webrtc", "Video", video_frame.render_time_ms(), "render_time", video_frame.render_time_ms()); @@ -121,12 +107,10 @@ bool VideoCaptureInput::EncoderThreadFunction(void* obj) { bool VideoCaptureInput::EncoderProcess() { static const int kThreadWaitTimeMs = 100; - int64_t capture_time = -1; if (capture_event_.Wait(kThreadWaitTimeMs)) { if (rtc::AtomicOps::AcquireLoad(&stop_)) return false; - int64_t encode_start_time = -1; VideoFrame deliver_frame; { rtc::CritScope lock(&crit_); @@ -136,24 +120,8 @@ bool VideoCaptureInput::EncoderProcess() { } } if (!deliver_frame.IsZeroSize()) { - capture_time = deliver_frame.render_time_ms(); - encode_start_time = Clock::GetRealTimeClock()->TimeInMilliseconds(); frame_callback_->DeliverFrame(deliver_frame); } - // Update the overuse detector with the duration. - if (encode_start_time != -1) { - int encode_time_ms = static_cast( - Clock::GetRealTimeClock()->TimeInMilliseconds() - encode_start_time); - stats_proxy_->OnEncodedFrame(encode_time_ms); - if (encoding_time_observer_) { - encoding_time_observer_->OnReportEncodedTime( - deliver_frame.ntp_time_ms(), encode_time_ms); - } - } - } - // We're done! - if (capture_time != -1) { - overuse_detector_->FrameSent(capture_time); } return true; } diff --git a/webrtc/video/video_capture_input.h b/webrtc/video/video_capture_input.h index 1c27cb0b7b..87b6452538 100644 --- a/webrtc/video/video_capture_input.h +++ b/webrtc/video/video_capture_input.h @@ -16,7 +16,6 @@ #include "webrtc/base/criticalsection.h" #include "webrtc/base/event.h" #include "webrtc/base/platform_thread.h" -#include "webrtc/base/scoped_ptr.h" #include "webrtc/base/thread_annotations.h" #include "webrtc/common_types.h" #include "webrtc/engine_configurations.h" @@ -30,11 +29,7 @@ namespace webrtc { class Config; -class CpuOveruseMetricsObserver; -class CpuOveruseObserver; class OveruseFrameDetector; -class ProcessThread; -class RegistrableCpuOveruseMetricsObserver; class SendStatisticsProxy; class VideoRenderer; @@ -48,12 +43,10 @@ class VideoCaptureCallback { namespace internal { class VideoCaptureInput : public webrtc::VideoCaptureInput { public: - VideoCaptureInput(ProcessThread* module_process_thread, - VideoCaptureCallback* frame_callback, + VideoCaptureInput(VideoCaptureCallback* frame_callback, VideoRenderer* local_renderer, SendStatisticsProxy* send_stats_proxy, - CpuOveruseObserver* overuse_observer, - EncodingTimeObserver* encoding_time_observer); + OveruseFrameDetector* overuse_detector); ~VideoCaptureInput(); void IncomingCapturedFrame(const VideoFrame& video_frame) override; @@ -64,7 +57,6 @@ class VideoCaptureInput : public webrtc::VideoCaptureInput { bool EncoderProcess(); rtc::CriticalSection crit_; - ProcessThread* const module_process_thread_; VideoCaptureCallback* const frame_callback_; VideoRenderer* const local_renderer_; @@ -81,8 +73,7 @@ class VideoCaptureInput : public webrtc::VideoCaptureInput { // Delta used for translating between NTP and internal timestamps. const int64_t delta_ntp_internal_ms_; - rtc::scoped_ptr overuse_detector_; - EncodingTimeObserver* const encoding_time_observer_; + OveruseFrameDetector* const overuse_detector_; }; } // namespace internal diff --git a/webrtc/video/video_capture_input_unittest.cc b/webrtc/video/video_capture_input_unittest.cc index 3fe6e3aeeb..ff1194b3cb 100644 --- a/webrtc/video/video_capture_input_unittest.cc +++ b/webrtc/video/video_capture_input_unittest.cc @@ -15,7 +15,6 @@ #include "testing/gtest/include/gtest/gtest.h" #include "webrtc/base/event.h" #include "webrtc/base/scoped_ptr.h" -#include "webrtc/modules/utility/include/mock/mock_process_thread.h" #include "webrtc/system_wrappers/include/ref_count.h" #include "webrtc/system_wrappers/include/scoped_vector.h" #include "webrtc/test/fake_texture_frame.h" @@ -47,28 +46,23 @@ VideoFrame* CreateVideoFrame(uint8_t length); class VideoCaptureInputTest : public ::testing::Test { protected: VideoCaptureInputTest() - : mock_process_thread_(new NiceMock), - mock_frame_callback_(new NiceMock), - output_frame_event_(false, false), - stats_proxy_(Clock::GetRealTimeClock(), + : stats_proxy_(Clock::GetRealTimeClock(), webrtc::VideoSendStream::Config(nullptr), - webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo) {} + webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo), + mock_frame_callback_(new NiceMock), + output_frame_event_(false, false) {} virtual void SetUp() { EXPECT_CALL(*mock_frame_callback_, DeliverFrame(_)) .WillRepeatedly( WithArg<0>(Invoke(this, &VideoCaptureInputTest::AddOutputFrame))); - input_.reset(new internal::VideoCaptureInput( - mock_process_thread_.get(), mock_frame_callback_.get(), nullptr, - &stats_proxy_, nullptr, nullptr)); - } - - virtual void TearDown() { - // VideoCaptureInput accesses |mock_process_thread_| in destructor and - // should - // be deleted first. - input_.reset(); + overuse_detector_.reset( + new OveruseFrameDetector(Clock::GetRealTimeClock(), CpuOveruseOptions(), + nullptr, nullptr, &stats_proxy_)); + input_.reset(new internal::VideoCaptureInput(mock_frame_callback_.get(), + nullptr, &stats_proxy_, + overuse_detector_.get())); } void AddInputFrame(VideoFrame* frame) { @@ -86,9 +80,12 @@ class VideoCaptureInputTest : public ::testing::Test { EXPECT_TRUE(output_frame_event_.Wait(FRAME_TIMEOUT_MS)); } - rtc::scoped_ptr mock_process_thread_; + SendStatisticsProxy stats_proxy_; + rtc::scoped_ptr mock_frame_callback_; + rtc::scoped_ptr overuse_detector_; + // Used to send input capture frames to VideoCaptureInput. rtc::scoped_ptr input_; @@ -104,7 +101,6 @@ class VideoCaptureInputTest : public ::testing::Test { // The pointers of Y plane buffers of output frames. This is used to verify // the frame are swapped and not copied. std::vector output_frame_ybuffers_; - SendStatisticsProxy stats_proxy_; }; TEST_F(VideoCaptureInputTest, DoesNotRetainHandleNorCopyBuffer) { diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index 73cc1a64dc..2ebbde987c 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -46,8 +46,7 @@ class VideoAnalyzer : public PacketReceiver, public Transport, public VideoRenderer, public VideoCaptureInput, - public EncodedFrameObserver, - public EncodingTimeObserver { + public EncodedFrameObserver { public: VideoAnalyzer(test::LayerFilteringTransport* transport, const std::string& test_label, @@ -65,6 +64,7 @@ class VideoAnalyzer : public PacketReceiver, graph_data_output_file_(graph_data_output_file), graph_title_(graph_title), ssrc_to_analyze_(ssrc_to_analyze), + encode_timing_proxy_(this), frames_to_process_(duration_frames), frames_recorded_(0), frames_processed_(0), @@ -129,8 +129,7 @@ class VideoAnalyzer : public PacketReceiver, return receiver_->DeliverPacket(media_type, packet, length, packet_time); } - // EncodingTimeObserver. - void OnReportEncodedTime(int64_t ntp_time_ms, int encode_time_ms) override { + void MeasuredEncodeTiming(int64_t ntp_time_ms, int encode_time_ms) { rtc::CritScope crit(&comparison_lock_); samples_encode_time_ms_[ntp_time_ms] = encode_time_ms; } @@ -208,7 +207,7 @@ class VideoAnalyzer : public PacketReceiver, assert(!reference_frame.IsZeroSize()); if (send_timestamp == reference_frame.timestamp() - 1) { // TODO(ivica): Make this work for > 2 streams. - // Look at rtp_sender.c:RTPSender::BuildRTPHeader. + // Look at RTPSender::BuildRTPHeader. ++send_timestamp; } EXPECT_EQ(reference_frame.timestamp(), send_timestamp); @@ -263,6 +262,8 @@ class VideoAnalyzer : public PacketReceiver, stats_polling_thread_.Stop(); } + EncodedFrameObserver* encode_timing_proxy() { return &encode_timing_proxy_; } + VideoCaptureInput* input_; test::LayerFilteringTransport* const transport_; PacketReceiver* receiver_; @@ -329,6 +330,21 @@ class VideoAnalyzer : public PacketReceiver, double ssim; }; + // This class receives the send-side OnEncodeTiming and is provided to not + // conflict with the receiver-side pre_decode_callback. + class OnEncodeTimingProxy : public EncodedFrameObserver { + public: + explicit OnEncodeTimingProxy(VideoAnalyzer* parent) : parent_(parent) {} + + void OnEncodeTiming(int64_t ntp_time_ms, int encode_time_ms) override { + parent_->MeasuredEncodeTiming(ntp_time_ms, encode_time_ms); + } + void EncodedFrameCallback(const EncodedFrame& frame) override {} + + private: + VideoAnalyzer* const parent_; + }; + void AddFrameComparison(const VideoFrame& reference, const VideoFrame& render, bool dropped, @@ -566,6 +582,7 @@ class VideoAnalyzer : public PacketReceiver, FILE* const graph_data_output_file_; const std::string graph_title_; const uint32_t ssrc_to_analyze_; + OnEncodeTimingProxy encode_timing_proxy_; std::vector samples_ GUARDED_BY(comparison_lock_); std::map samples_encode_time_ms_ GUARDED_BY(comparison_lock_); test::Statistics sender_time_ GUARDED_BY(comparison_lock_); @@ -976,10 +993,11 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { recv_transport.SetReceiver(sender_call_->Receiver()); SetupCommon(&analyzer, &recv_transport); - video_send_config_.encoding_time_observer = &analyzer; video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer; for (auto& config : video_receive_configs_) config.pre_decode_callback = &analyzer; + RTC_DCHECK(!video_send_config_.post_encode_callback); + video_send_config_.post_encode_callback = analyzer.encode_timing_proxy(); if (params_.screenshare.enabled) SetupScreenshare(); diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc index 08dcd77ac6..254805ff23 100644 --- a/webrtc/video/video_send_stream.cc +++ b/webrtc/video/video_send_stream.cc @@ -22,6 +22,7 @@ #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h" #include "webrtc/modules/pacing/packet_router.h" +#include "webrtc/modules/utility/include/process_thread.h" #include "webrtc/video/call_stats.h" #include "webrtc/video/encoder_state_feedback.h" #include "webrtc/video/payload_router.h" @@ -107,6 +108,18 @@ std::string VideoSendStream::Config::ToString() const { return ss.str(); } +namespace { + +CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { + CpuOveruseOptions options; + if (full_overuse_time) { + options.low_encode_usage_threshold_percent = 100; + options.high_encode_usage_threshold_percent = 120; + } + return options; +} +} // namespace + namespace internal { VideoSendStream::VideoSendStream( int num_cpu_cores, @@ -127,6 +140,12 @@ VideoSendStream::VideoSendStream( module_process_thread_(module_process_thread), call_stats_(call_stats), congestion_controller_(congestion_controller), + overuse_detector_( + Clock::GetRealTimeClock(), + GetCpuOveruseOptions(config.encoder_settings.full_overuse_time), + this, + config.post_encode_callback, + &stats_proxy_), encoder_feedback_(new EncoderStateFeedback()), use_config_bitrate_(true) { LOG(LS_INFO) << "VideoSendStream: " << config_.ToString(); @@ -144,10 +163,10 @@ VideoSendStream::VideoSendStream( const std::vector& ssrcs = config.rtp.ssrcs; - vie_encoder_.reset(new ViEEncoder( - num_cpu_cores, module_process_thread_, &stats_proxy_, - config.pre_encode_callback, congestion_controller_->pacer(), - bitrate_allocator)); + vie_encoder_.reset( + new ViEEncoder(num_cpu_cores, module_process_thread_, &stats_proxy_, + config.pre_encode_callback, &overuse_detector_, + congestion_controller_->pacer(), bitrate_allocator)); RTC_CHECK(vie_encoder_->Init()); vie_channel_.reset(new ViEChannel( @@ -207,8 +226,8 @@ VideoSendStream::VideoSendStream( vie_channel_->SetRTCPCName(config_.rtp.c_name.c_str()); input_.reset(new internal::VideoCaptureInput( - module_process_thread_, vie_encoder_.get(), config_.local_renderer, - &stats_proxy_, this, config_.encoding_time_observer)); + vie_encoder_.get(), config_.local_renderer, &stats_proxy_, + &overuse_detector_)); // 28 to match packet overhead in ModuleRtpRtcpImpl. RTC_DCHECK_LE(config_.rtp.max_packet_size, static_cast(0xFFFF - 28)); @@ -240,10 +259,13 @@ VideoSendStream::VideoSendStream( vie_channel_->RegisterRtcpPacketTypeCounterObserver(&stats_proxy_); vie_channel_->RegisterSendBitrateObserver(&stats_proxy_); vie_channel_->RegisterSendFrameCountObserver(&stats_proxy_); + + module_process_thread_->RegisterModule(&overuse_detector_); } VideoSendStream::~VideoSendStream() { LOG(LS_INFO) << "~VideoSendStream: " << config_.ToString(); + module_process_thread_->DeRegisterModule(&overuse_detector_); vie_channel_->RegisterSendFrameCountObserver(nullptr); vie_channel_->RegisterSendBitrateObserver(nullptr); vie_channel_->RegisterRtcpPacketTypeCounterObserver(nullptr); diff --git a/webrtc/video/video_send_stream.h b/webrtc/video/video_send_stream.h index e2911cc103..acd36ec363 100644 --- a/webrtc/video/video_send_stream.h +++ b/webrtc/video/video_send_stream.h @@ -86,6 +86,7 @@ class VideoSendStream : public webrtc::VideoSendStream, CallStats* const call_stats_; CongestionController* const congestion_controller_; + OveruseFrameDetector overuse_detector_; rtc::scoped_ptr input_; rtc::scoped_ptr vie_channel_; rtc::scoped_ptr vie_encoder_; diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 8adf4d0671..a9b2abbcab 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -27,12 +27,13 @@ #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/include/video_coding.h" #include "webrtc/modules/video_coding/include/video_coding_defines.h" -#include "webrtc/modules/video_coding/encoded_frame.h" #include "webrtc/system_wrappers/include/clock.h" #include "webrtc/system_wrappers/include/metrics.h" #include "webrtc/system_wrappers/include/tick_util.h" +#include "webrtc/video/overuse_frame_detector.h" #include "webrtc/video/payload_router.h" #include "webrtc/video/send_statistics_proxy.h" +#include "webrtc/video_frame.h" namespace webrtc { @@ -107,6 +108,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, ProcessThread* module_process_thread, SendStatisticsProxy* stats_proxy, I420FrameCallback* pre_encode_callback, + OveruseFrameDetector* overuse_detector, PacedSender* pacer, BitrateAllocator* bitrate_allocator) : number_of_cores_(number_of_cores), @@ -118,6 +120,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, send_payload_router_(NULL), stats_proxy_(stats_proxy), pre_encode_callback_(pre_encode_callback), + overuse_detector_(overuse_detector), pacer_(pacer), bitrate_allocator_(bitrate_allocator), time_of_last_frame_activity_ms_(0), @@ -461,12 +464,12 @@ int32_t ViEEncoder::SendData(const uint8_t payload_type, if (stats_proxy_ != NULL) stats_proxy_->OnSendEncodedImage(encoded_image, rtp_video_hdr); - return send_payload_router_->RoutePayload( - encoded_image._frameType, payload_type, encoded_image._timeStamp, - encoded_image.capture_time_ms_, encoded_image._buffer, - encoded_image._length, fragmentation_header, rtp_video_hdr) - ? 0 - : -1; + bool success = send_payload_router_->RoutePayload( + encoded_image._frameType, payload_type, encoded_image._timeStamp, + encoded_image.capture_time_ms_, encoded_image._buffer, + encoded_image._length, fragmentation_header, rtp_video_hdr); + overuse_detector_->FrameSent(encoded_image._timeStamp); + return success ? 0 : -1; } void ViEEncoder::OnEncoderImplementationName( diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index e36cb30adc..b8944dc6f8 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -33,6 +33,7 @@ class BitrateAllocator; class BitrateObserver; class Config; class EncodedImageCallback; +class OveruseFrameDetector; class PacedSender; class PayloadRouter; class ProcessThread; @@ -54,6 +55,7 @@ class ViEEncoder : public RtcpIntraFrameObserver, ProcessThread* module_process_thread, SendStatisticsProxy* stats_proxy, I420FrameCallback* pre_encode_callback, + OveruseFrameDetector* overuse_detector, PacedSender* pacer, BitrateAllocator* bitrate_allocator); ~ViEEncoder(); @@ -163,6 +165,7 @@ class ViEEncoder : public RtcpIntraFrameObserver, SendStatisticsProxy* const stats_proxy_; I420FrameCallback* const pre_encode_callback_; + OveruseFrameDetector* const overuse_detector_; PacedSender* const pacer_; BitrateAllocator* const bitrate_allocator_; diff --git a/webrtc/video_send_stream.h b/webrtc/video_send_stream.h index 83a96d3fed..63acb0944b 100644 --- a/webrtc/video_send_stream.h +++ b/webrtc/video_send_stream.h @@ -26,13 +26,6 @@ namespace webrtc { class LoadObserver; class VideoEncoder; -class EncodingTimeObserver { - public: - virtual ~EncodingTimeObserver() {} - - virtual void OnReportEncodedTime(int64_t ntp_time_ms, int encode_time_ms) = 0; -}; - // Class to deliver captured frame to the video send stream. class VideoCaptureInput { public: @@ -91,6 +84,11 @@ class VideoSendStream : public SendStream { // sources anymore. bool internal_source = false; + // Allow 100% encoder utilization. Used for HW encoders where CPU isn't + // expected to be the limiting factor, but a chip could be running at + // 30fps (for example) exactly. + bool full_overuse_time = false; + // Uninitialized VideoEncoder instance to be used for encoding. Will be // initialized from inside the VideoSendStream. VideoEncoder* encoder = nullptr; @@ -144,7 +142,9 @@ class VideoSendStream : public SendStream { I420FrameCallback* pre_encode_callback = nullptr; // Called for each encoded frame, e.g. used for file storage. 'nullptr' - // disables the callback. + // disables the callback. Also measures timing and passes the time + // spent on encoding. This timing will not fire if encoding takes longer + // than the measuring window, since the sample data will have been dropped. EncodedFrameObserver* post_encode_callback = nullptr; // Renderer for local preview. The local renderer will be called even if @@ -164,11 +164,6 @@ class VideoSendStream : public SendStream { // below the minimum configured bitrate. If this variable is false, the // stream may send at a rate higher than the estimated available bitrate. bool suspend_below_min_bitrate = false; - - // Called for each encoded frame. Passes the total time spent on encoding. - // TODO(ivica): Consolidate with post_encode_callback: - // https://code.google.com/p/webrtc/issues/detail?id=5042 - EncodingTimeObserver* encoding_time_observer = nullptr; }; // Gets interface used to insert captured frames. Valid as long as the