admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Measure encoding time on encode callbacks.

Browse Source 
Permits measuring encoding time even when performed on another thread,
typically for hardware encoding, instead of assuming that encoding is
blocking the calling thread.

Permitted encoding time is increased for hardware encoders since they
can be timed to keep 30fps, for instance, without indicating overload.

Merges EncodingTimeObserver into EncodedFrameObserver to have one post-encode
callback.

BUG=webrtc:5042, webrtc:5132
R=asapersson@webrtc.org, mflodman@webrtc.org

Review URL: https://codereview.webrtc.org/1569853002 .

Cr-Commit-Position: refs/heads/master@{#11499}
This commit is contained in:
Peter Boström 2016-02-05 11:13:28 +01:00
parent 8e8908aadd
commit e449915455
21 changed files with 353 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webrtc/frame_callback.h
View File

@ -12,6 +12,7 @@
#define WEBRTC_FRAME_CALLBACK_H_
#include <stddef.h>
#include <stdint.h>
#include "webrtc/common_types.h"
@ -43,6 +44,7 @@ class I420FrameCallback {
class EncodedFrameObserver {
public:
virtual void EncodedFrameCallback(const EncodedFrame& encoded_frame) = 0;
virtual void OnEncodeTiming(int64_t capture_ntp_ms, int encode_duration_ms) {}
protected:
virtual ~EncodedFrameObserver() {}

1
webrtc/media/webrtc/webrtcvideoengine2.cc
View File

@ -1768,6 +1768,7 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions(
AllocatedEncoder new_encoder = CreateVideoEncoder(codec_settings.codec);
parameters_.config.encoder_settings.encoder = new_encoder.encoder;
parameters_.config.encoder_settings.full_overuse_time = new_encoder.external;
parameters_.config.encoder_settings.payload_name = codec_settings.codec.name;
parameters_.config.encoder_settings.payload_type = codec_settings.codec.id;
if (new_encoder.external) {

38
webrtc/media/webrtc/webrtcvideoengine2_unittest.cc
View File

@ -153,6 +153,8 @@ class WebRtcVideoEngine2Test : public ::testing::Test {
cricket::WebRtcVideoDecoderFactory* decoder_factory,
const std::vector<VideoCodec>& codecs);
void TestExtendedEncoderOveruse(bool use_external_encoder);
webrtc::test::ScopedFieldTrials override_field_trials_;
// Used in WebRtcVideoEngine2VoiceTest, but defined here so it's properly
// initialized when the constructor is called.
@ -356,6 +358,42 @@ TEST_F(WebRtcVideoEngine2Test, UseExternalFactoryForVp8WhenSupported) {
EXPECT_EQ(0u, encoder_factory.encoders().size());
}
void WebRtcVideoEngine2Test::TestExtendedEncoderOveruse(
bool use_external_encoder) {
cricket::FakeWebRtcVideoEncoderFactory encoder_factory;
encoder_factory.AddSupportedVideoCodecType(webrtc::kVideoCodecVP8, "VP8");
cricket::VideoSendParameters parameters;
parameters.codecs.push_back(kVp8Codec);
rtc::scoped_ptr<VideoMediaChannel> channel;
FakeCall* fake_call = new FakeCall(webrtc::Call::Config());
call_.reset(fake_call);
if (use_external_encoder) {
channel.reset(
SetUpForExternalEncoderFactory(&encoder_factory, parameters.codecs));
} else {
engine_.Init();
channel.reset(engine_.CreateChannel(call_.get(), cricket::VideoOptions()));
}
ASSERT_TRUE(
channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc)));
EXPECT_TRUE(channel->SetSendParameters(parameters));
EXPECT_TRUE(channel->SetSend(true));
FakeVideoSendStream* stream = fake_call->GetVideoSendStreams()[0];
EXPECT_EQ(use_external_encoder,
stream->GetConfig().encoder_settings.full_overuse_time);
// Remove stream previously added to free the external encoder instance.
EXPECT_TRUE(channel->RemoveSendStream(kSsrc));
}
TEST_F(WebRtcVideoEngine2Test, EnablesFullEncoderTimeForExternalEncoders) {
TestExtendedEncoderOveruse(true);
}
TEST_F(WebRtcVideoEngine2Test, DisablesFullEncoderTimeForNonExternalEncoders) {
TestExtendedEncoderOveruse(false);
}
TEST_F(WebRtcVideoEngine2Test, CanConstructDecoderForVp9EncoderFactory) {
cricket::FakeWebRtcVideoEncoderFactory encoder_factory;
encoder_factory.AddSupportedVideoCodecType(webrtc::kVideoCodecVP9, "VP9");

2
webrtc/modules/video_coding/video_coding_impl.h
View File

@ -31,8 +31,6 @@
namespace webrtc {
class EncodedFrameObserver;
namespace vcm {
class VCMProcessTimer {

8
webrtc/video/encoder_state_feedback_unittest.cc
View File

@ -31,7 +31,13 @@ namespace webrtc {
class MockVieEncoder : public ViEEncoder {
public:
explicit MockVieEncoder(ProcessThread* process_thread, PacedSender* pacer)
: ViEEncoder(1, process_thread, nullptr, nullptr, pacer, nullptr) {}
: ViEEncoder(1,
process_thread,
nullptr,
nullptr,
nullptr,
pacer,
nullptr) {}
~MockVieEncoder() {}
MOCK_METHOD1(OnReceivedIntraFrameRequest,

2
webrtc/video/end_to_end_tests.cc
View File

@ -2653,7 +2653,7 @@ TEST_F(EndToEndTest, GetStats) {
stats.substreams.size() == expected_send_ssrcs_.size();
send_stats_filled_["CpuOveruseMetrics"] |=
stats.avg_encode_time_ms != 0 || stats.encode_usage_percent != 0;
stats.avg_encode_time_ms != 0 && stats.encode_usage_percent != 0;
send_stats_filled_["EncoderImplementationName"] |=
stats.encoder_implementation_name ==

191
webrtc/video/overuse_frame_detector.cc
View File

@ -20,7 +20,9 @@
#include "webrtc/base/checks.h"
#include "webrtc/base/exp_filter.h"
#include "webrtc/base/logging.h"
#include "webrtc/frame_callback.h"
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc/video_frame.h"
namespace webrtc {
@ -115,106 +117,47 @@ class OveruseFrameDetector::SendProcessingUsage {
rtc::scoped_ptr<rtc::ExpFilter> filtered_frame_diff_ms_;
};
// Class for calculating the processing time of frames.
class OveruseFrameDetector::FrameQueue {
public:
FrameQueue() : last_processing_time_ms_(-1) {}
~FrameQueue() {}
// Called when a frame is captured.
// Starts the measuring of the processing time of the frame.
void Start(int64_t capture_time, int64_t now) {
const size_t kMaxSize = 90; // Allows for processing time of 1.5s at 60fps.
if (frame_times_.size() > kMaxSize) {
LOG(LS_WARNING) << "Max size reached, removed oldest frame.";
frame_times_.erase(frame_times_.begin());
}
if (frame_times_.find(capture_time) != frame_times_.end()) {
// Frame should not exist.
assert(false);
return;
}
frame_times_[capture_time] = now;
}
// Called when the processing of a frame has finished.
// Returns the processing time of the frame.
int End(int64_t capture_time, int64_t now) {
std::map<int64_t, int64_t>::iterator it = frame_times_.find(capture_time);
if (it == frame_times_.end()) {
return -1;
}
// Remove any old frames up to current.
// Old frames have been skipped by the capture process thread.
// TODO(asapersson): Consider measuring time from first frame in list.
last_processing_time_ms_ = now - (*it).second;
frame_times_.erase(frame_times_.begin(), ++it);
return last_processing_time_ms_;
}
void Reset() { frame_times_.clear(); }
int NumFrames() const { return static_cast<int>(frame_times_.size()); }
int last_processing_time_ms() const { return last_processing_time_ms_; }
private:
// Captured frames mapped by the capture time.
std::map<int64_t, int64_t> frame_times_;
int last_processing_time_ms_;
};
OveruseFrameDetector::OveruseFrameDetector(
Clock* clock,
const CpuOveruseOptions& options,
CpuOveruseObserver* observer,
EncodedFrameObserver* encoder_timing,
CpuOveruseMetricsObserver* metrics_observer)
: options_(options),
observer_(observer),
encoder_timing_(encoder_timing),
metrics_observer_(metrics_observer),
clock_(clock),
num_process_times_(0),
last_capture_time_(0),
last_capture_time_ms_(-1),
last_processed_capture_time_ms_(-1),
num_pixels_(0),
next_process_time_(clock_->TimeInMilliseconds()),
last_overuse_time_(0),
next_process_time_ms_(clock_->TimeInMilliseconds()),
last_overuse_time_ms_(-1),
checks_above_threshold_(0),
num_overuse_detections_(0),
last_rampup_time_(0),
last_rampup_time_ms_(-1),
in_quick_rampup_(false),
current_rampup_delay_ms_(kStandardRampUpDelayMs),
last_sample_time_ms_(0),
usage_(new SendProcessingUsage(options)),
frame_queue_(new FrameQueue()) {
usage_(new SendProcessingUsage(options)) {
RTC_DCHECK(metrics_observer != nullptr);
// Make sure stats are initially up-to-date. This simplifies unit testing
// since we don't have to trigger an update using one of the methods which
// would also alter the overuse state.
UpdateCpuOveruseMetrics();
processing_thread_.DetachFromThread();
}
OveruseFrameDetector::~OveruseFrameDetector() {
}
int OveruseFrameDetector::LastProcessingTimeMs() const {
rtc::CritScope cs(&crit_);
return frame_queue_->last_processing_time_ms();
}
void OveruseFrameDetector::EncodedFrameTimeMeasured(int encode_duration_ms) {
if (!metrics_)
metrics_ = rtc::Optional<CpuOveruseMetrics>(CpuOveruseMetrics());
metrics_->encode_usage_percent = usage_->Value();
int OveruseFrameDetector::FramesInQueue() const {
rtc::CritScope cs(&crit_);
return frame_queue_->NumFrames();
}
void OveruseFrameDetector::UpdateCpuOveruseMetrics() {
metrics_.encode_usage_percent = usage_->Value();
metrics_observer_->CpuOveruseMetricsUpdated(metrics_);
metrics_observer_->OnEncodedFrameTimeMeasured(encode_duration_ms, *metrics_);
}
int64_t OveruseFrameDetector::TimeUntilNextProcess() {
RTC_DCHECK(processing_thread_.CalledOnValidThread());
return next_process_time_ - clock_->TimeInMilliseconds();
return next_process_time_ms_ - clock_->TimeInMilliseconds();
}
bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const {
@ -225,56 +168,80 @@ bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const {
}
bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now) const {
if (last_capture_time_ == 0) {
if (last_capture_time_ms_ == -1)
return false;
}
return (now - last_capture_time_) > options_.frame_timeout_interval_ms;
return (now - last_capture_time_ms_) > options_.frame_timeout_interval_ms;
}
void OveruseFrameDetector::ResetAll(int num_pixels) {
num_pixels_ = num_pixels;
usage_->Reset();
frame_queue_->Reset();
last_capture_time_ = 0;
frame_timing_.clear();
last_capture_time_ms_ = -1;
last_processed_capture_time_ms_ = -1;
num_process_times_ = 0;
UpdateCpuOveruseMetrics();
metrics_ = rtc::Optional<CpuOveruseMetrics>();
}
void OveruseFrameDetector::FrameCaptured(int width,
int height,
int64_t capture_time_ms) {
void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame) {
rtc::CritScope cs(&crit_);
int64_t now = clock_->TimeInMilliseconds();
if (FrameSizeChanged(width * height) || FrameTimeoutDetected(now)) {
ResetAll(width * height);
if (FrameSizeChanged(frame.width() * frame.height()) ||
FrameTimeoutDetected(now)) {
ResetAll(frame.width() * frame.height());
}
if (last_capture_time_ != 0)
usage_->AddCaptureSample(now - last_capture_time_);
if (last_capture_time_ms_ != -1)
usage_->AddCaptureSample(now - last_capture_time_ms_);
last_capture_time_ = now;
last_capture_time_ms_ = now;
frame_queue_->Start(capture_time_ms, now);
frame_timing_.push_back(
FrameTiming(frame.ntp_time_ms(), frame.timestamp(), now));
}
void OveruseFrameDetector::FrameSent(int64_t capture_time_ms) {
void OveruseFrameDetector::FrameSent(uint32_t timestamp) {
rtc::CritScope cs(&crit_);
int delay_ms = frame_queue_->End(capture_time_ms,
clock_->TimeInMilliseconds());
if (delay_ms > 0) {
AddProcessingTime(delay_ms);
}
}
void OveruseFrameDetector::AddProcessingTime(int elapsed_ms) {
// Delay before reporting actual encoding time, used to have the ability to
// detect total encoding time when encoding more than one layer. Encoding is
// here assumed to finish within a second (or that we get enough long-time
// samples before one second to trigger an overuse even when this is not the
// case).
static const int64_t kEncodingTimeMeasureWindowMs = 1000;
int64_t now = clock_->TimeInMilliseconds();
if (last_sample_time_ms_ != 0) {
int64_t diff_ms = now - last_sample_time_ms_;
usage_->AddSample(elapsed_ms, diff_ms);
for (auto& it : frame_timing_) {
if (it.timestamp == timestamp) {
it.last_send_ms = now;
break;
}
}
// TODO(pbos): Handle the case/log errors when not finding the corresponding
// frame (either very slow encoding or incorrect wrong timestamps returned
// from the encoder).
// This is currently the case for all frames on ChromeOS, so logging them
// would be spammy, and triggering overuse would be wrong.
// https://crbug.com/350106
while (!frame_timing_.empty()) {
FrameTiming timing = frame_timing_.front();
if (now - timing.capture_ms < kEncodingTimeMeasureWindowMs)
break;
if (timing.last_send_ms != -1) {
int encode_duration_ms =
static_cast<int>(timing.last_send_ms - timing.capture_ms);
if (encoder_timing_) {
encoder_timing_->OnEncodeTiming(timing.capture_ntp_ms,
encode_duration_ms);
}
if (last_processed_capture_time_ms_ != -1) {
int64_t diff_ms = timing.capture_ms - last_processed_capture_time_ms_;
usage_->AddSample(encode_duration_ms, diff_ms);
}
last_processed_capture_time_ms_ = timing.capture_ms;
EncodedFrameTimeMeasured(encode_duration_ms);
}
frame_timing_.pop_front();
}
last_sample_time_ms_ = now;
UpdateCpuOveruseMetrics();
}
int32_t OveruseFrameDetector::Process() {
@ -283,28 +250,28 @@ int32_t OveruseFrameDetector::Process() {
int64_t now = clock_->TimeInMilliseconds();
// Used to protect against Process() being called too often.
if (now < next_process_time_)
if (now < next_process_time_ms_)
return 0;
next_process_time_ = now + kProcessIntervalMs;
next_process_time_ms_ = now + kProcessIntervalMs;
CpuOveruseMetrics current_metrics;
{
rtc::CritScope cs(&crit_);
++num_process_times_;
current_metrics = metrics_;
if (num_process_times_ <= options_.min_process_count)
if (num_process_times_ <= options_.min_process_count || !metrics_)
return 0;
current_metrics = *metrics_;
}
if (IsOverusing(current_metrics)) {
// If the last thing we did was going up, and now have to back down, we need
// to check if this peak was short. If so we should back off to avoid going
// back and forth between this load, the system doesn't seem to handle it.
bool check_for_backoff = last_rampup_time_ > last_overuse_time_;
bool check_for_backoff = last_rampup_time_ms_ > last_overuse_time_ms_;
if (check_for_backoff) {
if (now - last_rampup_time_ < kStandardRampUpDelayMs ||
if (now - last_rampup_time_ms_ < kStandardRampUpDelayMs ||
num_overuse_detections_ > kMaxOverusesBeforeApplyRampupDelay) {
// Going up was not ok for very long, back off.
current_rampup_delay_ms_ *= kRampUpBackoffFactor;
@ -316,7 +283,7 @@ int32_t OveruseFrameDetector::Process() {
}
}
last_overuse_time_ = now;
last_overuse_time_ms_ = now;
in_quick_rampup_ = false;
checks_above_threshold_ = 0;
++num_overuse_detections_;
@ -324,7 +291,7 @@ int32_t OveruseFrameDetector::Process() {
if (observer_ != NULL)
observer_->OveruseDetected();
} else if (IsUnderusing(current_metrics, now)) {
last_rampup_time_ = now;
last_rampup_time_ms_ = now;
in_quick_rampup_ = true;
if (observer_ != NULL)
@ -355,7 +322,7 @@ bool OveruseFrameDetector::IsOverusing(const CpuOveruseMetrics& metrics) {
bool OveruseFrameDetector::IsUnderusing(const CpuOveruseMetrics& metrics,
int64_t time_now) {
int delay = in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
if (time_now < last_rampup_time_ + delay)
if (time_now < last_rampup_time_ms_ + delay)
return false;
return metrics.encode_usage_percent <

53
webrtc/video/overuse_frame_detector.h
View File

@ -11,8 +11,11 @@
#ifndef WEBRTC_VIDEO_OVERUSE_FRAME_DETECTOR_H_
#define WEBRTC_VIDEO_OVERUSE_FRAME_DETECTOR_H_
#include <list>
#include "webrtc/base/constructormagic.h"
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/optional.h"
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/base/exp_filter.h"
#include "webrtc/base/thread_annotations.h"
@ -22,6 +25,8 @@
namespace webrtc {
class Clock;
class EncodedFrameObserver;
class VideoFrame;
// CpuOveruseObserver is called when a system overuse is detected and
// VideoEngine cannot keep up the encoding frequency.
@ -68,10 +73,10 @@ struct CpuOveruseMetrics {
class CpuOveruseMetricsObserver {
public:
virtual ~CpuOveruseMetricsObserver() {}
virtual void CpuOveruseMetricsUpdated(const CpuOveruseMetrics& metrics) = 0;
virtual void OnEncodedFrameTimeMeasured(int encode_duration_ms,
const CpuOveruseMetrics& metrics) = 0;
};
// Use to detect system overuse based on the send-side processing time of
// incoming frames.
class OveruseFrameDetector : public Module {
@ -79,18 +84,15 @@ class OveruseFrameDetector : public Module {
OveruseFrameDetector(Clock* clock,
const CpuOveruseOptions& options,
CpuOveruseObserver* overuse_observer,
EncodedFrameObserver* encoder_timing_,
CpuOveruseMetricsObserver* metrics_observer);
~OveruseFrameDetector();
// Called for each captured frame.
void FrameCaptured(int width, int height, int64_t capture_time_ms);
void FrameCaptured(const VideoFrame& frame);
// Called for each sent frame.
void FrameSent(int64_t capture_time_ms);
// Only public for testing.
int LastProcessingTimeMs() const;
int FramesInQueue() const;
void FrameSent(uint32_t timestamp);
// Implements Module.
int64_t TimeUntilNextProcess() override;
@ -98,13 +100,20 @@ class OveruseFrameDetector : public Module {
private:
class SendProcessingUsage;
class FrameQueue;
struct FrameTiming {
FrameTiming(int64_t capture_ntp_ms, uint32_t timestamp, int64_t now)
: capture_ntp_ms(capture_ntp_ms),
timestamp(timestamp),
capture_ms(now),
last_send_ms(-1) {}
int64_t capture_ntp_ms;
uint32_t timestamp;
int64_t capture_ms;
int64_t last_send_ms;
};
void UpdateCpuOveruseMetrics() EXCLUSIVE_LOCKS_REQUIRED(crit_);
// TODO(asapersson): This method is only used on one thread, so it shouldn't
// need a guard.
void AddProcessingTime(int elapsed_ms) EXCLUSIVE_LOCKS_REQUIRED(crit_);
void EncodedFrameTimeMeasured(int encode_duration_ms)
EXCLUSIVE_LOCKS_REQUIRED(crit_);
// Only called on the processing thread.
bool IsOverusing(const CpuOveruseMetrics& metrics);
@ -125,34 +134,34 @@ class OveruseFrameDetector : public Module {
// Observer getting overuse reports.
CpuOveruseObserver* const observer_;
EncodedFrameObserver* const encoder_timing_;
// Stats metrics.
CpuOveruseMetricsObserver* const metrics_observer_;
CpuOveruseMetrics metrics_ GUARDED_BY(crit_);
rtc::Optional<CpuOveruseMetrics> metrics_ GUARDED_BY(crit_);
Clock* const clock_;
int64_t num_process_times_ GUARDED_BY(crit_);
int64_t last_capture_time_ GUARDED_BY(crit_);
int64_t last_capture_time_ms_ GUARDED_BY(crit_);
int64_t last_processed_capture_time_ms_ GUARDED_BY(crit_);
// Number of pixels of last captured frame.
int num_pixels_ GUARDED_BY(crit_);
// These seven members are only accessed on the processing thread.
int64_t next_process_time_;
int64_t last_overuse_time_;
int64_t next_process_time_ms_;
int64_t last_overuse_time_ms_;
int checks_above_threshold_;
int num_overuse_detections_;
int64_t last_rampup_time_;
int64_t last_rampup_time_ms_;
bool in_quick_rampup_;
int current_rampup_delay_ms_;
int64_t last_sample_time_ms_; // Only accessed by one thread.
// TODO(asapersson): Can these be regular members (avoid separate heap
// allocs)?
const rtc::scoped_ptr<SendProcessingUsage> usage_ GUARDED_BY(crit_);
const rtc::scoped_ptr<FrameQueue> frame_queue_ GUARDED_BY(crit_);
std::list<FrameTiming> frame_timing_ GUARDED_BY(crit_);
rtc::ThreadChecker processing_thread_;

166
webrtc/video/overuse_frame_detector_unittest.cc
View File

@ -15,6 +15,7 @@
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc/video_frame.h"
namespace webrtc {
namespace {
@ -59,11 +60,12 @@ class OveruseFrameDetectorTest : public ::testing::Test,
}
void ReinitializeOveruseDetector() {
overuse_detector_.reset(new OveruseFrameDetector(clock_.get(), options_,
observer_.get(), this));
overuse_detector_.reset(new OveruseFrameDetector(
clock_.get(), options_, observer_.get(), nullptr, this));
}
void CpuOveruseMetricsUpdated(const CpuOveruseMetrics& metrics) override {
void OnEncodedFrameTimeMeasured(int encode_time_ms,
const CpuOveruseMetrics& metrics) override {
metrics_ = metrics;
}
@ -72,17 +74,31 @@ class OveruseFrameDetectorTest : public ::testing::Test,
options_.high_encode_usage_threshold_percent) / 2.0f) + 0.5;
}
void InsertAndSendFramesWithInterval(
int num_frames, int interval_ms, int width, int height, int delay_ms) {
void InsertAndSendFramesWithInterval(int num_frames,
int interval_ms,
int width,
int height,
int delay_ms) {
VideoFrame frame;
frame.CreateEmptyFrame(width, height, width, width / 2, width / 2);
uint32_t timestamp = 0;
while (num_frames-- > 0) {
int64_t capture_time_ms = clock_->TimeInMilliseconds();
overuse_detector_->FrameCaptured(width, height, capture_time_ms);
frame.set_timestamp(timestamp);
overuse_detector_->FrameCaptured(frame);
clock_->AdvanceTimeMilliseconds(delay_ms);
overuse_detector_->FrameSent(capture_time_ms);
overuse_detector_->FrameSent(timestamp);
clock_->AdvanceTimeMilliseconds(interval_ms - delay_ms);
timestamp += interval_ms * 90;
}
}
void ForceUpdate(int width, int height) {
// Insert one frame, wait a second and then put in another to force update
// the usage. From the tests where these are used, adding another sample
// doesn't affect the expected outcome (this is mainly to check initial
// values and whether the overuse detector has been reset or not).
InsertAndSendFramesWithInterval(2, 1000, width, height, kFrameInterval33ms);
}
void TriggerOveruse(int num_times) {
const int kDelayMs = 32;
for (int i = 0; i < num_times; ++i) {
@ -131,7 +147,7 @@ TEST_F(OveruseFrameDetectorTest, OveruseAndRecover) {
TEST_F(OveruseFrameDetectorTest, OveruseAndRecoverWithNoObserver) {
overuse_detector_.reset(
new OveruseFrameDetector(clock_.get(), options_, nullptr, this));
new OveruseFrameDetector(clock_.get(), options_, nullptr, nullptr, this));
EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(0);
TriggerOveruse(options_.high_threshold_consecutive_count);
EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(0);
@ -149,8 +165,8 @@ TEST_F(OveruseFrameDetectorTest, DoubleOveruseAndRecover) {
TEST_F(OveruseFrameDetectorTest, TriggerUnderuseWithMinProcessCount) {
options_.min_process_count = 1;
CpuOveruseObserverImpl overuse_observer;
overuse_detector_.reset(new OveruseFrameDetector(clock_.get(), options_,
&overuse_observer, this));
overuse_detector_.reset(new OveruseFrameDetector(
clock_.get(), options_, &overuse_observer, nullptr, this));
InsertAndSendFramesWithInterval(
1200, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms);
overuse_detector_->Process();
@ -189,17 +205,18 @@ TEST_F(OveruseFrameDetectorTest, ProcessingUsage) {
}
TEST_F(OveruseFrameDetectorTest, ResetAfterResolutionChange) {
ForceUpdate(kWidth, kHeight);
EXPECT_EQ(InitialUsage(), UsagePercent());
InsertAndSendFramesWithInterval(
1000, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms);
EXPECT_NE(InitialUsage(), UsagePercent());
// Verify reset.
InsertAndSendFramesWithInterval(
1, kFrameInterval33ms, kWidth, kHeight + 1, kProcessTime5ms);
// Verify reset (with new width/height).
ForceUpdate(kWidth, kHeight + 1);
EXPECT_EQ(InitialUsage(), UsagePercent());
}
TEST_F(OveruseFrameDetectorTest, ResetAfterFrameTimeout) {
ForceUpdate(kWidth, kHeight);
EXPECT_EQ(InitialUsage(), UsagePercent());
InsertAndSendFramesWithInterval(
1000, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms);
@ -211,6 +228,7 @@ TEST_F(OveruseFrameDetectorTest, ResetAfterFrameTimeout) {
InsertAndSendFramesWithInterval(
2, options_.frame_timeout_interval_ms + 1, kWidth, kHeight,
kProcessTime5ms);
ForceUpdate(kWidth, kHeight);
EXPECT_EQ(InitialUsage(), UsagePercent());
}
@ -220,91 +238,65 @@ TEST_F(OveruseFrameDetectorTest, MinFrameSamplesBeforeUpdating) {
InsertAndSendFramesWithInterval(
40, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms);
EXPECT_EQ(InitialUsage(), UsagePercent());
// Pass time far enough to digest all previous samples.
clock_->AdvanceTimeMilliseconds(1000);
InsertAndSendFramesWithInterval(1, kFrameInterval33ms, kWidth, kHeight,
kProcessTime5ms);
// The last sample has not been processed here.
EXPECT_EQ(InitialUsage(), UsagePercent());
// Pass time far enough to digest all previous samples, 41 in total.
clock_->AdvanceTimeMilliseconds(1000);
InsertAndSendFramesWithInterval(
1, kFrameInterval33ms, kWidth, kHeight, kProcessTime5ms);
EXPECT_NE(InitialUsage(), UsagePercent());
}
TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) {
ForceUpdate(kWidth, kHeight);
EXPECT_EQ(InitialUsage(), UsagePercent());
}
TEST_F(OveruseFrameDetectorTest, FrameDelay_OneFrame) {
const int kProcessingTimeMs = 100;
overuse_detector_->FrameCaptured(kWidth, kHeight, 33);
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs);
EXPECT_EQ(-1, overuse_detector_->LastProcessingTimeMs());
overuse_detector_->FrameSent(33);
EXPECT_EQ(kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs());
EXPECT_EQ(0, overuse_detector_->FramesInQueue());
}
TEST_F(OveruseFrameDetectorTest, FrameDelay_TwoFrames) {
const int kProcessingTimeMs1 = 100;
const int kProcessingTimeMs2 = 50;
const int kTimeBetweenFramesMs = 200;
overuse_detector_->FrameCaptured(kWidth, kHeight, 33);
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs1);
overuse_detector_->FrameSent(33);
EXPECT_EQ(kProcessingTimeMs1, overuse_detector_->LastProcessingTimeMs());
clock_->AdvanceTimeMilliseconds(kTimeBetweenFramesMs);
overuse_detector_->FrameCaptured(kWidth, kHeight, 66);
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs2);
overuse_detector_->FrameSent(66);
EXPECT_EQ(kProcessingTimeMs2, overuse_detector_->LastProcessingTimeMs());
}
TEST_F(OveruseFrameDetectorTest, FrameDelay_MaxQueueSize) {
const int kMaxQueueSize = 91;
for (int i = 0; i < kMaxQueueSize * 2; ++i) {
overuse_detector_->FrameCaptured(kWidth, kHeight, i);
TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) {
EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(testing::AtLeast(1));
static const int kIntervalMs = 33;
static const size_t kNumFramesEncodingDelay = 3;
VideoFrame frame;
frame.CreateEmptyFrame(kWidth, kHeight, kWidth, kWidth / 2, kWidth / 2);
for (size_t i = 0; i < 1000; ++i) {
// Unique timestamps.
frame.set_timestamp(static_cast<uint32_t>(i));
overuse_detector_->FrameCaptured(frame);
clock_->AdvanceTimeMilliseconds(kIntervalMs);
if (i > kNumFramesEncodingDelay) {
overuse_detector_->FrameSent(
static_cast<uint32_t>(i - kNumFramesEncodingDelay));
}
overuse_detector_->Process();
}
EXPECT_EQ(kMaxQueueSize, overuse_detector_->FramesInQueue());
}
TEST_F(OveruseFrameDetectorTest, FrameDelay_NonProcessedFramesRemoved) {
const int kProcessingTimeMs = 100;
overuse_detector_->FrameCaptured(kWidth, kHeight, 33);
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs);
overuse_detector_->FrameCaptured(kWidth, kHeight, 35);
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs);
overuse_detector_->FrameCaptured(kWidth, kHeight, 66);
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs);
overuse_detector_->FrameCaptured(kWidth, kHeight, 99);
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs);
EXPECT_EQ(-1, overuse_detector_->LastProcessingTimeMs());
EXPECT_EQ(4, overuse_detector_->FramesInQueue());
overuse_detector_->FrameSent(66);
// Frame 33, 35 removed, 66 processed, 99 not processed.
EXPECT_EQ(2 * kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs());
EXPECT_EQ(1, overuse_detector_->FramesInQueue());
overuse_detector_->FrameSent(99);
EXPECT_EQ(kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs());
EXPECT_EQ(0, overuse_detector_->FramesInQueue());
}
TEST_F(OveruseFrameDetectorTest, FrameDelay_ResetClearsFrames) {
const int kProcessingTimeMs = 100;
overuse_detector_->FrameCaptured(kWidth, kHeight, 33);
EXPECT_EQ(1, overuse_detector_->FramesInQueue());
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs);
// Verify reset (resolution changed).
overuse_detector_->FrameCaptured(kWidth, kHeight + 1, 66);
EXPECT_EQ(1, overuse_detector_->FramesInQueue());
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs);
overuse_detector_->FrameSent(66);
EXPECT_EQ(kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs());
EXPECT_EQ(0, overuse_detector_->FramesInQueue());
}
TEST_F(OveruseFrameDetectorTest, FrameDelay_NonMatchingSendFrameIgnored) {
const int kProcessingTimeMs = 100;
overuse_detector_->FrameCaptured(kWidth, kHeight, 33);
clock_->AdvanceTimeMilliseconds(kProcessingTimeMs);
overuse_detector_->FrameSent(34);
EXPECT_EQ(-1, overuse_detector_->LastProcessingTimeMs());
overuse_detector_->FrameSent(33);
EXPECT_EQ(kProcessingTimeMs, overuse_detector_->LastProcessingTimeMs());
TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) {
// >85% encoding time should trigger overuse.
EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(testing::AtLeast(1));
static const int kIntervalMs = 33;
static const int kDelayMs = 30;
VideoFrame frame;
frame.CreateEmptyFrame(kWidth, kHeight, kWidth, kWidth / 2, kWidth / 2);
uint32_t timestamp = 0;
for (size_t i = 0; i < 1000; ++i) {
frame.set_timestamp(timestamp);
overuse_detector_->FrameCaptured(frame);
// Encode and send first parts almost instantly.
clock_->AdvanceTimeMilliseconds(1);
overuse_detector_->FrameSent(timestamp);
// Encode heavier part, resulting in >85% usage total.
clock_->AdvanceTimeMilliseconds(kDelayMs - 1);
overuse_detector_->FrameSent(timestamp);
clock_->AdvanceTimeMilliseconds(kIntervalMs - kDelayMs);
timestamp += kIntervalMs * 90;
overuse_detector_->Process();
}
}
} // namespace webrtc

13
webrtc/video/send_statistics_proxy.cc
View File

@ -189,9 +189,13 @@ void SendStatisticsProxy::OnOutgoingRate(uint32_t framerate, uint32_t bitrate) {
stats_.media_bitrate_bps = bitrate;
}
void SendStatisticsProxy::CpuOveruseMetricsUpdated(
void SendStatisticsProxy::OnEncodedFrameTimeMeasured(
int encode_time_ms,
const CpuOveruseMetrics& metrics) {
rtc::CritScope lock(&crit_);
uma_container_->encode_time_counter_.Add(encode_time_ms);
encode_time_.Apply(1.0f, encode_time_ms);
stats_.avg_encode_time_ms = round(encode_time_.filtered());
stats_.encode_usage_percent = metrics.encode_usage_percent;
}
@ -331,13 +335,6 @@ void SendStatisticsProxy::OnIncomingFrame(int width, int height) {
uma_container_->input_height_counter_.Add(height);
}
void SendStatisticsProxy::OnEncodedFrame(int encode_time_ms) {
rtc::CritScope lock(&crit_);
uma_container_->encode_time_counter_.Add(encode_time_ms);
encode_time_.Apply(1.0f, encode_time_ms);
stats_.avg_encode_time_ms = round(encode_time_.filtered());
}
void SendStatisticsProxy::RtcpPacketTypesCounterUpdated(
uint32_t ssrc,
const RtcpPacketTypeCounter& packet_counter) {

15
webrtc/video/send_statistics_proxy.h
View File

@ -52,12 +52,6 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver,
// Used to update incoming frame rate.
void OnIncomingFrame(int width, int height);
// Used to update encode time of frames.
void OnEncodedFrame(int encode_time_ms);
// From VideoEncoderRateObserver.
void OnSetRates(uint32_t bitrate_bps, int framerate) override;
void OnEncoderImplementationName(const char* implementation_name);
void OnOutgoingRate(uint32_t framerate, uint32_t bitrate);
void OnSuspendChange(bool is_suspended);
@ -67,9 +61,14 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver,
// how stats are collected.
void SetContentType(VideoEncoderConfig::ContentType content_type);
// Implements VideoEncoderRateObserver.
void OnSetRates(uint32_t bitrate_bps, int framerate) override;
// Implements CpuOveruseMetricsObserver.
void OnEncodedFrameTimeMeasured(int encode_time_ms,
const CpuOveruseMetrics& metrics) override;
protected:
// From CpuOveruseMetricsObserver.
void CpuOveruseMetricsUpdated(const CpuOveruseMetrics& metrics) override;
// From RtcpStatisticsCallback.
void StatisticsUpdated(const RtcpStatistics& statistics,
uint32_t ssrc) override;

7
webrtc/video/send_statistics_proxy_unittest.cc
View File

@ -289,12 +289,15 @@ TEST_F(SendStatisticsProxyTest, SendSideDelay) {
ExpectEqual(expected_, stats);
}
TEST_F(SendStatisticsProxyTest, OnEncodedFrame) {
TEST_F(SendStatisticsProxyTest, OnEncodedFrameTimeMeasured) {
const int kEncodeTimeMs = 11;
statistics_proxy_->OnEncodedFrame(kEncodeTimeMs);
CpuOveruseMetrics metrics;
metrics.encode_usage_percent = 80;
statistics_proxy_->OnEncodedFrameTimeMeasured(kEncodeTimeMs, metrics);
VideoSendStream::Stats stats = statistics_proxy_->GetStats();
EXPECT_EQ(kEncodeTimeMs, stats.avg_encode_time_ms);
EXPECT_EQ(metrics.encode_usage_percent, stats.encode_usage_percent);
}
TEST_F(SendStatisticsProxyTest, SwitchContentTypeUpdatesHistograms) {

46
webrtc/video/video_capture_input.cc
View File

@ -14,7 +14,6 @@
#include "webrtc/base/logging.h"
#include "webrtc/base/trace_event.h"
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/utility/include/process_thread.h"
#include "webrtc/modules/video_capture/video_capture_factory.h"
#include "webrtc/modules/video_processing/include/video_processing.h"
#include "webrtc/modules/video_render/video_render_defines.h"
@ -27,15 +26,11 @@
namespace webrtc {
namespace internal {
VideoCaptureInput::VideoCaptureInput(
ProcessThread* module_process_thread,
VideoCaptureCallback* frame_callback,
VideoRenderer* local_renderer,
SendStatisticsProxy* stats_proxy,
CpuOveruseObserver* overuse_observer,
EncodingTimeObserver* encoding_time_observer)
: module_process_thread_(module_process_thread),
frame_callback_(frame_callback),
VideoCaptureInput::VideoCaptureInput(VideoCaptureCallback* frame_callback,
VideoRenderer* local_renderer,
SendStatisticsProxy* stats_proxy,
OveruseFrameDetector* overuse_detector)
: frame_callback_(frame_callback),
local_renderer_(local_renderer),
stats_proxy_(stats_proxy),
encoder_thread_(EncoderThreadFunction, this, "EncoderThread"),
@ -45,19 +40,12 @@ VideoCaptureInput::VideoCaptureInput(
delta_ntp_internal_ms_(
Clock::GetRealTimeClock()->CurrentNtpInMilliseconds() -
TickTime::MillisecondTimestamp()),
overuse_detector_(new OveruseFrameDetector(Clock::GetRealTimeClock(),
CpuOveruseOptions(),
overuse_observer,
stats_proxy)),
encoding_time_observer_(encoding_time_observer) {
overuse_detector_(overuse_detector) {
encoder_thread_.Start();
encoder_thread_.SetPriority(rtc::kHighPriority);
module_process_thread_->RegisterModule(overuse_detector_.get());
}
VideoCaptureInput::~VideoCaptureInput() {
module_process_thread_->DeRegisterModule(overuse_detector_.get());
// Stop the thread.
rtc::AtomicOps::ReleaseStore(&stop_, 1);
capture_event_.Set();
@ -105,9 +93,7 @@ void VideoCaptureInput::IncomingCapturedFrame(const VideoFrame& video_frame) {
captured_frame_.ShallowCopy(incoming_frame);
last_captured_timestamp_ = incoming_frame.ntp_time_ms();
overuse_detector_->FrameCaptured(captured_frame_.width(),
captured_frame_.height(),
captured_frame_.render_time_ms());
overuse_detector_->FrameCaptured(captured_frame_);
TRACE_EVENT_ASYNC_BEGIN1("webrtc", "Video", video_frame.render_time_ms(),
"render_time", video_frame.render_time_ms());
@ -121,12 +107,10 @@ bool VideoCaptureInput::EncoderThreadFunction(void* obj) {
bool VideoCaptureInput::EncoderProcess() {
static const int kThreadWaitTimeMs = 100;
int64_t capture_time = -1;
if (capture_event_.Wait(kThreadWaitTimeMs)) {
if (rtc::AtomicOps::AcquireLoad(&stop_))
return false;
int64_t encode_start_time = -1;
VideoFrame deliver_frame;
{
rtc::CritScope lock(&crit_);
@ -136,24 +120,8 @@ bool VideoCaptureInput::EncoderProcess() {
}
}
if (!deliver_frame.IsZeroSize()) {
capture_time = deliver_frame.render_time_ms();
encode_start_time = Clock::GetRealTimeClock()->TimeInMilliseconds();
frame_callback_->DeliverFrame(deliver_frame);
}
// Update the overuse detector with the duration.
if (encode_start_time != -1) {
int encode_time_ms = static_cast<int>(
Clock::GetRealTimeClock()->TimeInMilliseconds() - encode_start_time);
stats_proxy_->OnEncodedFrame(encode_time_ms);
if (encoding_time_observer_) {
encoding_time_observer_->OnReportEncodedTime(
deliver_frame.ntp_time_ms(), encode_time_ms);
}
}
}
// We're done!
if (capture_time != -1) {
overuse_detector_->FrameSent(capture_time);
}
return true;
}

15
webrtc/video/video_capture_input.h
View File

@ -16,7 +16,6 @@
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/event.h"
#include "webrtc/base/platform_thread.h"
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/base/thread_annotations.h"
#include "webrtc/common_types.h"
#include "webrtc/engine_configurations.h"
@ -30,11 +29,7 @@
namespace webrtc {
class Config;
class CpuOveruseMetricsObserver;
class CpuOveruseObserver;
class OveruseFrameDetector;
class ProcessThread;
class RegistrableCpuOveruseMetricsObserver;
class SendStatisticsProxy;
class VideoRenderer;
@ -48,12 +43,10 @@ class VideoCaptureCallback {
namespace internal {
class VideoCaptureInput : public webrtc::VideoCaptureInput {
public:
VideoCaptureInput(ProcessThread* module_process_thread,
VideoCaptureCallback* frame_callback,
VideoCaptureInput(VideoCaptureCallback* frame_callback,
VideoRenderer* local_renderer,
SendStatisticsProxy* send_stats_proxy,
CpuOveruseObserver* overuse_observer,
EncodingTimeObserver* encoding_time_observer);
OveruseFrameDetector* overuse_detector);
~VideoCaptureInput();
void IncomingCapturedFrame(const VideoFrame& video_frame) override;
@ -64,7 +57,6 @@ class VideoCaptureInput : public webrtc::VideoCaptureInput {
bool EncoderProcess();
rtc::CriticalSection crit_;
ProcessThread* const module_process_thread_;
VideoCaptureCallback* const frame_callback_;
VideoRenderer* const local_renderer_;
@ -81,8 +73,7 @@ class VideoCaptureInput : public webrtc::VideoCaptureInput {
// Delta used for translating between NTP and internal timestamps.
const int64_t delta_ntp_internal_ms_;
rtc::scoped_ptr<OveruseFrameDetector> overuse_detector_;
EncodingTimeObserver* const encoding_time_observer_;
OveruseFrameDetector* const overuse_detector_;
};
} // namespace internal

32
webrtc/video/video_capture_input_unittest.cc
View File

@ -15,7 +15,6 @@
#include "testing/gtest/include/gtest/gtest.h"
#include "webrtc/base/event.h"
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/modules/utility/include/mock/mock_process_thread.h"
#include "webrtc/system_wrappers/include/ref_count.h"
#include "webrtc/system_wrappers/include/scoped_vector.h"
#include "webrtc/test/fake_texture_frame.h"
@ -47,28 +46,23 @@ VideoFrame* CreateVideoFrame(uint8_t length);
class VideoCaptureInputTest : public ::testing::Test {
protected:
VideoCaptureInputTest()
: mock_process_thread_(new NiceMock<MockProcessThread>),
mock_frame_callback_(new NiceMock<MockVideoCaptureCallback>),
output_frame_event_(false, false),
stats_proxy_(Clock::GetRealTimeClock(),
: stats_proxy_(Clock::GetRealTimeClock(),
webrtc::VideoSendStream::Config(nullptr),
webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo) {}
webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo),
mock_frame_callback_(new NiceMock<MockVideoCaptureCallback>),
output_frame_event_(false, false) {}
virtual void SetUp() {
EXPECT_CALL(*mock_frame_callback_, DeliverFrame(_))
.WillRepeatedly(
WithArg<0>(Invoke(this, &VideoCaptureInputTest::AddOutputFrame)));
input_.reset(new internal::VideoCaptureInput(
mock_process_thread_.get(), mock_frame_callback_.get(), nullptr,
&stats_proxy_, nullptr, nullptr));
}
virtual void TearDown() {
// VideoCaptureInput accesses |mock_process_thread_| in destructor and
// should
// be deleted first.
input_.reset();
overuse_detector_.reset(
new OveruseFrameDetector(Clock::GetRealTimeClock(), CpuOveruseOptions(),
nullptr, nullptr, &stats_proxy_));
input_.reset(new internal::VideoCaptureInput(mock_frame_callback_.get(),
nullptr, &stats_proxy_,
overuse_detector_.get()));
}
void AddInputFrame(VideoFrame* frame) {
@ -86,9 +80,12 @@ class VideoCaptureInputTest : public ::testing::Test {
EXPECT_TRUE(output_frame_event_.Wait(FRAME_TIMEOUT_MS));
}
rtc::scoped_ptr<MockProcessThread> mock_process_thread_;
SendStatisticsProxy stats_proxy_;
rtc::scoped_ptr<MockVideoCaptureCallback> mock_frame_callback_;
rtc::scoped_ptr<OveruseFrameDetector> overuse_detector_;
// Used to send input capture frames to VideoCaptureInput.
rtc::scoped_ptr<internal::VideoCaptureInput> input_;
@ -104,7 +101,6 @@ class VideoCaptureInputTest : public ::testing::Test {
// The pointers of Y plane buffers of output frames. This is used to verify
// the frame are swapped and not copied.
std::vector<const uint8_t*> output_frame_ybuffers_;
SendStatisticsProxy stats_proxy_;
};
TEST_F(VideoCaptureInputTest, DoesNotRetainHandleNorCopyBuffer) {

30
webrtc/video/video_quality_test.cc
View File

@ -46,8 +46,7 @@ class VideoAnalyzer : public PacketReceiver,
public Transport,
public VideoRenderer,
public VideoCaptureInput,
public EncodedFrameObserver,
public EncodingTimeObserver {
public EncodedFrameObserver {
public:
VideoAnalyzer(test::LayerFilteringTransport* transport,
const std::string& test_label,
@ -65,6 +64,7 @@ class VideoAnalyzer : public PacketReceiver,
graph_data_output_file_(graph_data_output_file),
graph_title_(graph_title),
ssrc_to_analyze_(ssrc_to_analyze),
encode_timing_proxy_(this),
frames_to_process_(duration_frames),
frames_recorded_(0),
frames_processed_(0),
@ -129,8 +129,7 @@ class VideoAnalyzer : public PacketReceiver,
return receiver_->DeliverPacket(media_type, packet, length, packet_time);
}
// EncodingTimeObserver.
void OnReportEncodedTime(int64_t ntp_time_ms, int encode_time_ms) override {
void MeasuredEncodeTiming(int64_t ntp_time_ms, int encode_time_ms) {
rtc::CritScope crit(&comparison_lock_);
samples_encode_time_ms_[ntp_time_ms] = encode_time_ms;
}
@ -208,7 +207,7 @@ class VideoAnalyzer : public PacketReceiver,
assert(!reference_frame.IsZeroSize());
if (send_timestamp == reference_frame.timestamp() - 1) {
// TODO(ivica): Make this work for > 2 streams.
// Look at rtp_sender.c:RTPSender::BuildRTPHeader.
// Look at RTPSender::BuildRTPHeader.
++send_timestamp;
}
EXPECT_EQ(reference_frame.timestamp(), send_timestamp);
@ -263,6 +262,8 @@ class VideoAnalyzer : public PacketReceiver,
stats_polling_thread_.Stop();
}
EncodedFrameObserver* encode_timing_proxy() { return &encode_timing_proxy_; }
VideoCaptureInput* input_;
test::LayerFilteringTransport* const transport_;
PacketReceiver* receiver_;
@ -329,6 +330,21 @@ class VideoAnalyzer : public PacketReceiver,
double ssim;
};
// This class receives the send-side OnEncodeTiming and is provided to not
// conflict with the receiver-side pre_decode_callback.
class OnEncodeTimingProxy : public EncodedFrameObserver {
public:
explicit OnEncodeTimingProxy(VideoAnalyzer* parent) : parent_(parent) {}
void OnEncodeTiming(int64_t ntp_time_ms, int encode_time_ms) override {
parent_->MeasuredEncodeTiming(ntp_time_ms, encode_time_ms);
}
void EncodedFrameCallback(const EncodedFrame& frame) override {}
private:
VideoAnalyzer* const parent_;
};
void AddFrameComparison(const VideoFrame& reference,
const VideoFrame& render,
bool dropped,
@ -566,6 +582,7 @@ class VideoAnalyzer : public PacketReceiver,
FILE* const graph_data_output_file_;
const std::string graph_title_;
const uint32_t ssrc_to_analyze_;
OnEncodeTimingProxy encode_timing_proxy_;
std::vector<Sample> samples_ GUARDED_BY(comparison_lock_);
std::map<int64_t, int> samples_encode_time_ms_ GUARDED_BY(comparison_lock_);
test::Statistics sender_time_ GUARDED_BY(comparison_lock_);
@ -976,10 +993,11 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) {
recv_transport.SetReceiver(sender_call_->Receiver());
SetupCommon(&analyzer, &recv_transport);
video_send_config_.encoding_time_observer = &analyzer;
video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer;
for (auto& config : video_receive_configs_)
config.pre_decode_callback = &analyzer;
RTC_DCHECK(!video_send_config_.post_encode_callback);
video_send_config_.post_encode_callback = analyzer.encode_timing_proxy();
if (params_.screenshare.enabled)
SetupScreenshare();

34
webrtc/video/video_send_stream.cc
View File

@ -22,6 +22,7 @@
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
#include "webrtc/modules/pacing/packet_router.h"
#include "webrtc/modules/utility/include/process_thread.h"
#include "webrtc/video/call_stats.h"
#include "webrtc/video/encoder_state_feedback.h"
#include "webrtc/video/payload_router.h"
@ -107,6 +108,18 @@ std::string VideoSendStream::Config::ToString() const {
return ss.str();
}
namespace {
CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
CpuOveruseOptions options;
if (full_overuse_time) {
options.low_encode_usage_threshold_percent = 100;
options.high_encode_usage_threshold_percent = 120;
}
return options;
}
} // namespace
namespace internal {
VideoSendStream::VideoSendStream(
int num_cpu_cores,
@ -127,6 +140,12 @@ VideoSendStream::VideoSendStream(
module_process_thread_(module_process_thread),
call_stats_(call_stats),
congestion_controller_(congestion_controller),
overuse_detector_(
Clock::GetRealTimeClock(),
GetCpuOveruseOptions(config.encoder_settings.full_overuse_time),
this,
config.post_encode_callback,
&stats_proxy_),
encoder_feedback_(new EncoderStateFeedback()),
use_config_bitrate_(true) {
LOG(LS_INFO) << "VideoSendStream: " << config_.ToString();
@ -144,10 +163,10 @@ VideoSendStream::VideoSendStream(
const std::vector<uint32_t>& ssrcs = config.rtp.ssrcs;
vie_encoder_.reset(new ViEEncoder(
num_cpu_cores, module_process_thread_, &stats_proxy_,
config.pre_encode_callback, congestion_controller_->pacer(),
bitrate_allocator));
vie_encoder_.reset(
new ViEEncoder(num_cpu_cores, module_process_thread_, &stats_proxy_,
config.pre_encode_callback, &overuse_detector_,
congestion_controller_->pacer(), bitrate_allocator));
RTC_CHECK(vie_encoder_->Init());
vie_channel_.reset(new ViEChannel(
@ -207,8 +226,8 @@ VideoSendStream::VideoSendStream(
vie_channel_->SetRTCPCName(config_.rtp.c_name.c_str());
input_.reset(new internal::VideoCaptureInput(
module_process_thread_, vie_encoder_.get(), config_.local_renderer,
&stats_proxy_, this, config_.encoding_time_observer));
vie_encoder_.get(), config_.local_renderer, &stats_proxy_,
&overuse_detector_));
// 28 to match packet overhead in ModuleRtpRtcpImpl.
RTC_DCHECK_LE(config_.rtp.max_packet_size, static_cast<size_t>(0xFFFF - 28));
@ -240,10 +259,13 @@ VideoSendStream::VideoSendStream(
vie_channel_->RegisterRtcpPacketTypeCounterObserver(&stats_proxy_);
vie_channel_->RegisterSendBitrateObserver(&stats_proxy_);
vie_channel_->RegisterSendFrameCountObserver(&stats_proxy_);
module_process_thread_->RegisterModule(&overuse_detector_);
}
VideoSendStream::~VideoSendStream() {
LOG(LS_INFO) << "~VideoSendStream: " << config_.ToString();
module_process_thread_->DeRegisterModule(&overuse_detector_);
vie_channel_->RegisterSendFrameCountObserver(nullptr);
vie_channel_->RegisterSendBitrateObserver(nullptr);
vie_channel_->RegisterRtcpPacketTypeCounterObserver(nullptr);

1
webrtc/video/video_send_stream.h
View File

@ -86,6 +86,7 @@ class VideoSendStream : public webrtc::VideoSendStream,
CallStats* const call_stats_;
CongestionController* const congestion_controller_;
OveruseFrameDetector overuse_detector_;
rtc::scoped_ptr<VideoCaptureInput> input_;
rtc::scoped_ptr<ViEChannel> vie_channel_;
rtc::scoped_ptr<ViEEncoder> vie_encoder_;

17
webrtc/video/vie_encoder.cc
View File

@ -27,12 +27,13 @@
#include "webrtc/modules/video_coding/include/video_codec_interface.h"
#include "webrtc/modules/video_coding/include/video_coding.h"
#include "webrtc/modules/video_coding/include/video_coding_defines.h"
#include "webrtc/modules/video_coding/encoded_frame.h"
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc/system_wrappers/include/metrics.h"
#include "webrtc/system_wrappers/include/tick_util.h"
#include "webrtc/video/overuse_frame_detector.h"
#include "webrtc/video/payload_router.h"
#include "webrtc/video/send_statistics_proxy.h"
#include "webrtc/video_frame.h"
namespace webrtc {
@ -107,6 +108,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
ProcessThread* module_process_thread,
SendStatisticsProxy* stats_proxy,
I420FrameCallback* pre_encode_callback,
OveruseFrameDetector* overuse_detector,
PacedSender* pacer,
BitrateAllocator* bitrate_allocator)
: number_of_cores_(number_of_cores),
@ -118,6 +120,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
send_payload_router_(NULL),
stats_proxy_(stats_proxy),
pre_encode_callback_(pre_encode_callback),
overuse_detector_(overuse_detector),
pacer_(pacer),
bitrate_allocator_(bitrate_allocator),
time_of_last_frame_activity_ms_(0),
@ -461,12 +464,12 @@ int32_t ViEEncoder::SendData(const uint8_t payload_type,
if (stats_proxy_ != NULL)
stats_proxy_->OnSendEncodedImage(encoded_image, rtp_video_hdr);
return send_payload_router_->RoutePayload(
encoded_image._frameType, payload_type, encoded_image._timeStamp,
encoded_image.capture_time_ms_, encoded_image._buffer,
encoded_image._length, fragmentation_header, rtp_video_hdr)
? 0
: -1;
bool success = send_payload_router_->RoutePayload(
encoded_image._frameType, payload_type, encoded_image._timeStamp,
encoded_image.capture_time_ms_, encoded_image._buffer,
encoded_image._length, fragmentation_header, rtp_video_hdr);
overuse_detector_->FrameSent(encoded_image._timeStamp);
return success ? 0 : -1;
}
void ViEEncoder::OnEncoderImplementationName(

3
webrtc/video/vie_encoder.h
View File

@ -33,6 +33,7 @@ class BitrateAllocator;
class BitrateObserver;
class Config;
class EncodedImageCallback;
class OveruseFrameDetector;
class PacedSender;
class PayloadRouter;
class ProcessThread;
@ -54,6 +55,7 @@ class ViEEncoder : public RtcpIntraFrameObserver,
ProcessThread* module_process_thread,
SendStatisticsProxy* stats_proxy,
I420FrameCallback* pre_encode_callback,
OveruseFrameDetector* overuse_detector,
PacedSender* pacer,
BitrateAllocator* bitrate_allocator);
~ViEEncoder();
@ -163,6 +165,7 @@ class ViEEncoder : public RtcpIntraFrameObserver,
SendStatisticsProxy* const stats_proxy_;
I420FrameCallback* const pre_encode_callback_;
OveruseFrameDetector* const overuse_detector_;
PacedSender* const pacer_;
BitrateAllocator* const bitrate_allocator_;

21
webrtc/video_send_stream.h
View File

@ -26,13 +26,6 @@ namespace webrtc {
class LoadObserver;
class VideoEncoder;
class EncodingTimeObserver {
public:
virtual ~EncodingTimeObserver() {}
virtual void OnReportEncodedTime(int64_t ntp_time_ms, int encode_time_ms) = 0;
};
// Class to deliver captured frame to the video send stream.
class VideoCaptureInput {
public:
@ -91,6 +84,11 @@ class VideoSendStream : public SendStream {
// sources anymore.
bool internal_source = false;
// Allow 100% encoder utilization. Used for HW encoders where CPU isn't
// expected to be the limiting factor, but a chip could be running at
// 30fps (for example) exactly.
bool full_overuse_time = false;
// Uninitialized VideoEncoder instance to be used for encoding. Will be
// initialized from inside the VideoSendStream.
VideoEncoder* encoder = nullptr;
@ -144,7 +142,9 @@ class VideoSendStream : public SendStream {
I420FrameCallback* pre_encode_callback = nullptr;
// Called for each encoded frame, e.g. used for file storage. 'nullptr'
// disables the callback.
// disables the callback. Also measures timing and passes the time
// spent on encoding. This timing will not fire if encoding takes longer
// than the measuring window, since the sample data will have been dropped.
EncodedFrameObserver* post_encode_callback = nullptr;
// Renderer for local preview. The local renderer will be called even if
@ -164,11 +164,6 @@ class VideoSendStream : public SendStream {
// below the minimum configured bitrate. If this variable is false, the
// stream may send at a rate higher than the estimated available bitrate.
bool suspend_below_min_bitrate = false;
// Called for each encoded frame. Passes the total time spent on encoding.
// TODO(ivica): Consolidate with post_encode_callback:
// https://code.google.com/p/webrtc/issues/detail?id=5042
EncodingTimeObserver* encoding_time_observer = nullptr;
};
// Gets interface used to insert captured frames. Valid as long as the