admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor OverUseFrameDetector to use the timestamps attached to EncodedImage.

Browse Source 
Bug: webrtc:8504
Change-Id: I3f99c3c1e528f59b45724921a91f65b485f5b820
Reviewed-on: https://webrtc-review.googlesource.com/23720
Commit-Queue: Niels Moller <nisse@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20979}
This commit is contained in:
Niels Möller 2017-12-01 11:25:01 +01:00 committed by Commit Bot
parent b215bc70c1
commit eee7cedf69
5 changed files with 132 additions and 539 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
video/overuse_frame_detector.cc
View File

@ -23,7 +23,6 @@
#include "common_video/include/frame_callback.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/timeutils.h"
#include "system_wrappers/include/field_trial.h"
@ -48,31 +47,15 @@ const double kRampUpBackoffFactor = 2.0;
// Max number of overuses detected before always applying the rampup delay.
const int kMaxOverusesBeforeApplyRampupDelay = 4;
// The maximum exponent to use in VCMExpFilter.
const float kMaxExp = 7.0f;
// Default value used before first reconfiguration.
const int kDefaultFrameRate = 30;
// Default sample diff, default frame rate.
const float kDefaultSampleDiffMs = 1000.0f / kDefaultFrameRate;
// A factor applied to the sample diff on OnTargetFramerateUpdated to determine
// a max limit for the sample diff. For instance, with a framerate of 30fps,
// the sample diff is capped to (1000 / 30) * 1.35 = 45ms. This prevents
// triggering too soon if there are individual very large outliers.
const float kMaxSampleDiffMarginFactor = 1.35f;
// Minimum framerate allowed for usage calculation. This prevents crazy long
// encode times from being accepted if the frame rate happens to be low.
const int kMinFramerate = 7;
const int kMaxFramerate = 30;
const auto kScaleReasonCpu = AdaptationObserverInterface::AdaptReason::kCpu;
} // namespace
CpuOveruseOptions::CpuOveruseOptions()
: high_encode_usage_threshold_percent(85),
frame_timeout_interval_ms(1500),
min_frame_samples(120),
min_process_count(3),
high_threshold_consecutive_count(2) {
high_threshold_consecutive_count(2),
filter_time_ms(5000) {
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
// This is proof-of-concept code for letting the physical core count affect
// the interval into which we attempt to scale. For now, the code is Mac OS
@ -123,72 +106,43 @@ CpuOveruseOptions::CpuOveruseOptions()
class OveruseFrameDetector::SendProcessingUsage {
public:
explicit SendProcessingUsage(const CpuOveruseOptions& options)
: kWeightFactorFrameDiff(0.998f),
kWeightFactorProcessing(0.995f),
kInitialSampleDiffMs(40.0f),
count_(0),
options_(options),
max_sample_diff_ms_(kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor),
filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)),
filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) {
: options_(options) {
Reset();
}
virtual ~SendProcessingUsage() {}
void Reset() {
count_ = 0;
max_sample_diff_ms_ = kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor;
filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff);
filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs);
filtered_processing_ms_->Reset(kWeightFactorProcessing);
filtered_processing_ms_->Apply(1.0f, InitialProcessingMs());
// Start in between the underuse and overuse threshold.
load_estimate_ = (options_.low_encode_usage_threshold_percent +
options_.high_encode_usage_threshold_percent) /
200.0;
}
void SetMaxSampleDiffMs(float diff_ms) { max_sample_diff_ms_ = diff_ms; }
void AddSample(double encode_time, double diff_time) {
RTC_CHECK_GE(diff_time, 0.0);
void AddCaptureSample(float sample_ms) {
float exp = sample_ms / kDefaultSampleDiffMs;
exp = std::min(exp, kMaxExp);
filtered_frame_diff_ms_->Apply(exp, sample_ms);
}
void AddSample(float processing_ms, int64_t diff_last_sample_ms) {
++count_;
float exp = diff_last_sample_ms / kDefaultSampleDiffMs;
exp = std::min(exp, kMaxExp);
filtered_processing_ms_->Apply(exp, processing_ms);
}
virtual int Value() {
if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) {
return static_cast<int>(InitialUsageInPercent() + 0.5f);
// Use the filter update
//
// load <-- x/d (1-exp (-d/T)) + exp (-d/T) load
//
// where we must take care for small d, using the proper limit
// (1 - exp(-d/tau)) / d = 1/tau - d/2tau^2 + O(d^2)
double tau = (1e-3 * options_.filter_time_ms);
double e = diff_time / tau;
double c;
if (e < 0.0001) {
c = (1 - e / 2) / tau;
} else {
c = -expm1(-e) / diff_time;
}
float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f);
frame_diff_ms = std::min(frame_diff_ms, max_sample_diff_ms_);
float encode_usage_percent =
100.0f * filtered_processing_ms_->filtered() / frame_diff_ms;
return static_cast<int>(encode_usage_percent + 0.5);
load_estimate_ = c * encode_time + exp(-e) * load_estimate_;
}
virtual int Value() { return static_cast<int>(100.0 * load_estimate_ + 0.5); }
private:
float InitialUsageInPercent() const {
// Start in between the underuse and overuse threshold.
return (options_.low_encode_usage_threshold_percent +
options_.high_encode_usage_threshold_percent) / 2.0f;
}
float InitialProcessingMs() const {
return InitialUsageInPercent() * kInitialSampleDiffMs / 100;
}
const float kWeightFactorFrameDiff;
const float kWeightFactorProcessing;
const float kInitialSampleDiffMs;
uint64_t count_;
const CpuOveruseOptions options_;
float max_sample_diff_ms_;
std::unique_ptr<rtc::ExpFilter> filtered_processing_ms_;
std::unique_ptr<rtc::ExpFilter> filtered_frame_diff_ms_;
double load_estimate_;
};
// Class used for manual testing of overuse, enabled via field trial flag.
@ -218,6 +172,7 @@ class OveruseFrameDetector::OverdoseInjector
int64_t now_ms = rtc::TimeMillis();
if (last_toggling_ms_ == -1) {
last_toggling_ms_ = now_ms;
} else {
switch (state_) {
case State::kNormal:
@ -255,7 +210,6 @@ class OveruseFrameDetector::OverdoseInjector
overried_usage_value.emplace(5);
break;
}
return overried_usage_value.value_or(SendProcessingUsage::Value());
}
@ -348,7 +302,6 @@ OveruseFrameDetector::OveruseFrameDetector(
last_capture_time_us_(-1),
last_processed_capture_time_us_(-1),
num_pixels_(0),
max_framerate_(kDefaultFrameRate),
last_overuse_time_ms_(-1),
checks_above_threshold_(0),
num_overuse_detections_(0),
@ -399,93 +352,41 @@ bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now_us) const {
options_.frame_timeout_interval_ms * rtc::kNumMicrosecsPerMillisec;
}
void OveruseFrameDetector::ResetAll(int num_pixels) {
// Reset state, as a result resolution being changed. Do not however change
// the current frame rate back to the default.
void OveruseFrameDetector::ResetAll() {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
num_pixels_ = num_pixels;
usage_->Reset();
frame_timing_.clear();
last_capture_time_us_ = -1;
last_processed_capture_time_us_ = -1;
num_process_times_ = 0;
metrics_ = rtc::Optional<CpuOveruseMetrics>();
OnTargetFramerateUpdated(max_framerate_);
}
void OveruseFrameDetector::OnTargetFramerateUpdated(int framerate_fps) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
RTC_DCHECK_GE(framerate_fps, 0);
max_framerate_ = std::min(kMaxFramerate, framerate_fps);
usage_->SetMaxSampleDiffMs((1000 / std::max(kMinFramerate, max_framerate_)) *
kMaxSampleDiffMarginFactor);
}
void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame,
int64_t time_when_first_seen_us) {
void OveruseFrameDetector::FrameCaptured(int width, int height) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
if (FrameSizeChanged(frame.width() * frame.height()) ||
FrameTimeoutDetected(time_when_first_seen_us)) {
ResetAll(frame.width() * frame.height());
if (FrameSizeChanged(width * height)) {
ResetAll();
num_pixels_ = width * height;
}
if (last_capture_time_us_ != -1)
usage_->AddCaptureSample(
1e-3 * (time_when_first_seen_us - last_capture_time_us_));
last_capture_time_us_ = time_when_first_seen_us;
frame_timing_.push_back(FrameTiming(frame.timestamp_us(), frame.timestamp(),
time_when_first_seen_us));
}
void OveruseFrameDetector::FrameSent(uint32_t timestamp,
int64_t time_sent_in_us) {
void OveruseFrameDetector::FrameEncoded(int64_t capture_time_us,
int64_t encode_duration_us) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
// Delay before reporting actual encoding time, used to have the ability to
// detect total encoding time when encoding more than one layer. Encoding is
// here assumed to finish within a second (or that we get enough long-time
// samples before one second to trigger an overuse even when this is not the
// case).
static const int64_t kEncodingTimeMeasureWindowMs = 1000;
for (auto& it : frame_timing_) {
if (it.timestamp == timestamp) {
it.last_send_us = time_sent_in_us;
break;
}
if (FrameTimeoutDetected(capture_time_us)) {
ResetAll();
} else if (last_capture_time_us_ != -1) {
usage_->AddSample(1e-6 * encode_duration_us,
1e-6 * (capture_time_us - last_capture_time_us_));
}
// TODO(pbos): Handle the case/log errors when not finding the corresponding
// frame (either very slow encoding or incorrect wrong timestamps returned
// from the encoder).
// This is currently the case for all frames on ChromeOS, so logging them
// would be spammy, and triggering overuse would be wrong.
// https://crbug.com/350106
while (!frame_timing_.empty()) {
FrameTiming timing = frame_timing_.front();
if (time_sent_in_us - timing.capture_us <
kEncodingTimeMeasureWindowMs * rtc::kNumMicrosecsPerMillisec) {
break;
}
if (timing.last_send_us != -1) {
int encode_duration_us =
static_cast<int>(timing.last_send_us - timing.capture_us);
if (encoder_timing_) {
// TODO(nisse): Update encoder_timing_ to also use us units.
encoder_timing_->OnEncodeTiming(timing.capture_time_us /
rtc::kNumMicrosecsPerMillisec,
encode_duration_us /
rtc::kNumMicrosecsPerMillisec);
}
if (last_processed_capture_time_us_ != -1) {
int64_t diff_us = timing.capture_us - last_processed_capture_time_us_;
usage_->AddSample(1e-3 * encode_duration_us, 1e-3 * diff_us);
}
last_processed_capture_time_us_ = timing.capture_us;
EncodedFrameTimeMeasured(encode_duration_us /
rtc::kNumMicrosecsPerMillisec);
}
frame_timing_.pop_front();
last_capture_time_us_ = capture_time_us;
EncodedFrameTimeMeasured(encode_duration_us / rtc::kNumMicrosecsPerMillisec);
if (encoder_timing_) {
// TODO(nisse): Update encoder_timing_ to also use us units.
encoder_timing_->OnEncodeTiming(
capture_time_us / rtc::kNumMicrosecsPerMillisec,
encode_duration_us / rtc::kNumMicrosecsPerMillisec);
}
}

31
video/overuse_frame_detector.h
View File

@ -17,7 +17,6 @@
#include "api/optional.h"
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/constructormagic.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/sequenced_task_checker.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/thread_annotations.h"
@ -35,12 +34,12 @@ struct CpuOveruseOptions {
// General settings.
int frame_timeout_interval_ms; // The maximum allowed interval between two
// frames before resetting estimations.
int min_frame_samples; // The minimum number of frames required.
int min_process_count; // The number of initial process times required before
// triggering an overuse/underuse.
int high_threshold_consecutive_count; // The number of consecutive checks
// above the high threshold before
// triggering an overuse.
int filter_time_ms; // Time constant for averaging
};
struct CpuOveruseMetrics {
@ -77,18 +76,11 @@ class OveruseFrameDetector {
// StartCheckForOveruse has been called.
void StopCheckForOveruse();
// Defines the current maximum framerate targeted by the capturer. This is
// used to make sure the encode usage percent doesn't drop unduly if the
// capturer has quiet periods (for instance caused by screen capturers with
// variable capture rate depending on content updates), otherwise we might
// experience adaptation toggling.
virtual void OnTargetFramerateUpdated(int framerate_fps);
// Called for each captured frame.
void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us);
void FrameCaptured(int width, int height);
// Called for each sent frame.
void FrameSent(uint32_t timestamp, int64_t time_sent_in_us);
// Called for each encoded frame.
void FrameEncoded(int64_t capture_time_us, int64_t encode_duration_us);
protected:
void CheckForOveruse(); // Protected for test purposes.
@ -97,17 +89,6 @@ class OveruseFrameDetector {
class OverdoseInjector;
class SendProcessingUsage;
class CheckOveruseTask;
struct FrameTiming {
FrameTiming(int64_t capture_time_us, uint32_t timestamp, int64_t now)
: capture_time_us(capture_time_us),
timestamp(timestamp),
capture_us(now),
last_send_us(-1) {}
int64_t capture_time_us;
uint32_t timestamp;
int64_t capture_us;
int64_t last_send_us;
};
void EncodedFrameTimeMeasured(int encode_duration_ms);
bool IsOverusing(const CpuOveruseMetrics& metrics);
@ -116,7 +97,7 @@ class OveruseFrameDetector {
bool FrameTimeoutDetected(int64_t now) const;
bool FrameSizeChanged(int num_pixels) const;
void ResetAll(int num_pixels);
void ResetAll();
static std::unique_ptr<SendProcessingUsage> CreateSendProcessingUsage(
const CpuOveruseOptions& options);
@ -142,7 +123,6 @@ class OveruseFrameDetector {
// Number of pixels of last captured frame.
int num_pixels_ RTC_GUARDED_BY(task_checker_);
int max_framerate_ RTC_GUARDED_BY(task_checker_);
int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_);
int checks_above_threshold_ RTC_GUARDED_BY(task_checker_);
int num_overuse_detections_ RTC_GUARDED_BY(task_checker_);
@ -154,7 +134,6 @@ class OveruseFrameDetector {
// allocs)?
const std::unique_ptr<SendProcessingUsage> usage_
RTC_GUARDED_BY(task_checker_);
std::list<FrameTiming> frame_timing_ RTC_GUARDED_BY(task_checker_);
RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector);
};

238
video/overuse_frame_detector_unittest.cc
View File

@ -15,6 +15,8 @@
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/event.h"
#include "rtc_base/fakeclock.h"
#include "rtc_base/logging.h"
#include "rtc_base/random.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "video/overuse_frame_detector.h"
@ -22,6 +24,8 @@
namespace webrtc {
using ::testing::InvokeWithoutArgs;
using ::testing::AtLeast;
using ::testing::_;
namespace {
const int kWidth = 640;
@ -97,27 +101,20 @@ class OveruseFrameDetectorTest : public ::testing::Test,
int width,
int height,
int delay_us) {
VideoFrame frame(I420Buffer::Create(width, height),
webrtc::kVideoRotation_0, 0);
uint32_t timestamp = 0;
while (num_frames-- > 0) {
frame.set_timestamp(timestamp);
overuse_detector_->FrameCaptured(frame, rtc::TimeMicros());
clock_.AdvanceTimeMicros(delay_us);
overuse_detector_->FrameSent(timestamp, rtc::TimeMicros());
clock_.AdvanceTimeMicros(interval_us - delay_us);
timestamp += interval_us * 90 / 1000;
overuse_detector_->FrameCaptured(width, height);
overuse_detector_->FrameEncoded(rtc::TimeMicros(), delay_us);
clock_.AdvanceTimeMicros(interval_us);
}
}
void ForceUpdate(int width, int height) {
// Insert one frame, wait a second and then put in another to force update
// the usage. From the tests where these are used, adding another sample
// doesn't affect the expected outcome (this is mainly to check initial
// values and whether the overuse detector has been reset or not).
InsertAndSendFramesWithInterval(2, rtc::kNumMicrosecsPerSec,
width, height, kFrameIntervalUs);
// This is mainly to check initial values and whether the overuse
// detector has been reset or not.
InsertAndSendFramesWithInterval(1, rtc::kNumMicrosecsPerSec, width, height,
kFrameIntervalUs);
}
void TriggerOveruse(int num_times) {
const int kDelayUs = 32 * rtc::kNumMicrosecsPerMillisec;
for (int i = 0; i < num_times; ++i) {
@ -269,76 +266,11 @@ TEST_F(OveruseFrameDetectorTest, ResetAfterFrameTimeout) {
EXPECT_EQ(InitialUsage(), UsagePercent());
}
TEST_F(OveruseFrameDetectorTest, MinFrameSamplesBeforeUpdating) {
options_.min_frame_samples = 40;
ReinitializeOveruseDetector();
InsertAndSendFramesWithInterval(
40, kFrameIntervalUs, kWidth, kHeight, kProcessTimeUs);
EXPECT_EQ(InitialUsage(), UsagePercent());
// Pass time far enough to digest all previous samples.
clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec);
InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight,
kProcessTimeUs);
// The last sample has not been processed here.
EXPECT_EQ(InitialUsage(), UsagePercent());
// Pass time far enough to digest all previous samples, 41 in total.
clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec);
InsertAndSendFramesWithInterval(
1, kFrameIntervalUs, kWidth, kHeight, kProcessTimeUs);
EXPECT_NE(InitialUsage(), UsagePercent());
}
TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) {
ForceUpdate(kWidth, kHeight);
EXPECT_EQ(InitialUsage(), UsagePercent());
}
TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) {
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_))
.Times(testing::AtLeast(1));
static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec;
static const size_t kNumFramesEncodingDelay = 3;
VideoFrame frame(I420Buffer::Create(kWidth, kHeight),
webrtc::kVideoRotation_0, 0);
for (size_t i = 0; i < 1000; ++i) {
// Unique timestamps.
frame.set_timestamp(static_cast<uint32_t>(i));
overuse_detector_->FrameCaptured(frame, rtc::TimeMicros());
clock_.AdvanceTimeMicros(kIntervalUs);
if (i > kNumFramesEncodingDelay) {
overuse_detector_->FrameSent(
static_cast<uint32_t>(i - kNumFramesEncodingDelay),
rtc::TimeMicros());
}
overuse_detector_->CheckForOveruse();
}
}
TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) {
// >85% encoding time should trigger overuse.
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_))
.Times(testing::AtLeast(1));
static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec;
static const int kDelayUs = 30 * rtc::kNumMicrosecsPerMillisec;
VideoFrame frame(I420Buffer::Create(kWidth, kHeight),
webrtc::kVideoRotation_0, 0);
uint32_t timestamp = 0;
for (size_t i = 0; i < 1000; ++i) {
frame.set_timestamp(timestamp);
overuse_detector_->FrameCaptured(frame, rtc::TimeMicros());
// Encode and send first parts almost instantly.
clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec);
overuse_detector_->FrameSent(timestamp, rtc::TimeMicros());
// Encode heavier part, resulting in >85% usage total.
clock_.AdvanceTimeMicros(kDelayUs - rtc::kNumMicrosecsPerMillisec);
overuse_detector_->FrameSent(timestamp, rtc::TimeMicros());
clock_.AdvanceTimeMicros(kIntervalUs - kDelayUs);
timestamp += kIntervalUs * 90 / 1000;
overuse_detector_->CheckForOveruse();
}
}
TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) {
rtc::TaskQueue queue("OveruseFrameDetectorTestQueue");
@ -369,115 +301,59 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) {
EXPECT_TRUE(event.Wait(10000));
}
TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) {
const int kCapturerMaxFrameRate = 30;
const int kEncodeMaxFrameRate = 20; // Maximum fps the encoder can sustain.
// Models screencast, with irregular arrival of frames which are heavy
// to encode.
TEST_F(OveruseFrameDetectorTest, NoOveruseForLargeRandomFrameInterval) {
EXPECT_CALL(*(observer_.get()), AdaptDown(_)).Times(0);
EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(AtLeast(1));
const int kNumFrames = 500;
const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec;
const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec;
const int kMaxIntervalUs = 1000 * rtc::kNumMicrosecsPerMillisec;
webrtc::Random random(17);
for (int i = 0; i < kNumFrames; i++) {
int interval_us = random.Rand(kMinIntervalUs, kMaxIntervalUs);
overuse_detector_->FrameCaptured(kWidth, kHeight);
overuse_detector_->FrameEncoded(rtc::TimeMicros(), kEncodeTimeUs);
// Trigger overuse.
int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kCapturerMaxFrameRate;
// Processing time just below over use limit given kEncodeMaxFrameRate.
int64_t processing_time_us =
(98 * OveruseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100;
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
processing_time_us);
overuse_detector_->CheckForOveruse();
clock_.AdvanceTimeMicros(interval_us);
}
// Simulate frame rate reduction and normal usage.
frame_interval_us = rtc::kNumMicrosecsPerSec / kEncodeMaxFrameRate;
overuse_detector_->OnTargetFramerateUpdated(kEncodeMaxFrameRate);
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0);
for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
processing_time_us);
overuse_detector_->CheckForOveruse();
}
// Reduce processing time to trigger underuse.
processing_time_us =
(98 * UnderuseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100;
EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(1);
InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
processing_time_us);
overuse_detector_->CheckForOveruse();
// Average usage 19%. Check that estimate is in the right ball park.
EXPECT_NEAR(UsagePercent(), 20, 10);
}
TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) {
const int kMinFrameRate = 7; // Minimum fps allowed by current detector impl.
overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate);
// Models screencast, with irregular arrival of frames, often
// exceeding the timeout interval.
TEST_F(OveruseFrameDetectorTest, NoOveruseForRandomFrameIntervalWithReset) {
EXPECT_CALL(*(observer_.get()), AdaptDown(_)).Times(0);
EXPECT_CALL(*(observer_.get()), AdaptUp(reason_)).Times(AtLeast(1));
const int kNumFrames = 500;
const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec;
const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec;
const int kMaxIntervalUs = 3000 * rtc::kNumMicrosecsPerMillisec;
webrtc::Random random(17);
for (int i = 0; i < kNumFrames; i++) {
int interval_us = random.Rand(kMinIntervalUs, kMaxIntervalUs);
overuse_detector_->FrameCaptured(kWidth, kHeight);
overuse_detector_->FrameEncoded(rtc::TimeMicros(), kEncodeTimeUs);
// Normal usage just at the limit.
int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMinFrameRate;
// Processing time just below over use limit given kEncodeMaxFrameRate.
int64_t processing_time_us =
(98 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100;
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0);
for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
processing_time_us);
overuse_detector_->CheckForOveruse();
}
// Over the limit to overuse.
processing_time_us =
(102 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100;
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
processing_time_us);
overuse_detector_->CheckForOveruse();
}
// Reduce input frame rate. Should still trigger overuse.
overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate - 1);
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight,
processing_time_us);
overuse_detector_->CheckForOveruse();
}
}
TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) {
const int kMaxFrameRate = 20;
overuse_detector_->OnTargetFramerateUpdated(kMaxFrameRate);
int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMaxFrameRate;
// Maximum frame interval allowed is 35% above ideal.
int64_t max_frame_interval_us = (135 * frame_interval_us) / 100;
// Maximum processing time, without triggering overuse, allowed with the above
// frame interval.
int64_t max_processing_time_us =
(max_frame_interval_us * options_.high_encode_usage_threshold_percent) /
100;
// Processing time just below overuse limit given kMaxFrameRate.
int64_t processing_time_us = (98 * max_processing_time_us) / 100;
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(0);
for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth,
kHeight, processing_time_us);
overuse_detector_->CheckForOveruse();
}
// Go above limit, trigger overuse.
processing_time_us = (102 * max_processing_time_us) / 100;
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth,
kHeight, processing_time_us);
overuse_detector_->CheckForOveruse();
}
// Increase frame interval, should still trigger overuse.
max_frame_interval_us *= 2;
EXPECT_CALL(*(observer_.get()), AdaptDown(reason_)).Times(1);
for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) {
InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth,
kHeight, processing_time_us);
overuse_detector_->CheckForOveruse();
clock_.AdvanceTimeMicros(interval_us);
}
// Average usage 6.6%, but since the frame_timeout_interval_ms is
// only 1500 ms, we often reset the estimate to the initial value.
// Check that estimate is in the right ball park.
EXPECT_GE(UsagePercent(), 1);
EXPECT_LE(UsagePercent(), InitialUsage() + 5);
}
} // namespace webrtc

39
video/video_stream_encoder.cc
View File

@ -508,12 +508,6 @@ void VideoStreamEncoder::SetSource(
bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_);
initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop;
ConfigureQualityScaler();
if (!IsFramerateScalingEnabled(degradation_preference) &&
max_framerate_ != -1) {
// If frame rate scaling is no longer allowed, remove any potential
// allowance for longer frame intervals.
overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
}
});
}
@ -628,15 +622,6 @@ void VideoStreamEncoder::ReconfigureEncoder() {
sink_->OnEncoderConfigurationChanged(
std::move(streams), encoder_config_.min_transmit_bitrate_bps);
// Get the current target framerate, ie the maximum framerate as specified by
// the current codec configuration, or any limit imposed by cpu adaption in
// maintain-resolution or balanced mode. This is used to make sure overuse
// detection doesn't needlessly trigger in low and/or variable framerate
// scenarios.
int target_framerate = std::min(
max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps);
overuse_detector_->OnTargetFramerateUpdated(target_framerate);
ConfigureQualityScaler();
}
@ -821,7 +806,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
"Encode");
overuse_detector_->FrameCaptured(out_frame, time_when_posted_us);
overuse_detector_->FrameCaptured(out_frame.width(), out_frame.height());
video_sender_.AddVideoFrame(out_frame, nullptr);
}
@ -847,12 +832,21 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage(
EncodedImageCallback::Result result =
sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
int64_t time_sent_us = rtc::TimeMicros();
uint32_t timestamp = encoded_image._timeStamp;
int64_t capture_time_us =
encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
int64_t encode_duration_us;
if (encoded_image.timing_.flags != TimingFrameFlags::kInvalid) {
encode_duration_us = rtc::kNumMicrosecsPerMillisec *
(encoded_image.timing_.encode_finish_ms -
encoded_image.timing_.encode_start_ms);
} else {
encode_duration_us = -1;
}
const int qp = encoded_image.qp_;
encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] {
encoder_queue_.PostTask([this, capture_time_us, encode_duration_us, qp] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
overuse_detector_->FrameSent(timestamp, time_sent_us);
if (encode_duration_us >= 0)
overuse_detector_->FrameEncoded(capture_time_us, encode_duration_us);
if (quality_scaler_ && qp >= 0)
quality_scaler_->ReportQP(qp);
});
@ -1003,8 +997,6 @@ void VideoStreamEncoder::AdaptDown(AdaptReason reason) {
if (requested_framerate == -1)
return;
RTC_DCHECK_NE(max_framerate_, -1);
overuse_detector_->OnTargetFramerateUpdated(
std::min(max_framerate_, requested_framerate));
GetAdaptCounter().IncrementFramerate(reason);
break;
}
@ -1088,11 +1080,8 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) {
const int requested_framerate =
source_proxy_->RequestHigherFramerateThan(fps);
if (requested_framerate == -1) {
overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
return;
}
overuse_detector_->OnTargetFramerateUpdated(
std::min(max_framerate_, requested_framerate));
GetAdaptCounter().DecrementFramerate(reason);
break;
}

170
video/video_stream_encoder_unittest.cc
View File

@ -63,39 +63,11 @@ class TestBuffer : public webrtc::I420Buffer {
rtc::Event* const event_;
};
class CpuOveruseDetectorProxy : public OveruseFrameDetector {
public:
CpuOveruseDetectorProxy(const CpuOveruseOptions& options,
AdaptationObserverInterface* overuse_observer,
EncodedFrameObserver* encoder_timing_,
CpuOveruseMetricsObserver* metrics_observer)
: OveruseFrameDetector(options,
overuse_observer,
encoder_timing_,
metrics_observer),
last_target_framerate_fps_(-1) {}
virtual ~CpuOveruseDetectorProxy() {}
void OnTargetFramerateUpdated(int framerate_fps) override {
rtc::CritScope cs(&lock_);
last_target_framerate_fps_ = framerate_fps;
OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
}
int GetLastTargetFramerate() {
rtc::CritScope cs(&lock_);
return last_target_framerate_fps_;
}
private:
rtc::CriticalSection lock_;
int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
};
class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
public:
VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
const VideoSendStream::Config::EncoderSettings& settings)
VideoStreamEncoderUnderTest(
SendStatisticsProxy* stats_proxy,
const VideoSendStream::Config::EncoderSettings& settings)
: VideoStreamEncoder(
1 /* number_of_cores */,
stats_proxy,
@ -103,7 +75,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
nullptr /* pre_encode_callback */,
nullptr /* encoder_timing */,
std::unique_ptr<OveruseFrameDetector>(
overuse_detector_proxy_ = new CpuOveruseDetectorProxy(
overuse_detector_ = new OveruseFrameDetector(
GetCpuOveruseOptions(settings.full_overuse_time),
this,
nullptr,
@ -136,7 +108,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
CpuOveruseDetectorProxy* overuse_detector_proxy_;
OveruseFrameDetector* overuse_detector_;
};
class VideoStreamFactory
@ -2238,126 +2210,6 @@ TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
const int kFrameWidth = 1280;
const int kFrameHeight = 720;
const int kFramerate = 24;
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
test::FrameForwarder source;
video_stream_encoder_->SetSource(
&source, VideoSendStream::DegradationPreference::kMaintainResolution);
// Insert a single frame, triggering initial configuration.
source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
EXPECT_EQ(
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
kDefaultFramerate);
// Trigger reconfigure encoder (without resetting the entire instance).
VideoEncoderConfig video_encoder_config;
video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
video_encoder_config.number_of_streams = 1;
video_encoder_config.video_stream_factory =
new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
kMaxPayloadLength, false);
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
// Detector should be updated with fps limit from codec config.
EXPECT_EQ(
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
kFramerate);
// Trigger overuse, max framerate should be reduced.
VideoSendStream::Stats stats = stats_proxy_->GetStats();
stats.input_frame_rate = kFramerate;
stats_proxy_->SetMockStats(stats);
video_stream_encoder_->TriggerCpuOveruse();
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
int adapted_framerate =
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
EXPECT_LT(adapted_framerate, kFramerate);
// Trigger underuse, max framerate should go back to codec configured fps.
// Set extra low fps, to make sure it's actually reset, not just incremented.
stats = stats_proxy_->GetStats();
stats.input_frame_rate = adapted_framerate / 2;
stats_proxy_->SetMockStats(stats);
video_stream_encoder_->TriggerCpuNormalUsage();
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
EXPECT_EQ(
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
kFramerate);
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest,
OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
const int kFrameWidth = 1280;
const int kFrameHeight = 720;
const int kLowFramerate = 15;
const int kHighFramerate = 25;
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
test::FrameForwarder source;
video_stream_encoder_->SetSource(
&source, VideoSendStream::DegradationPreference::kMaintainResolution);
// Trigger initial configuration.
VideoEncoderConfig video_encoder_config;
video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
video_encoder_config.number_of_streams = 1;
video_encoder_config.video_stream_factory =
new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
kMaxPayloadLength, false);
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
EXPECT_EQ(
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
kLowFramerate);
// Trigger overuse, max framerate should be reduced.
VideoSendStream::Stats stats = stats_proxy_->GetStats();
stats.input_frame_rate = kLowFramerate;
stats_proxy_->SetMockStats(stats);
video_stream_encoder_->TriggerCpuOveruse();
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
int adapted_framerate =
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
EXPECT_LT(adapted_framerate, kLowFramerate);
// Reconfigure the encoder with a new (higher max framerate), max fps should
// still respect the adaptation.
video_encoder_config.video_stream_factory =
new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
kMaxPayloadLength, false);
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
EXPECT_EQ(
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
adapted_framerate);
// Trigger underuse, max framerate should go back to codec configured fps.
stats = stats_proxy_->GetStats();
stats.input_frame_rate = adapted_framerate;
stats_proxy_->SetMockStats(stats);
video_stream_encoder_->TriggerCpuNormalUsage();
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
EXPECT_EQ(
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
kHighFramerate);
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest,
OveruseDetectorUpdatedOnDegradationPreferenceChange) {
const int kFrameWidth = 1280;
@ -2380,9 +2232,7 @@ TEST_F(VideoStreamEncoderTest,
kMaxPayloadLength, false);
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
EXPECT_EQ(
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
kFramerate);
EXPECT_GT(source.sink_wants().max_framerate_fps, kFramerate);
// Trigger overuse, max framerate should be reduced.
VideoSendStream::Stats stats = stats_proxy_->GetStats();
@ -2390,8 +2240,8 @@ TEST_F(VideoStreamEncoderTest,
stats_proxy_->SetMockStats(stats);
video_stream_encoder_->TriggerCpuOveruse();
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
int adapted_framerate =
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
int adapted_framerate = source.sink_wants().max_framerate_fps;
EXPECT_LT(adapted_framerate, kFramerate);
// Change degradation preference to not enable framerate scaling. Target
@ -2399,9 +2249,7 @@ TEST_F(VideoStreamEncoderTest,
video_stream_encoder_->SetSource(
&source, VideoSendStream::DegradationPreference::kMaintainFramerate);
video_stream_encoder_->WaitUntilTaskQueueIsIdle();
EXPECT_EQ(
video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
kFramerate);
EXPECT_GT(source.sink_wants().max_framerate_fps, kFramerate);
video_stream_encoder_->Stop();
}