admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Track both "network" and "media" rate in EncoderOvershootDetector

Browse Source 
The network rate is based on a strict leaky buck with no underrun.
The media rate has almost the same algorithm, but allows the bufer level
to become negative down to -5 frames worth of data. This allows tracking
the average bitrate over a window and can be used as an alternative
upper bound when the encoder target is not link constrained.

Bug: webrtc:10155
Change-Id: I0d72157f7c0f0799e15572bf4797fa1b5636b22e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/129723
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27392}
This commit is contained in:
Erik Språng 2019-04-01 12:57:28 +02:00 committed by Commit Bot
parent a9daea4377
commit 6c072efe9f
4 changed files with 154 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
video/encoder_bitrate_adjuster.cc
View File

@ -98,9 +98,9 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
// encoder does not support temporal layers. Merge target bitrates for
// this spatial layer.
RTC_DCHECK(overshoot_detectors_[si][0]);
utilization_factor =
overshoot_detectors_[si][0]->GetUtilizationFactor(now_ms).value_or(
kDefaultUtilizationFactor);
utilization_factor = overshoot_detectors_[si][0]
->GetNetworkRateUtilizationFactor(now_ms)
.value_or(kDefaultUtilizationFactor);
} else if (spatial_layer_bitrate_bps > 0) {
// Multiple temporal layers enabled for this spatial layer. Update rate
// for each of them and make a weighted average of utilization factors,
@ -110,7 +110,8 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
for (size_t ti = 0; ti < active_tls_[si]; ++ti) {
RTC_DCHECK(overshoot_detectors_[si][ti]);
const absl::optional<double> ti_utilization_factor =
overshoot_detectors_[si][ti]->GetUtilizationFactor(now_ms);
overshoot_detectors_[si][ti]->GetNetworkRateUtilizationFactor(
now_ms);
if (!ti_utilization_factor) {
utilization_factor = kDefaultUtilizationFactor;
break;

132
video/encoder_overshoot_detector.cc
View File

@ -13,14 +13,22 @@
#include <algorithm>
namespace webrtc {
namespace {
// The buffer level for media-rate utilization is allowed to go below zero,
// down to
// -(|kMaxMediaUnderrunFrames| / |target_framerate_fps_|) * |target_bitrate_|.
static constexpr double kMaxMediaUnderrunFrames = 5.0;
} // namespace
EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms)
: window_size_ms_(window_size_ms),
time_last_update_ms_(-1),
sum_utilization_factors_(0.0),
sum_network_utilization_factors_(0.0),
sum_media_utilization_factors_(0.0),
target_bitrate_(DataRate::Zero()),
target_framerate_fps_(0),
buffer_level_bits_(0) {}
network_buffer_level_bits_(0),
media_buffer_level_bits_(0) {}
EncoderOvershootDetector::~EncoderOvershootDetector() = default;
@ -34,8 +42,10 @@ void EncoderOvershootDetector::SetTargetRate(DataRate target_bitrate,
// Stream was just enabled, reset state.
time_last_update_ms_ = time_ms;
utilization_factors_.clear();
sum_utilization_factors_ = 0.0;
buffer_level_bits_ = 0;
sum_network_utilization_factors_ = 0.0;
sum_media_utilization_factors_ = 0.0;
network_buffer_level_bits_ = 0;
media_buffer_level_bits_ = 0;
}
target_bitrate_ = target_bitrate;
@ -48,57 +58,66 @@ void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) {
LeakBits(time_ms);
// Ideal size of a frame given the current rates.
const int64_t ideal_frame_size = IdealFrameSizeBits();
if (ideal_frame_size == 0) {
const int64_t ideal_frame_size_bits = IdealFrameSizeBits();
if (ideal_frame_size_bits == 0) {
// Frame without updated bitrate and/or framerate, ignore it.
return;
}
const double network_utilization_factor = HandleEncodedFrame(
bytes * 8, ideal_frame_size_bits, time_ms, &network_buffer_level_bits_);
const double media_utilization_factor = HandleEncodedFrame(
bytes * 8, ideal_frame_size_bits, time_ms, &media_buffer_level_bits_);
sum_network_utilization_factors_ += network_utilization_factor;
sum_media_utilization_factors_ += media_utilization_factor;
utilization_factors_.emplace_back(network_utilization_factor,
media_utilization_factor, time_ms);
}
double EncoderOvershootDetector::HandleEncodedFrame(
size_t frame_size_bits,
int64_t ideal_frame_size_bits,
int64_t time_ms,
int64_t* buffer_level_bits) const {
// Add new frame to the buffer level. If doing so exceeds the ideal buffer
// size, penalize this frame but cap overshoot to current buffer level rather
// than size of this frame. This is done so that a single large frame is not
// penalized if the encoder afterwards compensates by dropping frames and/or
// reducing frame size. If however a large frame is followed by more data,
// we cannot pace that next frame out within one frame space.
const int64_t bitsum = (bytes * 8) + buffer_level_bits_;
const int64_t bitsum = frame_size_bits + *buffer_level_bits;
int64_t overshoot_bits = 0;
if (bitsum > ideal_frame_size) {
overshoot_bits = std::min(buffer_level_bits_, bitsum - ideal_frame_size);
if (bitsum > ideal_frame_size_bits) {
overshoot_bits =
std::min(*buffer_level_bits, bitsum - ideal_frame_size_bits);
}
// Add entry for the (over) utilization for this frame. Factor is capped
// at 1.0 so that we don't risk overshooting on sudden changes.
double frame_utilization_factor;
double utilization_factor;
if (utilization_factors_.empty()) {
// First frame, cannot estimate overshoot based on previous one so
// for this particular frame, just like as size vs optimal size.
frame_utilization_factor =
std::max(1.0, static_cast<double>(bytes) * 8 / ideal_frame_size);
utilization_factor = std::max(
1.0, static_cast<double>(frame_size_bits) / ideal_frame_size_bits);
} else {
frame_utilization_factor =
1.0 + (static_cast<double>(overshoot_bits) / ideal_frame_size);
utilization_factor =
1.0 + (static_cast<double>(overshoot_bits) / ideal_frame_size_bits);
}
utilization_factors_.emplace_back(frame_utilization_factor, time_ms);
sum_utilization_factors_ += frame_utilization_factor;
// Remove the overshot bits from the virtual buffer so we don't penalize
// those bits multiple times.
buffer_level_bits_ -= overshoot_bits;
buffer_level_bits_ += bytes * 8;
*buffer_level_bits -= overshoot_bits;
*buffer_level_bits += frame_size_bits;
return utilization_factor;
}
absl::optional<double> EncoderOvershootDetector::GetUtilizationFactor(
int64_t time_ms) {
// Cull old data points.
const int64_t cutoff_time_ms = time_ms - window_size_ms_;
while (!utilization_factors_.empty() &&
utilization_factors_.front().update_time_ms < cutoff_time_ms) {
// Make sure sum is never allowed to become negative due rounding errors.
sum_utilization_factors_ =
std::max(0.0, sum_utilization_factors_ -
utilization_factors_.front().utilization_factor);
utilization_factors_.pop_front();
}
absl::optional<double>
EncoderOvershootDetector::GetNetworkRateUtilizationFactor(int64_t time_ms) {
CullOldUpdates(time_ms);
// No data points within window, return.
if (utilization_factors_.empty()) {
@ -107,16 +126,30 @@ absl::optional<double> EncoderOvershootDetector::GetUtilizationFactor(
// TODO(sprang): Consider changing from arithmetic mean to some other
// function such as 90th percentile.
return sum_utilization_factors_ / utilization_factors_.size();
return sum_network_utilization_factors_ / utilization_factors_.size();
}
absl::optional<double> EncoderOvershootDetector::GetMediaRateUtilizationFactor(
int64_t time_ms) {
CullOldUpdates(time_ms);
// No data points within window, return.
if (utilization_factors_.empty()) {
return absl::nullopt;
}
return sum_media_utilization_factors_ / utilization_factors_.size();
}
void EncoderOvershootDetector::Reset() {
time_last_update_ms_ = -1;
utilization_factors_.clear();
target_bitrate_ = DataRate::Zero();
sum_utilization_factors_ = 0.0;
sum_network_utilization_factors_ = 0.0;
sum_media_utilization_factors_ = 0.0;
target_framerate_fps_ = 0.0;
buffer_level_bits_ = 0;
network_buffer_level_bits_ = 0;
media_buffer_level_bits_ = 0;
}
int64_t EncoderOvershootDetector::IdealFrameSizeBits() const {
@ -134,11 +167,38 @@ void EncoderOvershootDetector::LeakBits(int64_t time_ms) {
if (time_last_update_ms_ != -1 && target_bitrate_ > DataRate::Zero()) {
int64_t time_delta_ms = time_ms - time_last_update_ms_;
// Leak bits according to the current target bitrate.
int64_t leaked_bits = std::min(
buffer_level_bits_, (target_bitrate_.bps() * time_delta_ms) / 1000);
buffer_level_bits_ -= leaked_bits;
const int64_t leaked_bits = (target_bitrate_.bps() * time_delta_ms) / 1000;
// Network buffer may not go below zero.
network_buffer_level_bits_ =
std::max<int64_t>(0, network_buffer_level_bits_ - leaked_bits);
// Media buffer my go down to minus |kMaxMediaUnderrunFrames| frames worth
// of data.
const double max_underrun_seconds =
std::min(kMaxMediaUnderrunFrames, target_framerate_fps_) /
target_framerate_fps_;
media_buffer_level_bits_ = std::max<int64_t>(
-max_underrun_seconds * target_bitrate_.bps<int64_t>(),
media_buffer_level_bits_ - leaked_bits);
}
time_last_update_ms_ = time_ms;
}
void EncoderOvershootDetector::CullOldUpdates(int64_t time_ms) {
// Cull old data points.
const int64_t cutoff_time_ms = time_ms - window_size_ms_;
while (!utilization_factors_.empty() &&
utilization_factors_.front().update_time_ms < cutoff_time_ms) {
// Make sure sum is never allowed to become negative due rounding errors.
sum_network_utilization_factors_ = std::max(
0.0, sum_network_utilization_factors_ -
utilization_factors_.front().network_utilization_factor);
sum_media_utilization_factors_ = std::max(
0.0, sum_media_utilization_factors_ -
utilization_factors_.front().media_utilization_factor);
utilization_factors_.pop_front();
}
}
} // namespace webrtc

35
video/encoder_overshoot_detector.h
View File

@ -26,28 +26,51 @@ class EncoderOvershootDetector {
void SetTargetRate(DataRate target_bitrate,
double target_framerate_fps,
int64_t time_ms);
// A frame has been encoded or dropped. |bytes| == 0 indicates a drop.
void OnEncodedFrame(size_t bytes, int64_t time_ms);
absl::optional<double> GetUtilizationFactor(int64_t time_ms);
// This utilization factor reaches 1.0 only if the encoder produces encoded
// frame in such a way that they can be sent onto the network at
// |target_bitrate| without building growing queues.
absl::optional<double> GetNetworkRateUtilizationFactor(int64_t time_ms);
// This utilization factor is based just on actual encoded frame sizes in
// relation to ideal sizes. An undershoot may be compensated by an
// overshoot so that the average over time is close to |target_bitrate|.
absl::optional<double> GetMediaRateUtilizationFactor(int64_t time_ms);
void Reset();
private:
int64_t IdealFrameSizeBits() const;
void LeakBits(int64_t time_ms);
void CullOldUpdates(int64_t time_ms);
// Updates provided buffer and checks if overuse ensues, returns
// the calculated utilization factor for this frame.
double HandleEncodedFrame(size_t frame_size_bits,
int64_t ideal_frame_size_bits,
int64_t time_ms,
int64_t* buffer_level_bits) const;
const int64_t window_size_ms_;
int64_t time_last_update_ms_;
struct BitrateUpdate {
BitrateUpdate(double utilization_factor, int64_t update_time_ms)
: utilization_factor(utilization_factor),
BitrateUpdate(double network_utilization_factor,
double media_utilization_factor,
int64_t update_time_ms)
: network_utilization_factor(network_utilization_factor),
media_utilization_factor(media_utilization_factor),
update_time_ms(update_time_ms) {}
double utilization_factor;
// The utilization factor based on strict network rate.
double network_utilization_factor;
// The utilization based on average media rate.
double media_utilization_factor;
int64_t update_time_ms;
};
std::deque<BitrateUpdate> utilization_factors_;
double sum_utilization_factors_;
double sum_network_utilization_factors_;
double sum_media_utilization_factors_;
DataRate target_bitrate_;
double target_framerate_fps_;
int64_t buffer_level_bits_;
int64_t network_buffer_level_bits_;
int64_t media_buffer_level_bits_;
};
} // namespace webrtc

34
video/encoder_overshoot_detector_unittest.cc
View File

@ -51,10 +51,17 @@ class EncoderOvershootDetectorTest : public ::testing::Test {
target_framerate_fps_);
}
absl::optional<double> utilization_factor =
detector_.GetUtilizationFactor(rtc::TimeMillis());
EXPECT_NEAR(utilization_factor.value_or(-1), expected_utilization_factor,
allowed_error);
// At constant utilization, both network and media utilization should be
// close to expected.
const absl::optional<double> network_utilization_factor =
detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis());
EXPECT_NEAR(network_utilization_factor.value_or(-1),
expected_utilization_factor, allowed_error);
const absl::optional<double> media_utilization_factor =
detector_.GetMediaRateUtilizationFactor(rtc::TimeMillis());
EXPECT_NEAR(media_utilization_factor.value_or(-1),
expected_utilization_factor, allowed_error);
}
static constexpr int64_t kWindowSizeMs = 3000;
@ -71,11 +78,13 @@ TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) {
rtc::TimeMillis());
// No data points, can't determine overshoot rate.
EXPECT_FALSE(detector_.GetUtilizationFactor(rtc::TimeMillis()).has_value());
EXPECT_FALSE(
detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()).has_value());
detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis());
clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec * time_interval_ms);
EXPECT_TRUE(detector_.GetUtilizationFactor(rtc::TimeMillis()).has_value());
EXPECT_TRUE(
detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()).has_value());
}
TEST_F(EncoderOvershootDetectorTest, OptimalSize) {
@ -145,9 +154,14 @@ TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) {
detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis());
}
absl::optional<double> utilization_factor =
detector_.GetUtilizationFactor(rtc::TimeMillis());
EXPECT_NEAR(utilization_factor.value_or(-1), 1.05, 0.01);
}
// Expect 5% overshoot for network rate, see above.
const absl::optional<double> network_utilization_factor =
detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis());
EXPECT_NEAR(network_utilization_factor.value_or(-1), 1.05, 0.01);
// Expect media rate to be on average correct.
const absl::optional<double> media_utilization_factor =
detector_.GetMediaRateUtilizationFactor(rtc::TimeMillis());
EXPECT_NEAR(media_utilization_factor.value_or(-1), 1.00, 0.01);
}
} // namespace webrtc