diff --git a/video/encoder_bitrate_adjuster.cc b/video/encoder_bitrate_adjuster.cc index 6fa35047ff..97fe363c08 100644 --- a/video/encoder_bitrate_adjuster.cc +++ b/video/encoder_bitrate_adjuster.cc @@ -98,9 +98,9 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( // encoder does not support temporal layers. Merge target bitrates for // this spatial layer. RTC_DCHECK(overshoot_detectors_[si][0]); - utilization_factor = - overshoot_detectors_[si][0]->GetUtilizationFactor(now_ms).value_or( - kDefaultUtilizationFactor); + utilization_factor = overshoot_detectors_[si][0] + ->GetNetworkRateUtilizationFactor(now_ms) + .value_or(kDefaultUtilizationFactor); } else if (spatial_layer_bitrate_bps > 0) { // Multiple temporal layers enabled for this spatial layer. Update rate // for each of them and make a weighted average of utilization factors, @@ -110,7 +110,8 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( for (size_t ti = 0; ti < active_tls_[si]; ++ti) { RTC_DCHECK(overshoot_detectors_[si][ti]); const absl::optional ti_utilization_factor = - overshoot_detectors_[si][ti]->GetUtilizationFactor(now_ms); + overshoot_detectors_[si][ti]->GetNetworkRateUtilizationFactor( + now_ms); if (!ti_utilization_factor) { utilization_factor = kDefaultUtilizationFactor; break; diff --git a/video/encoder_overshoot_detector.cc b/video/encoder_overshoot_detector.cc index f1fa3bf8f3..0b9926e18c 100644 --- a/video/encoder_overshoot_detector.cc +++ b/video/encoder_overshoot_detector.cc @@ -13,14 +13,22 @@ #include namespace webrtc { +namespace { +// The buffer level for media-rate utilization is allowed to go below zero, +// down to +// -(|kMaxMediaUnderrunFrames| / |target_framerate_fps_|) * |target_bitrate_|. +static constexpr double kMaxMediaUnderrunFrames = 5.0; +} // namespace EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms) : window_size_ms_(window_size_ms), time_last_update_ms_(-1), - sum_utilization_factors_(0.0), + sum_network_utilization_factors_(0.0), + sum_media_utilization_factors_(0.0), target_bitrate_(DataRate::Zero()), target_framerate_fps_(0), - buffer_level_bits_(0) {} + network_buffer_level_bits_(0), + media_buffer_level_bits_(0) {} EncoderOvershootDetector::~EncoderOvershootDetector() = default; @@ -34,8 +42,10 @@ void EncoderOvershootDetector::SetTargetRate(DataRate target_bitrate, // Stream was just enabled, reset state. time_last_update_ms_ = time_ms; utilization_factors_.clear(); - sum_utilization_factors_ = 0.0; - buffer_level_bits_ = 0; + sum_network_utilization_factors_ = 0.0; + sum_media_utilization_factors_ = 0.0; + network_buffer_level_bits_ = 0; + media_buffer_level_bits_ = 0; } target_bitrate_ = target_bitrate; @@ -48,57 +58,66 @@ void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) { LeakBits(time_ms); // Ideal size of a frame given the current rates. - const int64_t ideal_frame_size = IdealFrameSizeBits(); - if (ideal_frame_size == 0) { + const int64_t ideal_frame_size_bits = IdealFrameSizeBits(); + if (ideal_frame_size_bits == 0) { // Frame without updated bitrate and/or framerate, ignore it. return; } + const double network_utilization_factor = HandleEncodedFrame( + bytes * 8, ideal_frame_size_bits, time_ms, &network_buffer_level_bits_); + const double media_utilization_factor = HandleEncodedFrame( + bytes * 8, ideal_frame_size_bits, time_ms, &media_buffer_level_bits_); + + sum_network_utilization_factors_ += network_utilization_factor; + sum_media_utilization_factors_ += media_utilization_factor; + + utilization_factors_.emplace_back(network_utilization_factor, + media_utilization_factor, time_ms); +} + +double EncoderOvershootDetector::HandleEncodedFrame( + size_t frame_size_bits, + int64_t ideal_frame_size_bits, + int64_t time_ms, + int64_t* buffer_level_bits) const { // Add new frame to the buffer level. If doing so exceeds the ideal buffer // size, penalize this frame but cap overshoot to current buffer level rather // than size of this frame. This is done so that a single large frame is not // penalized if the encoder afterwards compensates by dropping frames and/or // reducing frame size. If however a large frame is followed by more data, // we cannot pace that next frame out within one frame space. - const int64_t bitsum = (bytes * 8) + buffer_level_bits_; + const int64_t bitsum = frame_size_bits + *buffer_level_bits; int64_t overshoot_bits = 0; - if (bitsum > ideal_frame_size) { - overshoot_bits = std::min(buffer_level_bits_, bitsum - ideal_frame_size); + if (bitsum > ideal_frame_size_bits) { + overshoot_bits = + std::min(*buffer_level_bits, bitsum - ideal_frame_size_bits); } // Add entry for the (over) utilization for this frame. Factor is capped // at 1.0 so that we don't risk overshooting on sudden changes. - double frame_utilization_factor; + double utilization_factor; if (utilization_factors_.empty()) { // First frame, cannot estimate overshoot based on previous one so // for this particular frame, just like as size vs optimal size. - frame_utilization_factor = - std::max(1.0, static_cast(bytes) * 8 / ideal_frame_size); + utilization_factor = std::max( + 1.0, static_cast(frame_size_bits) / ideal_frame_size_bits); } else { - frame_utilization_factor = - 1.0 + (static_cast(overshoot_bits) / ideal_frame_size); + utilization_factor = + 1.0 + (static_cast(overshoot_bits) / ideal_frame_size_bits); } - utilization_factors_.emplace_back(frame_utilization_factor, time_ms); - sum_utilization_factors_ += frame_utilization_factor; // Remove the overshot bits from the virtual buffer so we don't penalize // those bits multiple times. - buffer_level_bits_ -= overshoot_bits; - buffer_level_bits_ += bytes * 8; + *buffer_level_bits -= overshoot_bits; + *buffer_level_bits += frame_size_bits; + + return utilization_factor; } -absl::optional EncoderOvershootDetector::GetUtilizationFactor( - int64_t time_ms) { - // Cull old data points. - const int64_t cutoff_time_ms = time_ms - window_size_ms_; - while (!utilization_factors_.empty() && - utilization_factors_.front().update_time_ms < cutoff_time_ms) { - // Make sure sum is never allowed to become negative due rounding errors. - sum_utilization_factors_ = - std::max(0.0, sum_utilization_factors_ - - utilization_factors_.front().utilization_factor); - utilization_factors_.pop_front(); - } +absl::optional +EncoderOvershootDetector::GetNetworkRateUtilizationFactor(int64_t time_ms) { + CullOldUpdates(time_ms); // No data points within window, return. if (utilization_factors_.empty()) { @@ -107,16 +126,30 @@ absl::optional EncoderOvershootDetector::GetUtilizationFactor( // TODO(sprang): Consider changing from arithmetic mean to some other // function such as 90th percentile. - return sum_utilization_factors_ / utilization_factors_.size(); + return sum_network_utilization_factors_ / utilization_factors_.size(); +} + +absl::optional EncoderOvershootDetector::GetMediaRateUtilizationFactor( + int64_t time_ms) { + CullOldUpdates(time_ms); + + // No data points within window, return. + if (utilization_factors_.empty()) { + return absl::nullopt; + } + + return sum_media_utilization_factors_ / utilization_factors_.size(); } void EncoderOvershootDetector::Reset() { time_last_update_ms_ = -1; utilization_factors_.clear(); target_bitrate_ = DataRate::Zero(); - sum_utilization_factors_ = 0.0; + sum_network_utilization_factors_ = 0.0; + sum_media_utilization_factors_ = 0.0; target_framerate_fps_ = 0.0; - buffer_level_bits_ = 0; + network_buffer_level_bits_ = 0; + media_buffer_level_bits_ = 0; } int64_t EncoderOvershootDetector::IdealFrameSizeBits() const { @@ -134,11 +167,38 @@ void EncoderOvershootDetector::LeakBits(int64_t time_ms) { if (time_last_update_ms_ != -1 && target_bitrate_ > DataRate::Zero()) { int64_t time_delta_ms = time_ms - time_last_update_ms_; // Leak bits according to the current target bitrate. - int64_t leaked_bits = std::min( - buffer_level_bits_, (target_bitrate_.bps() * time_delta_ms) / 1000); - buffer_level_bits_ -= leaked_bits; + const int64_t leaked_bits = (target_bitrate_.bps() * time_delta_ms) / 1000; + + // Network buffer may not go below zero. + network_buffer_level_bits_ = + std::max(0, network_buffer_level_bits_ - leaked_bits); + + // Media buffer my go down to minus |kMaxMediaUnderrunFrames| frames worth + // of data. + const double max_underrun_seconds = + std::min(kMaxMediaUnderrunFrames, target_framerate_fps_) / + target_framerate_fps_; + media_buffer_level_bits_ = std::max( + -max_underrun_seconds * target_bitrate_.bps(), + media_buffer_level_bits_ - leaked_bits); } time_last_update_ms_ = time_ms; } +void EncoderOvershootDetector::CullOldUpdates(int64_t time_ms) { + // Cull old data points. + const int64_t cutoff_time_ms = time_ms - window_size_ms_; + while (!utilization_factors_.empty() && + utilization_factors_.front().update_time_ms < cutoff_time_ms) { + // Make sure sum is never allowed to become negative due rounding errors. + sum_network_utilization_factors_ = std::max( + 0.0, sum_network_utilization_factors_ - + utilization_factors_.front().network_utilization_factor); + sum_media_utilization_factors_ = std::max( + 0.0, sum_media_utilization_factors_ - + utilization_factors_.front().media_utilization_factor); + utilization_factors_.pop_front(); + } +} + } // namespace webrtc diff --git a/video/encoder_overshoot_detector.h b/video/encoder_overshoot_detector.h index 4fca2d9151..f7ef990830 100644 --- a/video/encoder_overshoot_detector.h +++ b/video/encoder_overshoot_detector.h @@ -26,28 +26,51 @@ class EncoderOvershootDetector { void SetTargetRate(DataRate target_bitrate, double target_framerate_fps, int64_t time_ms); + // A frame has been encoded or dropped. |bytes| == 0 indicates a drop. void OnEncodedFrame(size_t bytes, int64_t time_ms); - absl::optional GetUtilizationFactor(int64_t time_ms); + // This utilization factor reaches 1.0 only if the encoder produces encoded + // frame in such a way that they can be sent onto the network at + // |target_bitrate| without building growing queues. + absl::optional GetNetworkRateUtilizationFactor(int64_t time_ms); + // This utilization factor is based just on actual encoded frame sizes in + // relation to ideal sizes. An undershoot may be compensated by an + // overshoot so that the average over time is close to |target_bitrate|. + absl::optional GetMediaRateUtilizationFactor(int64_t time_ms); void Reset(); private: int64_t IdealFrameSizeBits() const; void LeakBits(int64_t time_ms); + void CullOldUpdates(int64_t time_ms); + // Updates provided buffer and checks if overuse ensues, returns + // the calculated utilization factor for this frame. + double HandleEncodedFrame(size_t frame_size_bits, + int64_t ideal_frame_size_bits, + int64_t time_ms, + int64_t* buffer_level_bits) const; const int64_t window_size_ms_; int64_t time_last_update_ms_; struct BitrateUpdate { - BitrateUpdate(double utilization_factor, int64_t update_time_ms) - : utilization_factor(utilization_factor), + BitrateUpdate(double network_utilization_factor, + double media_utilization_factor, + int64_t update_time_ms) + : network_utilization_factor(network_utilization_factor), + media_utilization_factor(media_utilization_factor), update_time_ms(update_time_ms) {} - double utilization_factor; + // The utilization factor based on strict network rate. + double network_utilization_factor; + // The utilization based on average media rate. + double media_utilization_factor; int64_t update_time_ms; }; std::deque utilization_factors_; - double sum_utilization_factors_; + double sum_network_utilization_factors_; + double sum_media_utilization_factors_; DataRate target_bitrate_; double target_framerate_fps_; - int64_t buffer_level_bits_; + int64_t network_buffer_level_bits_; + int64_t media_buffer_level_bits_; }; } // namespace webrtc diff --git a/video/encoder_overshoot_detector_unittest.cc b/video/encoder_overshoot_detector_unittest.cc index 3f393df91a..0c1a2989a1 100644 --- a/video/encoder_overshoot_detector_unittest.cc +++ b/video/encoder_overshoot_detector_unittest.cc @@ -51,10 +51,17 @@ class EncoderOvershootDetectorTest : public ::testing::Test { target_framerate_fps_); } - absl::optional utilization_factor = - detector_.GetUtilizationFactor(rtc::TimeMillis()); - EXPECT_NEAR(utilization_factor.value_or(-1), expected_utilization_factor, - allowed_error); + // At constant utilization, both network and media utilization should be + // close to expected. + const absl::optional network_utilization_factor = + detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()); + EXPECT_NEAR(network_utilization_factor.value_or(-1), + expected_utilization_factor, allowed_error); + + const absl::optional media_utilization_factor = + detector_.GetMediaRateUtilizationFactor(rtc::TimeMillis()); + EXPECT_NEAR(media_utilization_factor.value_or(-1), + expected_utilization_factor, allowed_error); } static constexpr int64_t kWindowSizeMs = 3000; @@ -71,11 +78,13 @@ TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) { rtc::TimeMillis()); // No data points, can't determine overshoot rate. - EXPECT_FALSE(detector_.GetUtilizationFactor(rtc::TimeMillis()).has_value()); + EXPECT_FALSE( + detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()).has_value()); detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis()); clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec * time_interval_ms); - EXPECT_TRUE(detector_.GetUtilizationFactor(rtc::TimeMillis()).has_value()); + EXPECT_TRUE( + detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()).has_value()); } TEST_F(EncoderOvershootDetectorTest, OptimalSize) { @@ -145,9 +154,14 @@ TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) { detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis()); } - absl::optional utilization_factor = - detector_.GetUtilizationFactor(rtc::TimeMillis()); - EXPECT_NEAR(utilization_factor.value_or(-1), 1.05, 0.01); -} + // Expect 5% overshoot for network rate, see above. + const absl::optional network_utilization_factor = + detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()); + EXPECT_NEAR(network_utilization_factor.value_or(-1), 1.05, 0.01); + // Expect media rate to be on average correct. + const absl::optional media_utilization_factor = + detector_.GetMediaRateUtilizationFactor(rtc::TimeMillis()); + EXPECT_NEAR(media_utilization_factor.value_or(-1), 1.00, 0.01); +} } // namespace webrtc