diff --git a/video/BUILD.gn b/video/BUILD.gn index a9773f6895..7e85a556d0 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -390,6 +390,8 @@ rtc_library("video_stream_encoder_impl") { "frame_encode_metadata_writer.h", "quality_convergence_monitor.cc", "quality_convergence_monitor.h", + "rate_utilization_tracker.cc", + "rate_utilization_tracker.h", "video_source_sink_controller.cc", "video_source_sink_controller.h", "video_stream_encoder.cc", @@ -409,6 +411,9 @@ rtc_library("video_stream_encoder_impl") { "../api/task_queue:pending_task_safety_flag", "../api/task_queue:task_queue", "../api/units:data_rate", + "../api/units:data_size", + "../api/units:time_delta", + "../api/units:timestamp", "../api/video:encoded_image", "../api/video:render_resolution", "../api/video:video_adaptation", @@ -776,6 +781,7 @@ if (rtc_include_tests) { "quality_convergence_monitor_unittest.cc", "quality_limitation_reason_tracker_unittest.cc", "quality_scaling_tests.cc", + "rate_utilization_tracker_unittest.cc", "receive_statistics_proxy_unittest.cc", "report_block_stats_unittest.cc", "rtp_video_stream_receiver2_unittest.cc", @@ -841,6 +847,7 @@ if (rtc_include_tests) { "../api/test/metrics:metric", "../api/test/video:function_video_factory", "../api/units:data_rate", + "../api/units:data_size", "../api/units:frequency", "../api/units:time_delta", "../api/units:timestamp", diff --git a/video/rate_utilization_tracker.cc b/video/rate_utilization_tracker.cc new file mode 100644 index 0000000000..373493c1f0 --- /dev/null +++ b/video/rate_utilization_tracker.cc @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/rate_utilization_tracker.h" + +#include + +namespace webrtc { + +RateUtilizationTracker::RateUtilizationTracker( + size_t max_num_encoded_data_points, + TimeDelta max_duration) + : max_data_points_(max_num_encoded_data_points), + max_duration_(max_duration), + current_rate_(DataRate::Zero()) { + RTC_CHECK_GE(max_num_encoded_data_points, 0); + RTC_CHECK_GT(max_duration, TimeDelta::Zero()); +} + +void RateUtilizationTracker::OnDataRateChanged(DataRate rate, Timestamp time) { + current_rate_ = rate; + if (data_points_.empty()) { + // First entry should be contain first produced data, so just return after + // setting `current_rate_`. + return; + } else { + RateUsageUpdate& last_data_point = data_points_.back(); + RTC_CHECK_GE(time, last_data_point.time); + if (last_data_point.time == time) { + last_data_point.target_rate = rate; + } else { + data_points_.push_back({.time = time, + .target_rate = rate, + .produced_data = DataSize::Zero()}); + } + } + + CullOldData(time); +} + +void RateUtilizationTracker::OnDataProduced(DataSize size, Timestamp time) { + if (data_points_.empty()) { + data_points_.push_back( + {.time = time, .target_rate = current_rate_, .produced_data = size}); + } else { + RateUsageUpdate& last_data_point = data_points_.back(); + RTC_CHECK_GE(time, last_data_point.time); + if (last_data_point.time == time) { + last_data_point.produced_data += size; + } else { + data_points_.push_back( + {.time = time, .target_rate = current_rate_, .produced_data = size}); + } + } + + CullOldData(time); +} + +absl::optional RateUtilizationTracker::GetRateUtilizationFactor( + Timestamp time) const { + if (data_points_.empty()) { + return absl::nullopt; + } + + RTC_CHECK_GE(time, data_points_.back().time); + DataSize allocated_send_data_size = DataSize::Zero(); + DataSize total_produced_data = DataSize::Zero(); + + // Keep track of the last time data was produced - how much it was and how + // much rate budget has been allocated since then. + DataSize data_allocated_for_last_data = DataSize::Zero(); + DataSize size_of_last_data = DataSize::Zero(); + + RTC_DCHECK(!data_points_.front().produced_data.IsZero()); + for (size_t i = 0; i < data_points_.size(); ++i) { + const RateUsageUpdate& update = data_points_[i]; + total_produced_data += update.produced_data; + + DataSize allocated_since_previous_data_point = + i == 0 ? DataSize::Zero() + : (update.time - data_points_[i - 1].time) * + data_points_[i - 1].target_rate; + allocated_send_data_size += allocated_since_previous_data_point; + + if (update.produced_data.IsZero()) { + // Just a rate update past the last seen produced data. + data_allocated_for_last_data += allocated_since_previous_data_point; + } else { + // A newer data point with produced data, reset accumulator for rate + // allocated past the last data point. + size_of_last_data = update.produced_data; + data_allocated_for_last_data = DataSize::Zero(); + } + } + + if (allocated_send_data_size.IsZero() && current_rate_.IsZero()) { + // No allocated rate across all of the data points, ignore. + return absl::nullopt; + } + + // Calculate the rate past the very last data point until the polling time. + const RateUsageUpdate& last_update = data_points_.back(); + DataSize allocated_since_last_data_point = + (time - last_update.time) * last_update.target_rate; + + // If the last produced data packet is larger than the accumulated rate + // allocation window since then, use that data point size instead (minus any + // data rate accumulated in rate updates after that data point was produced). + allocated_send_data_size += + std::max(allocated_since_last_data_point, + size_of_last_data - data_allocated_for_last_data); + + return total_produced_data.bytes() / allocated_send_data_size.bytes(); +} + +void RateUtilizationTracker::CullOldData(Timestamp time) { + // Remove data points that are either too old, exceed the limit of number of + // data points - and make sure the first entry in the list contains actual + // data produced since we calculate send usage since that time. + + // We don't allow negative times so always start window at absolute time >= 0. + const Timestamp oldest_included_time = + time.ms() > max_duration_.ms() ? time - max_duration_ : Timestamp::Zero(); + + while (!data_points_.empty() && + (data_points_.front().time < oldest_included_time || + data_points_.size() > max_data_points_ || + data_points_.front().produced_data.IsZero())) { + data_points_.pop_front(); + } +} + +} // namespace webrtc diff --git a/video/rate_utilization_tracker.h b/video/rate_utilization_tracker.h new file mode 100644 index 0000000000..23f4088a05 --- /dev/null +++ b/video/rate_utilization_tracker.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_RATE_UTILIZATION_TRACKER_H_ +#define VIDEO_RATE_UTILIZATION_TRACKER_H_ + +#include + +#include "absl/types/optional.h" +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" + +namespace webrtc { + +// Helper class that tracks the rate of utilization over a sliding window. +// tl;dr: if an encoder has a target rate of 1000kbps but in practice +// produces 500kbps it would have a utilization factor of 0.5. +// The tracker looks only at discrete events, and keeps only a fixed amount +// of data points (e.g. encoded frames) or points newer than a given time +// limit, whichever is lower. + +// More precisely This class measures the allocated cumulative byte budget (as +// specified by one or more rate updates) and the actual cumulative number of +// bytes produced over a sliding window. A utilization factor (produced bytes / +// budgeted bytes) is calculated seen from the first data point timestamp until +// the last data point timestamp plus the amount time needed to send that last +// data point given no further updates to the rate. The implication of this is a +// smoother value, and e.g. setting a rate and adding a data point, then +// immediately querying the utilization reports 1.0 utilization instead of some +// undefined state. + +class RateUtilizationTracker { + public: + RateUtilizationTracker(size_t max_num_encoded_data_points, + TimeDelta max_duration); + + // The timestamps used should never decrease relative the last one. + void OnDataRateChanged(DataRate rate, Timestamp time); + void OnDataProduced(DataSize size, Timestamp time); + absl::optional GetRateUtilizationFactor(Timestamp time) const; + + private: + struct RateUsageUpdate { + Timestamp time; + DataRate target_rate; + DataSize produced_data; + }; + + void CullOldData(Timestamp time); + + const size_t max_data_points_; + const TimeDelta max_duration_; + DataRate current_rate_; + std::deque data_points_; +}; + +} // namespace webrtc + +#endif // VIDEO_RATE_UTILIZATION_TRACKER_H_ diff --git a/video/rate_utilization_tracker_unittest.cc b/video/rate_utilization_tracker_unittest.cc new file mode 100644 index 0000000000..33efbdd619 --- /dev/null +++ b/video/rate_utilization_tracker_unittest.cc @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/rate_utilization_tracker.h" + +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::Not; + +constexpr int kDefaultMaxDataPoints = 10; +constexpr TimeDelta kDefaultTimeWindow = TimeDelta::Seconds(1); +constexpr Timestamp kStartTime = Timestamp::Millis(9876654); +constexpr double kAllowedError = 0.002; // 0.2% error allowed. + +MATCHER_P(PrettyCloseTo, expected, "") { + return arg && std::abs(*arg - expected) < kAllowedError; +} + +TEST(RateUtilizationTrackerTest, NoDataInNoDataOut) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + EXPECT_FALSE(tracker.GetRateUtilizationFactor(kStartTime).has_value()); +} + +TEST(RateUtilizationTrackerTest, NoUtilizationWithoutDataPoints) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + tracker.OnDataRateChanged(DataRate::KilobitsPerSec(100), kStartTime); + EXPECT_FALSE(tracker.GetRateUtilizationFactor(kStartTime).has_value()); +} + +TEST(RateUtilizationTrackerTest, NoUtilizationWithoutRateUpdates) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + tracker.OnDataProduced(DataSize::Bytes(100), kStartTime); + EXPECT_FALSE(tracker.GetRateUtilizationFactor(kStartTime).has_value()); +} + +TEST(RateUtilizationTrackerTest, SingleDataPoint) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize, kStartTime); + + // From the start, the window is extended to cover the expected duration for + // the last frame - resulting in 100% utilization. + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime), PrettyCloseTo(1.0)); + + // At the expected frame interval the utilization is still 100%. + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + kFrameInterval), + PrettyCloseTo(1.0)); + + // After two frame intervals the utilization is half the expected. + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 2 * kFrameInterval), + PrettyCloseTo(0.5)); +} + +TEST(RateUtilizationTrackerTest, TwoDataPoints) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize, kStartTime); + tracker.OnDataProduced(kIdealFrameSize, kStartTime + kFrameInterval); + + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 2 * kFrameInterval), + PrettyCloseTo(1.0)); + + // After two three frame interval we have two utilizated intervals and one + // unitilzed => 2/3 utilization. + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 3 * kFrameInterval), + PrettyCloseTo(2.0 / 3.0)); +} + +TEST(RateUtilizationTrackerTest, TwoDataPointsConsistentOveruse) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize * 2, kStartTime); + tracker.OnDataProduced(kIdealFrameSize * 2, kStartTime + kFrameInterval); + + // Note that the last data point is presumed to be sent at the designated rate + // and no new data points produced until the buffers empty. Thus the + // overshoot is just 4/3 unstead of 4/2. + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 2 * kFrameInterval), + PrettyCloseTo(4.0 / 3.0)); +} + +TEST(RateUtilizationTrackerTest, OveruseWithFrameDrop) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + // First frame is 2x larger than it should be. + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize * 2, kStartTime); + // Compensate by dropping a frame before the next nominal-size one. + tracker.OnDataProduced(kIdealFrameSize, kStartTime + 2 * kFrameInterval); + + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 3 * kFrameInterval), + PrettyCloseTo(1.0)); +} + +TEST(RateUtilizationTrackerTest, VaryingRate) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + // Rate goes up, rate comes down... + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize, kStartTime); + tracker.OnDataRateChanged(kTargetRate * 2, kStartTime + kFrameInterval); + tracker.OnDataProduced(kIdealFrameSize * 2, kStartTime + kFrameInterval); + tracker.OnDataRateChanged(kTargetRate, kStartTime + 2 * kFrameInterval); + tracker.OnDataProduced(kIdealFrameSize, kStartTime + 2 * kFrameInterval); + + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 3 * kFrameInterval), + PrettyCloseTo(1.0)); +} + +TEST(RateUtilizationTrackerTest, VaryingRateMidFrameInterval) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + // First frame 1/3 too large + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize * (3.0 / 2.0), kStartTime); + + // Mid frame interval double the target rate. Should lead to no overshoot. + tracker.OnDataRateChanged(kTargetRate * 2, kStartTime + kFrameInterval / 2); + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + kFrameInterval), + PrettyCloseTo(1.0)); +} + +TEST(RateUtilizationTrackerTest, VaryingRateAfterLastDataPoint) { + RateUtilizationTracker tracker(kDefaultMaxDataPoints, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + tracker.OnDataRateChanged(kTargetRate, kStartTime); + // Data point is just after the rate update. + tracker.OnDataProduced(kIdealFrameSize, kStartTime + TimeDelta::Micros(1)); + + // Half an interval past the last frame double the target rate. + tracker.OnDataRateChanged(kTargetRate * 2, kStartTime + kFrameInterval / 2); + + // The last data point should now extend only to 2/3 the way to the next frame + // interval. + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + + kFrameInterval * (2.0 / 3.0)), + PrettyCloseTo(1.0)); + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + + kFrameInterval * (2.3 / 3.0)), + Not(PrettyCloseTo(1.0))); +} + +TEST(RateUtilizationTrackerTest, DataPointLimit) { + // Set max data points to two. + RateUtilizationTracker tracker(/*max_data_points=*/2, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + // Insert two frames that are too large. + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize * 2, kStartTime); + tracker.OnDataProduced(kIdealFrameSize * 2, kStartTime + 1 * kFrameInterval); + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 1 * kFrameInterval), + Not(PrettyCloseTo(1.0))); + + // Insert two frames of the correct size. Past grievances have been forgotten. + tracker.OnDataProduced(kIdealFrameSize, kStartTime + 2 * kFrameInterval); + tracker.OnDataProduced(kIdealFrameSize, kStartTime + 3 * kFrameInterval); + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 3 * kFrameInterval), + PrettyCloseTo(1.0)); +} + +TEST(RateUtilizationTrackerTest, WindowSizeLimit) { + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + // Number of data points enough, but time window too small. + RateUtilizationTracker tracker(/*max_data_points=*/4, /*time_window=*/ + 2 * kFrameInterval - TimeDelta::Millis(1)); + + // Insert two frames that are too large. + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize * 2, kStartTime); + tracker.OnDataProduced(kIdealFrameSize * 2, kStartTime + 1 * kFrameInterval); + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 1 * kFrameInterval), + Not(PrettyCloseTo(1.0))); + + // Insert two frames of the correct size. Past grievances have been forgotten. + tracker.OnDataProduced(kIdealFrameSize, kStartTime + 2 * kFrameInterval); + tracker.OnDataProduced(kIdealFrameSize, kStartTime + 3 * kFrameInterval); + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime + 3 * kFrameInterval), + PrettyCloseTo(1.0)); +} + +TEST(RateUtilizationTrackerTest, EqualTimestampsTreatedAtSameDataPoint) { + // Set max data points to two. + RateUtilizationTracker tracker(/*max_data_points=*/2, kDefaultTimeWindow); + constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / 33; + constexpr DataRate kTargetRate = DataRate::KilobitsPerSec(100); + constexpr DataSize kIdealFrameSize = kTargetRate * kFrameInterval; + + tracker.OnDataRateChanged(kTargetRate, kStartTime); + tracker.OnDataProduced(kIdealFrameSize, kStartTime); + EXPECT_THAT(tracker.GetRateUtilizationFactor(kStartTime), PrettyCloseTo(1.0)); + + // This is viewed as an undershoot. + tracker.OnDataProduced(kIdealFrameSize, kStartTime + (kFrameInterval * 2)); + EXPECT_THAT( + tracker.GetRateUtilizationFactor(kStartTime + (kFrameInterval * 2)), + PrettyCloseTo(2.0 / 3.0)); + + // Add the same data point again. Treated as layered frame so will accumulate + // in the same data point. This is expected to have a send time twice as long + // now, reducing the undershoot. + tracker.OnDataProduced(kIdealFrameSize, kStartTime + (kFrameInterval * 2)); + EXPECT_THAT( + tracker.GetRateUtilizationFactor(kStartTime + (kFrameInterval * 2)), + PrettyCloseTo(3.0 / 4.0)); +} + +} // namespace +} // namespace webrtc