diff --git a/rtc_base/experiments/rate_control_settings.cc b/rtc_base/experiments/rate_control_settings.cc index cb6e0dead4..0f19b788e0 100644 --- a/rtc_base/experiments/rate_control_settings.cc +++ b/rtc_base/experiments/rate_control_settings.cc @@ -121,11 +121,12 @@ RateControlSettings::RateControlSettings( ParseHysteresisFactor(key_value_config, kScreenshareHysteresisFieldTrialname, kDefaultScreenshareHysteresisFactor)), - probe_max_allocation_("probe_max_allocation", true) { + probe_max_allocation_("probe_max_allocation", true), + bitrate_adjuster_("bitrate_adjuster", false) { ParseFieldTrial( {&congestion_window_, &congestion_window_pushback_, &pacing_factor_, &alr_probing_, &trust_vp8_, &trust_vp9_, &video_hysteresis_, - &screenshare_hysteresis_, &probe_max_allocation_}, + &screenshare_hysteresis_, &probe_max_allocation_, &bitrate_adjuster_}, key_value_config->Lookup("WebRTC-VideoRateControl")); } @@ -206,4 +207,8 @@ bool RateControlSettings::TriggerProbeOnMaxAllocatedBitrateChange() const { return probe_max_allocation_.Get(); } +bool RateControlSettings::UseEncoderBitrateAdjuster() const { + return bitrate_adjuster_.Get(); +} + } // namespace webrtc diff --git a/rtc_base/experiments/rate_control_settings.h b/rtc_base/experiments/rate_control_settings.h index e7dc868591..e40c7aac97 100644 --- a/rtc_base/experiments/rate_control_settings.h +++ b/rtc_base/experiments/rate_control_settings.h @@ -50,6 +50,7 @@ class RateControlSettings final { VideoEncoderConfig::ContentType content_type) const; bool TriggerProbeOnMaxAllocatedBitrateChange() const; + bool UseEncoderBitrateAdjuster() const; private: explicit RateControlSettings( @@ -67,6 +68,7 @@ class RateControlSettings final { FieldTrialParameter video_hysteresis_; FieldTrialParameter screenshare_hysteresis_; FieldTrialParameter probe_max_allocation_; + FieldTrialParameter bitrate_adjuster_; }; } // namespace webrtc diff --git a/rtc_base/experiments/rate_control_settings_unittest.cc b/rtc_base/experiments/rate_control_settings_unittest.cc index ae9a1921e5..0d8c3767cd 100644 --- a/rtc_base/experiments/rate_control_settings_unittest.cc +++ b/rtc_base/experiments/rate_control_settings_unittest.cc @@ -124,6 +124,20 @@ TEST(RateControlSettingsTest, TriggerProbeOnMaxAllocatedBitrateChange) { .TriggerProbeOnMaxAllocatedBitrateChange()); } +TEST(RateControlSettingsTest, UseEncoderBitrateAdjuster) { + // Should be off by default. + EXPECT_FALSE( + RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()); + + { + // Can be turned on via field trial. + test::ScopedFieldTrials field_trials( + "WebRTC-VideoRateControl/bitrate_adjuster:true/"); + EXPECT_TRUE(RateControlSettings::ParseFromFieldTrials() + .UseEncoderBitrateAdjuster()); + } +} + } // namespace } // namespace webrtc diff --git a/video/BUILD.gn b/video/BUILD.gn index 9874742414..610a2fda42 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -169,6 +169,10 @@ rtc_source_set("video_stream_encoder_impl") { # visibility = [ "../api/video:video_stream_encoder_create" ] sources = [ + "encoder_bitrate_adjuster.cc", + "encoder_bitrate_adjuster.h", + "encoder_overshoot_detector.cc", + "encoder_overshoot_detector.h", "overuse_frame_detector.cc", "overuse_frame_detector.h", "partial_frame_assembler.cc", @@ -183,7 +187,9 @@ rtc_source_set("video_stream_encoder_impl") { } deps = [ + "../api/units:data_rate", "../api/video:encoded_image", + "../api/video:video_bitrate_allocation", "../api/video:video_bitrate_allocator", "../api/video:video_bitrate_allocator_factory", "../api/video:video_frame", @@ -205,6 +211,7 @@ rtc_source_set("video_stream_encoder_impl") { "../rtc_base:sequenced_task_checker", "../rtc_base:timeutils", "../rtc_base/experiments:quality_scaling_experiment", + "../rtc_base/experiments:rate_control_settings", "../rtc_base/system:fallthrough", "../rtc_base/task_utils:repeating_task", "../system_wrappers:field_trial", @@ -456,7 +463,9 @@ if (rtc_include_tests) { "buffered_frame_decryptor_unittest.cc", "call_stats_unittest.cc", "cpu_scaling_tests.cc", + "encoder_bitrate_adjuster_unittest.cc", "encoder_key_frame_callback_unittest.cc", + "encoder_overshoot_detector_unittest.cc", "end_to_end_tests/bandwidth_tests.cc", "end_to_end_tests/call_operation_tests.cc", "end_to_end_tests/codec_tests.cc", @@ -505,6 +514,7 @@ if (rtc_include_tests) { "../api:scoped_refptr", "../api:simulated_network_api", "../api/test/video:function_video_factory", + "../api/units:data_rate", "../api/video:builtin_video_bitrate_allocator_factory", "../api/video:encoded_image", "../api/video:video_bitrate_allocation", diff --git a/video/encoder_bitrate_adjuster.cc b/video/encoder_bitrate_adjuster.cc new file mode 100644 index 0000000000..eb6017c787 --- /dev/null +++ b/video/encoder_bitrate_adjuster.cc @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/encoder_bitrate_adjuster.h" + +#include + +#include "absl/memory/memory.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +constexpr int64_t EncoderBitrateAdjuster::kWindowSizeMs; +constexpr size_t EncoderBitrateAdjuster::kMinFramesSinceLayoutChange; +constexpr double EncoderBitrateAdjuster::kDefaultUtilizationFactor; + +EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings) + : current_total_framerate_fps_(0), + frames_since_layout_change_(0), + min_bitrates_bps_{} { + if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9) { + for (size_t si = 0; si < codec_settings.VP9().numberOfSpatialLayers; ++si) { + if (codec_settings.spatialLayers[si].active) { + min_bitrates_bps_[si] = + std::max(codec_settings.minBitrate * 1000, + codec_settings.spatialLayers[si].minBitrate * 1000); + } + } + } else { + for (size_t si = 0; si < codec_settings.numberOfSimulcastStreams; ++si) { + if (codec_settings.simulcastStream[si].active) { + min_bitrates_bps_[si] = + std::max(codec_settings.minBitrate * 1000, + codec_settings.simulcastStream[si].minBitrate * 1000); + } + } + } +} + +EncoderBitrateAdjuster::~EncoderBitrateAdjuster() = default; + +VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( + const VideoBitrateAllocation& bitrate_allocation, + int framerate_fps) { + current_bitrate_allocation_ = bitrate_allocation; + current_total_framerate_fps_ = framerate_fps; + + // First check that overshoot detectors exist, and store per spatial layer + // how many active temporal layers we have. + size_t active_tls_[kMaxSpatialLayers] = {}; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + active_tls_[si] = 0; + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + // Layer is enabled iff it has both positive bitrate and framerate target. + if (bitrate_allocation.GetBitrate(si, ti) > 0 && + current_fps_allocation_[si].size() > ti && + current_fps_allocation_[si][ti] > 0) { + ++active_tls_[si]; + if (!overshoot_detectors_[si][ti]) { + overshoot_detectors_[si][ti] = + absl::make_unique(kWindowSizeMs); + frames_since_layout_change_ = 0; + } + } else if (overshoot_detectors_[si][ti]) { + // Layer removed, destroy overshoot detector. + overshoot_detectors_[si][ti].reset(); + frames_since_layout_change_ = 0; + } + } + } + + // Next poll the overshoot detectors and populate the adjusted allocation. + const int64_t now_ms = rtc::TimeMillis(); + VideoBitrateAllocation adjusted_allocation; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + const uint32_t spatial_layer_bitrate_bps = + bitrate_allocation.GetSpatialLayerSum(si); + + // Adjustment is done per spatial layer only (not per temporal layer). + double utilization_factor; + if (frames_since_layout_change_ < kMinFramesSinceLayoutChange) { + utilization_factor = kDefaultUtilizationFactor; + } else if (active_tls_[si] == 0 || spatial_layer_bitrate_bps == 0) { + // No signaled temporal layers, or no bitrate set. Could either be unused + // spatial layer or bitrate dynamic mode; pass bitrate through without any + // change. + utilization_factor = 1.0; + } else if (active_tls_[si] == 1) { + // A single active temporal layer, this might mean single layer or that + // encoder does not support temporal layers. Merge target bitrates for + // this spatial layer. + RTC_DCHECK(overshoot_detectors_[si][0]); + utilization_factor = + overshoot_detectors_[si][0]->GetUtilizationFactor(now_ms).value_or( + kDefaultUtilizationFactor); + } else if (spatial_layer_bitrate_bps > 0) { + // Multiple temporal layers enabled for this spatial layer. Update rate + // for each of them and make a weighted average of utilization factors, + // with bitrate fraction used as weight. + // If any layer is missing a utilization factor, fall back to default. + utilization_factor = 0.0; + for (size_t ti = 0; ti < active_tls_[si]; ++ti) { + RTC_DCHECK(overshoot_detectors_[si][ti]); + const absl::optional ti_utilization_factor = + overshoot_detectors_[si][ti]->GetUtilizationFactor(now_ms); + if (!ti_utilization_factor) { + utilization_factor = kDefaultUtilizationFactor; + break; + } + const double weight = + static_cast(bitrate_allocation.GetBitrate(si, ti)) / + spatial_layer_bitrate_bps; + utilization_factor += weight * ti_utilization_factor.value(); + } + } else { + RTC_NOTREACHED(); + } + + // Don't boost target bitrate if encoder is under-using. + utilization_factor = std::max(utilization_factor, 1.0); + + // Don't reduce encoder target below 50%, in which case the frame dropper + // should kick in instead. + utilization_factor = std::min(utilization_factor, 2.0); + + if (min_bitrates_bps_[si] > 0 && spatial_layer_bitrate_bps > 0 && + min_bitrates_bps_[si] < spatial_layer_bitrate_bps) { + // Make sure rate adjuster doesn't push target bitrate below minimum. + utilization_factor = std::min( + utilization_factor, static_cast(spatial_layer_bitrate_bps) / + min_bitrates_bps_[si]); + } + + // Populate the adjusted allocation with determined utilization factor. + if (active_tls_[si] == 1 && + spatial_layer_bitrate_bps > bitrate_allocation.GetBitrate(si, 0)) { + // Bitrate allocation indicates temporal layer usage, but encoder + // does not seem to support it. Pipe all bitrate into a single + // overshoot detector. + uint32_t adjusted_layer_bitrate_bps = static_cast( + spatial_layer_bitrate_bps / utilization_factor + 0.5); + adjusted_allocation.SetBitrate(si, 0, adjusted_layer_bitrate_bps); + } else { + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (bitrate_allocation.HasBitrate(si, ti)) { + uint32_t adjusted_layer_bitrate_bps = static_cast( + bitrate_allocation.GetBitrate(si, ti) / utilization_factor + 0.5); + adjusted_allocation.SetBitrate(si, ti, adjusted_layer_bitrate_bps); + } + } + } + + // In case of rounding errors, add bitrate to TL0 until min bitrate + // constraint has been met. + const uint32_t adjusted_spatial_layer_sum = + adjusted_allocation.GetSpatialLayerSum(si); + if (spatial_layer_bitrate_bps > 0 && + adjusted_spatial_layer_sum < min_bitrates_bps_[si]) { + adjusted_allocation.SetBitrate(si, 0, + adjusted_allocation.GetBitrate(si, 0) + + min_bitrates_bps_[si] - + adjusted_spatial_layer_sum); + } + + // Update all detectors with the new adjusted bitrate targets. + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + const uint32_t layer_bitrate_bps = adjusted_allocation.GetBitrate(si, ti); + // Overshoot detector may not exist, eg for ScreenshareLayers case. + if (layer_bitrate_bps > 0 && overshoot_detectors_[si][ti]) { + // Number of frames in this layer alone is not cumulative, so + // subtract fps from any low temporal layer. + const double fps_fraction = + static_cast( + current_fps_allocation_[si][ti] - + (ti == 0 ? 0 : current_fps_allocation_[si][ti - 1])) / + VideoEncoder::EncoderInfo::kMaxFramerateFraction; + + overshoot_detectors_[si][ti]->SetTargetRate( + DataRate::bps(layer_bitrate_bps), + fps_fraction * current_total_framerate_fps_, now_ms); + } + } + } + + return adjusted_allocation; +} + +void EncoderBitrateAdjuster::OnEncoderInfo( + const VideoEncoder::EncoderInfo& encoder_info) { + // Copy allocation into current state and re-allocate. + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + current_fps_allocation_[si] = encoder_info.fps_allocation[si]; + } + + // Trigger re-allocation so that overshoot detectors have correct targets. + AdjustRateAllocation(current_bitrate_allocation_, + current_total_framerate_fps_); +} + +void EncoderBitrateAdjuster::OnEncodedFrame(const EncodedImage& encoded_image, + int temporal_index) { + ++frames_since_layout_change_; + // Detectors may not exist, for instance if ScreenshareLayers is used. + auto& detector = + overshoot_detectors_[encoded_image.SpatialIndex().value_or(0)] + [temporal_index]; + if (detector) { + detector->OnEncodedFrame(encoded_image.size(), rtc::TimeMillis()); + } +} + +void EncoderBitrateAdjuster::Reset() { + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + overshoot_detectors_[si][ti].reset(); + } + } + // Call AdjustRateAllocation() with the last know bitrate allocation, so that + // the appropriate overuse detectors are immediately re-created. + AdjustRateAllocation(current_bitrate_allocation_, + current_total_framerate_fps_); +} + +} // namespace webrtc diff --git a/video/encoder_bitrate_adjuster.h b/video/encoder_bitrate_adjuster.h new file mode 100644 index 0000000000..8901ad4034 --- /dev/null +++ b/video/encoder_bitrate_adjuster.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ENCODER_BITRATE_ADJUSTER_H_ +#define VIDEO_ENCODER_BITRATE_ADJUSTER_H_ + +#include + +#include "api/video/encoded_image.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/video_encoder.h" +#include "video/encoder_overshoot_detector.h" + +namespace webrtc { + +class EncoderBitrateAdjuster { + public: + // Size of sliding window used to track overshoot rate. + static constexpr int64_t kWindowSizeMs = 3000; + // Minimum number of frames since last layout change required to trust the + // overshoot statistics. Otherwise falls back to default utilization. + // By layout change, we mean any spatial/temporal layer being either enabled + // or disabled. + static constexpr size_t kMinFramesSinceLayoutChange = 30; + // Default utilization, before reliable metrics are available, is set to 20% + // overshoot. This is conservative so that badly misbehaving encoders don't + // build too much queue at the very start. + static constexpr double kDefaultUtilizationFactor = 1.2; + + explicit EncoderBitrateAdjuster(const VideoCodec& codec_settings); + ~EncoderBitrateAdjuster(); + + // Adjusts the given rate allocation to make it paceable within the target + // rates. + VideoBitrateAllocation AdjustRateAllocation( + const VideoBitrateAllocation& bitrate_allocation, + int framerate_fps); + + // Updated overuse detectors with data about the encoder, specifically about + // the temporal layer frame rate allocation. + void OnEncoderInfo(const VideoEncoder::EncoderInfo& encoder_info); + + // Updates the overuse detectors according to the encoded image size. + void OnEncodedFrame(const EncodedImage& encoded_image, int temporal_index); + + void Reset(); + + private: + VideoBitrateAllocation current_bitrate_allocation_; + int current_total_framerate_fps_; + // FPS allocation of temporal layers, per spatial layer. Represented as a Q8 + // fraction; 0 = 0%, 255 = 100%. See VideoEncoder::EncoderInfo.fps_allocation. + absl::InlinedVector + current_fps_allocation_[kMaxSpatialLayers]; + + // Frames since layout was changed, mean that any spatial or temporal layer + // was either disabled or enabled. + size_t frames_since_layout_change_; + std::unique_ptr + overshoot_detectors_[kMaxSpatialLayers][kMaxTemporalStreams]; + + // Minimum bitrates allowed, per spatial layer. + uint32_t min_bitrates_bps_[kMaxSpatialLayers]; +}; + +} // namespace webrtc + +#endif // VIDEO_ENCODER_BITRATE_ADJUSTER_H_ diff --git a/video/encoder_bitrate_adjuster_unittest.cc b/video/encoder_bitrate_adjuster_unittest.cc new file mode 100644 index 0000000000..312fde7cb3 --- /dev/null +++ b/video/encoder_bitrate_adjuster_unittest.cc @@ -0,0 +1,380 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/encoder_bitrate_adjuster.h" + +#include + +#include "absl/memory/memory.h" +#include "api/units/data_rate.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "test/gtest.h" + +namespace webrtc { + +class EncoderBitrateAdjusterTest : public ::testing::Test { + public: + static constexpr int64_t kWindowSizeMs = 3000; + static constexpr int kDefaultBitrateBps = 300000; + static constexpr int kDefaultFrameRateFps = 30; + EncoderBitrateAdjusterTest() + : target_bitrate_(DataRate::bps(kDefaultBitrateBps)), + target_framerate_fps_(kDefaultFrameRateFps), + tl_pattern_idx_{} {} + + protected: + void SetUpAdjuster(size_t num_spatial_layers, + size_t num_temporal_layers, + bool vp9_svc) { + // Initialize some default VideoCodec instance with the given number of + // layers. + if (vp9_svc) { + codec_.codecType = VideoCodecType::kVideoCodecVP9; + codec_.numberOfSimulcastStreams = 1; + codec_.VP9()->numberOfSpatialLayers = num_spatial_layers; + codec_.VP9()->numberOfTemporalLayers = num_temporal_layers; + for (size_t si = 0; si < num_spatial_layers; ++si) { + codec_.spatialLayers[si].minBitrate = 100 * (1 << si); + codec_.spatialLayers[si].targetBitrate = 200 * (1 << si); + codec_.spatialLayers[si].maxBitrate = 300 * (1 << si); + codec_.spatialLayers[si].active = true; + codec_.spatialLayers[si].numberOfTemporalLayers = num_temporal_layers; + } + } else { + codec_.codecType = VideoCodecType::kVideoCodecVP8; + codec_.numberOfSimulcastStreams = num_spatial_layers; + codec_.VP8()->numberOfTemporalLayers = num_temporal_layers; + for (size_t si = 0; si < num_spatial_layers; ++si) { + codec_.simulcastStream[si].minBitrate = 100 * (1 << si); + codec_.simulcastStream[si].targetBitrate = 200 * (1 << si); + codec_.simulcastStream[si].maxBitrate = 300 * (1 << si); + codec_.simulcastStream[si].active = true; + codec_.simulcastStream[si].numberOfTemporalLayers = num_temporal_layers; + } + } + + for (size_t si = 0; si < num_spatial_layers; ++si) { + encoder_info_.fps_allocation[si].resize(num_temporal_layers); + double fraction = 1.0; + for (int ti = num_temporal_layers - 1; ti >= 0; --ti) { + encoder_info_.fps_allocation[si][ti] = static_cast( + VideoEncoder::EncoderInfo::kMaxFramerateFraction * fraction + 0.5); + fraction /= 2.0; + } + } + + adjuster_ = absl::make_unique(codec_); + adjuster_->OnEncoderInfo(encoder_info_); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + } + + void InsertFrames(std::vector> utilization_factors, + int64_t duration_ms) { + constexpr size_t kMaxFrameSize = 100000; + uint8_t buffer[kMaxFrameSize]; + + const int64_t start_us = rtc::TimeMicros(); + while (rtc::TimeMicros() < + start_us + (duration_ms * rtc::kNumMicrosecsPerMillisec)) { + clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / + target_framerate_fps_); + for (size_t si = 0; si < NumSpatialLayers(); ++si) { + const std::vector& tl_pattern = + kTlPatterns[NumTemporalLayers(si) - 1]; + const size_t ti = + tl_pattern[(tl_pattern_idx_[si]++) % tl_pattern.size()]; + + uint32_t layer_bitrate_bps = + current_adjusted_allocation_.GetBitrate(si, ti); + double layer_framerate_fps = target_framerate_fps_; + if (encoder_info_.fps_allocation[si].size() > ti) { + uint8_t layer_fps_fraction = encoder_info_.fps_allocation[si][ti]; + if (ti > 0) { + // We're interested in the frame rate for this layer only, not + // cumulative frame rate. + layer_fps_fraction -= encoder_info_.fps_allocation[si][ti - 1]; + } + layer_framerate_fps = + (target_framerate_fps_ * layer_fps_fraction) / + VideoEncoder::EncoderInfo::kMaxFramerateFraction; + } + double utilization_factor = 1.0; + if (utilization_factors.size() > si && + utilization_factors[si].size() > ti) { + utilization_factor = utilization_factors[si][ti]; + } + size_t frame_size_bytes = utilization_factor * + (layer_bitrate_bps / 8.0) / + layer_framerate_fps; + + EncodedImage image(buffer, 0, kMaxFrameSize); + image.set_size(frame_size_bytes); + image.SetSpatialIndex(si); + adjuster_->OnEncodedFrame(image, ti); + } + } + } + + size_t NumSpatialLayers() const { + if (codec_.codecType == VideoCodecType::kVideoCodecVP9) { + return codec_.VP9().numberOfSpatialLayers; + } + return codec_.numberOfSimulcastStreams; + } + + size_t NumTemporalLayers(int spatial_index) { + if (codec_.codecType == VideoCodecType::kVideoCodecVP9) { + return codec_.spatialLayers[spatial_index].numberOfTemporalLayers; + } + return codec_.simulcastStream[spatial_index].numberOfTemporalLayers; + } + + void ExpectNear(const VideoBitrateAllocation& expected_allocation, + const VideoBitrateAllocation& actual_allocation, + double allowed_error_fraction) { + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (expected_allocation.HasBitrate(si, ti)) { + EXPECT_TRUE(actual_allocation.HasBitrate(si, ti)); + uint32_t expected_layer_bitrate_bps = + expected_allocation.GetBitrate(si, ti); + EXPECT_NEAR(expected_layer_bitrate_bps, + actual_allocation.GetBitrate(si, ti), + static_cast(expected_layer_bitrate_bps * + allowed_error_fraction)); + } else { + EXPECT_FALSE(actual_allocation.HasBitrate(si, ti)); + } + } + } + } + + VideoBitrateAllocation MultiplyAllocation( + const VideoBitrateAllocation& allocation, + double factor) { + VideoBitrateAllocation multiplied_allocation; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (allocation.HasBitrate(si, ti)) { + multiplied_allocation.SetBitrate( + si, ti, + static_cast(factor * allocation.GetBitrate(si, ti) + + 0.5)); + } + } + } + return multiplied_allocation; + } + + VideoCodec codec_; + VideoEncoder::EncoderInfo encoder_info_; + std::unique_ptr adjuster_; + VideoBitrateAllocation current_input_allocation_; + VideoBitrateAllocation current_adjusted_allocation_; + rtc::ScopedFakeClock clock_; + DataRate target_bitrate_; + double target_framerate_fps_; + int tl_pattern_idx_[kMaxSpatialLayers]; + + const std::vector kTlPatterns[kMaxTemporalStreams] = { + {0}, + {0, 1}, + {0, 2, 1, 2}, + {0, 3, 2, 3, 1, 3, 2, 3}}; +}; + +TEST_F(EncoderBitrateAdjusterTest, SingleLayerOptimal) { + // Single layer, well behaved encoder. + current_input_allocation_.SetBitrate(0, 0, 300000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 1, false); + InsertFrames({{1.0}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + // Adjusted allocation near input. Allow 1% error margin due to rounding + // errors etc. + ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01); +} + +TEST_F(EncoderBitrateAdjusterTest, SingleLayerOveruse) { + // Single layer, well behaved encoder. + current_input_allocation_.SetBitrate(0, 0, 300000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 1, false); + InsertFrames({{1.2}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + // Adjusted allocation lowered by 20%. + ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.2), + current_adjusted_allocation_, 0.01); +} + +TEST_F(EncoderBitrateAdjusterTest, SingleLayerUnderuse) { + // Single layer, well behaved encoder. + current_input_allocation_.SetBitrate(0, 0, 300000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 1, false); + InsertFrames({{0.5}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + // Undershoot, adjusted should exactly match input. + ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.00); +} + +TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersOptimalSize) { + // Three temporal layers, 60%/20%/20% bps distro, well behaved encoder. + current_input_allocation_.SetBitrate(0, 0, 180000); + current_input_allocation_.SetBitrate(0, 1, 60000); + current_input_allocation_.SetBitrate(0, 2, 60000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 3, false); + InsertFrames({{1.0, 1.0, 1.0}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01); +} + +TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersOvershoot) { + // Three temporal layers, 60%/20%/20% bps distro. + // 10% overshoot on all layers. + current_input_allocation_.SetBitrate(0, 0, 180000); + current_input_allocation_.SetBitrate(0, 1, 60000); + current_input_allocation_.SetBitrate(0, 2, 60000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 3, false); + InsertFrames({{1.1, 1.1, 1.1}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + // Adjusted allocation lowered by 10%. + ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.1), + current_adjusted_allocation_, 0.01); +} + +TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersUndershoot) { + // Three temporal layers, 60%/20%/20% bps distro, undershoot all layers. + current_input_allocation_.SetBitrate(0, 0, 180000); + current_input_allocation_.SetBitrate(0, 1, 60000); + current_input_allocation_.SetBitrate(0, 2, 60000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 3, false); + InsertFrames({{0.8, 0.8, 0.8}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + // Adjusted allocation identical since we don't boost bitrates. + ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.0); +} + +TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersSkewedOvershoot) { + // Three temporal layers, 60%/20%/20% bps distro. + // 10% overshoot on base layer, 20% on higher layers. + current_input_allocation_.SetBitrate(0, 0, 180000); + current_input_allocation_.SetBitrate(0, 1, 60000); + current_input_allocation_.SetBitrate(0, 2, 60000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 3, false); + InsertFrames({{1.1, 1.2, 1.2}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + // Expected overshoot is weighted by bitrate: + // (0.6 * 1.1 + 0.2 * 1.2 + 0.2 * 1.2) = 1.14 + ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.14), + current_adjusted_allocation_, 0.01); +} + +TEST_F(EncoderBitrateAdjusterTest, FourTemporalLayersSkewedOvershoot) { + // Three temporal layers, 40%/30%/15%/15% bps distro. + // 10% overshoot on base layer, 20% on higher layers. + current_input_allocation_.SetBitrate(0, 0, 120000); + current_input_allocation_.SetBitrate(0, 1, 90000); + current_input_allocation_.SetBitrate(0, 2, 45000); + current_input_allocation_.SetBitrate(0, 3, 45000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 4, false); + InsertFrames({{1.1, 1.2, 1.2, 1.2}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + // Expected overshoot is weighted by bitrate: + // (0.4 * 1.1 + 0.3 * 1.2 + 0.15 * 1.2 + 0.15 * 1.2) = 1.16 + ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.16), + current_adjusted_allocation_, 0.01); +} + +TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersNonLayeredEncoder) { + // Three temporal layers, 60%/20%/20% bps allocation, 10% overshoot, + // encoder does not actually support temporal layers. + current_input_allocation_.SetBitrate(0, 0, 180000); + current_input_allocation_.SetBitrate(0, 1, 60000); + current_input_allocation_.SetBitrate(0, 2, 60000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 1, false); + InsertFrames({{1.1}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + // Expect the actual 10% overuse to be detected and the allocation to + // only contain the one entry. + VideoBitrateAllocation expected_allocation; + expected_allocation.SetBitrate( + 0, 0, + static_cast(current_input_allocation_.get_sum_bps() / 1.10)); + ExpectNear(expected_allocation, current_adjusted_allocation_, 0.01); +} + +TEST_F(EncoderBitrateAdjusterTest, IgnoredStream) { + // Encoder with three temporal layers, but in a mode that does not support + // deterministic frame rate. Those are ignored, even if bitrate overshoots. + current_input_allocation_.SetBitrate(0, 0, 180000); + current_input_allocation_.SetBitrate(0, 1, 60000); + target_framerate_fps_ = 30; + SetUpAdjuster(1, 1, false); + encoder_info_.fps_allocation[0].clear(); + adjuster_->OnEncoderInfo(encoder_info_); + + InsertFrames({{1.1}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + + // Values passed through. + ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.00); +} + +TEST_F(EncoderBitrateAdjusterTest, DifferentSpatialOvershoots) { + // Two streams, both with three temporal layers. + // S0 has 5% overshoot, S1 has 25% overshoot. + current_input_allocation_.SetBitrate(0, 0, 180000); + current_input_allocation_.SetBitrate(0, 1, 60000); + current_input_allocation_.SetBitrate(0, 2, 60000); + current_input_allocation_.SetBitrate(1, 0, 400000); + current_input_allocation_.SetBitrate(1, 1, 150000); + current_input_allocation_.SetBitrate(1, 2, 150000); + target_framerate_fps_ = 30; + // Run twice, once configured as simulcast and once as VP9 SVC. + for (int i = 0; i < 2; ++i) { + SetUpAdjuster(2, 3, i == 0); + InsertFrames({{1.05, 1.05, 1.05}, {1.25, 1.25, 1.25}}, kWindowSizeMs); + current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( + current_input_allocation_, target_framerate_fps_); + VideoBitrateAllocation expected_allocation; + for (size_t ti = 0; ti < 3; ++ti) { + expected_allocation.SetBitrate( + 0, ti, + static_cast(current_input_allocation_.GetBitrate(0, ti) / + 1.05)); + expected_allocation.SetBitrate( + 1, ti, + static_cast(current_input_allocation_.GetBitrate(1, ti) / + 1.25)); + } + ExpectNear(expected_allocation, current_adjusted_allocation_, 0.01); + } +} + +} // namespace webrtc diff --git a/video/encoder_overshoot_detector.cc b/video/encoder_overshoot_detector.cc new file mode 100644 index 0000000000..f1fa3bf8f3 --- /dev/null +++ b/video/encoder_overshoot_detector.cc @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/encoder_overshoot_detector.h" + +#include + +namespace webrtc { + +EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms) + : window_size_ms_(window_size_ms), + time_last_update_ms_(-1), + sum_utilization_factors_(0.0), + target_bitrate_(DataRate::Zero()), + target_framerate_fps_(0), + buffer_level_bits_(0) {} + +EncoderOvershootDetector::~EncoderOvershootDetector() = default; + +void EncoderOvershootDetector::SetTargetRate(DataRate target_bitrate, + double target_framerate_fps, + int64_t time_ms) { + // First leak bits according to the previous target rate. + if (target_bitrate_ != DataRate::Zero()) { + LeakBits(time_ms); + } else if (target_bitrate != DataRate::Zero()) { + // Stream was just enabled, reset state. + time_last_update_ms_ = time_ms; + utilization_factors_.clear(); + sum_utilization_factors_ = 0.0; + buffer_level_bits_ = 0; + } + + target_bitrate_ = target_bitrate; + target_framerate_fps_ = target_framerate_fps; +} + +void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) { + // Leak bits from the virtual pacer buffer, according to the current target + // bitrate. + LeakBits(time_ms); + + // Ideal size of a frame given the current rates. + const int64_t ideal_frame_size = IdealFrameSizeBits(); + if (ideal_frame_size == 0) { + // Frame without updated bitrate and/or framerate, ignore it. + return; + } + + // Add new frame to the buffer level. If doing so exceeds the ideal buffer + // size, penalize this frame but cap overshoot to current buffer level rather + // than size of this frame. This is done so that a single large frame is not + // penalized if the encoder afterwards compensates by dropping frames and/or + // reducing frame size. If however a large frame is followed by more data, + // we cannot pace that next frame out within one frame space. + const int64_t bitsum = (bytes * 8) + buffer_level_bits_; + int64_t overshoot_bits = 0; + if (bitsum > ideal_frame_size) { + overshoot_bits = std::min(buffer_level_bits_, bitsum - ideal_frame_size); + } + + // Add entry for the (over) utilization for this frame. Factor is capped + // at 1.0 so that we don't risk overshooting on sudden changes. + double frame_utilization_factor; + if (utilization_factors_.empty()) { + // First frame, cannot estimate overshoot based on previous one so + // for this particular frame, just like as size vs optimal size. + frame_utilization_factor = + std::max(1.0, static_cast(bytes) * 8 / ideal_frame_size); + } else { + frame_utilization_factor = + 1.0 + (static_cast(overshoot_bits) / ideal_frame_size); + } + utilization_factors_.emplace_back(frame_utilization_factor, time_ms); + sum_utilization_factors_ += frame_utilization_factor; + + // Remove the overshot bits from the virtual buffer so we don't penalize + // those bits multiple times. + buffer_level_bits_ -= overshoot_bits; + buffer_level_bits_ += bytes * 8; +} + +absl::optional EncoderOvershootDetector::GetUtilizationFactor( + int64_t time_ms) { + // Cull old data points. + const int64_t cutoff_time_ms = time_ms - window_size_ms_; + while (!utilization_factors_.empty() && + utilization_factors_.front().update_time_ms < cutoff_time_ms) { + // Make sure sum is never allowed to become negative due rounding errors. + sum_utilization_factors_ = + std::max(0.0, sum_utilization_factors_ - + utilization_factors_.front().utilization_factor); + utilization_factors_.pop_front(); + } + + // No data points within window, return. + if (utilization_factors_.empty()) { + return absl::nullopt; + } + + // TODO(sprang): Consider changing from arithmetic mean to some other + // function such as 90th percentile. + return sum_utilization_factors_ / utilization_factors_.size(); +} + +void EncoderOvershootDetector::Reset() { + time_last_update_ms_ = -1; + utilization_factors_.clear(); + target_bitrate_ = DataRate::Zero(); + sum_utilization_factors_ = 0.0; + target_framerate_fps_ = 0.0; + buffer_level_bits_ = 0; +} + +int64_t EncoderOvershootDetector::IdealFrameSizeBits() const { + if (target_framerate_fps_ <= 0 || target_bitrate_ == DataRate::Zero()) { + return 0; + } + + // Current ideal frame size, based on the current target bitrate. + return static_cast( + (target_bitrate_.bps() + target_framerate_fps_ / 2) / + target_framerate_fps_); +} + +void EncoderOvershootDetector::LeakBits(int64_t time_ms) { + if (time_last_update_ms_ != -1 && target_bitrate_ > DataRate::Zero()) { + int64_t time_delta_ms = time_ms - time_last_update_ms_; + // Leak bits according to the current target bitrate. + int64_t leaked_bits = std::min( + buffer_level_bits_, (target_bitrate_.bps() * time_delta_ms) / 1000); + buffer_level_bits_ -= leaked_bits; + } + time_last_update_ms_ = time_ms; +} + +} // namespace webrtc diff --git a/video/encoder_overshoot_detector.h b/video/encoder_overshoot_detector.h new file mode 100644 index 0000000000..4fca2d9151 --- /dev/null +++ b/video/encoder_overshoot_detector.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ENCODER_OVERSHOOT_DETECTOR_H_ +#define VIDEO_ENCODER_OVERSHOOT_DETECTOR_H_ + +#include + +#include "absl/types/optional.h" +#include "api/units/data_rate.h" + +namespace webrtc { + +class EncoderOvershootDetector { + public: + explicit EncoderOvershootDetector(int64_t window_size_ms); + ~EncoderOvershootDetector(); + + void SetTargetRate(DataRate target_bitrate, + double target_framerate_fps, + int64_t time_ms); + void OnEncodedFrame(size_t bytes, int64_t time_ms); + absl::optional GetUtilizationFactor(int64_t time_ms); + void Reset(); + + private: + int64_t IdealFrameSizeBits() const; + void LeakBits(int64_t time_ms); + + const int64_t window_size_ms_; + int64_t time_last_update_ms_; + struct BitrateUpdate { + BitrateUpdate(double utilization_factor, int64_t update_time_ms) + : utilization_factor(utilization_factor), + update_time_ms(update_time_ms) {} + double utilization_factor; + int64_t update_time_ms; + }; + std::deque utilization_factors_; + double sum_utilization_factors_; + DataRate target_bitrate_; + double target_framerate_fps_; + int64_t buffer_level_bits_; +}; + +} // namespace webrtc + +#endif // VIDEO_ENCODER_OVERSHOOT_DETECTOR_H_ diff --git a/video/encoder_overshoot_detector_unittest.cc b/video/encoder_overshoot_detector_unittest.cc new file mode 100644 index 0000000000..3f393df91a --- /dev/null +++ b/video/encoder_overshoot_detector_unittest.cc @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/encoder_overshoot_detector.h" +#include "api/units/data_rate.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/time_utils.h" +#include "test/gtest.h" + +namespace webrtc { + +class EncoderOvershootDetectorTest : public ::testing::Test { + public: + static constexpr int kDefaultBitrateBps = 300000; + static constexpr double kDefaultFrameRateFps = 15; + EncoderOvershootDetectorTest() + : detector_(kWindowSizeMs), + target_bitrate_(DataRate::bps(kDefaultBitrateBps)), + target_framerate_fps_(kDefaultFrameRateFps) {} + + protected: + void RunConstantUtilizationTest(double actual_utilization_factor, + double expected_utilization_factor, + double allowed_error, + int64_t test_duration_ms) { + const int frame_size_bytes = + static_cast(actual_utilization_factor * + (target_bitrate_.bps() / target_framerate_fps_) / 8); + detector_.SetTargetRate(target_bitrate_, target_framerate_fps_, + rtc::TimeMillis()); + + if (rtc::TimeMillis() == 0) { + // Encode a first frame which by definition has no overuse factor. + detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis()); + clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / + target_framerate_fps_); + } + + int64_t runtime_us = 0; + while (runtime_us < test_duration_ms * 1000) { + detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis()); + runtime_us += rtc::kNumMicrosecsPerSec / target_framerate_fps_; + clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / + target_framerate_fps_); + } + + absl::optional utilization_factor = + detector_.GetUtilizationFactor(rtc::TimeMillis()); + EXPECT_NEAR(utilization_factor.value_or(-1), expected_utilization_factor, + allowed_error); + } + + static constexpr int64_t kWindowSizeMs = 3000; + EncoderOvershootDetector detector_; + rtc::ScopedFakeClock clock_; + DataRate target_bitrate_; + double target_framerate_fps_; +}; + +TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) { + const int frame_size_bytes = 1000; + const int64_t time_interval_ms = 33; + detector_.SetTargetRate(target_bitrate_, target_framerate_fps_, + rtc::TimeMillis()); + + // No data points, can't determine overshoot rate. + EXPECT_FALSE(detector_.GetUtilizationFactor(rtc::TimeMillis()).has_value()); + + detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis()); + clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec * time_interval_ms); + EXPECT_TRUE(detector_.GetUtilizationFactor(rtc::TimeMillis()).has_value()); +} + +TEST_F(EncoderOvershootDetectorTest, OptimalSize) { + // Optimally behaved encoder. + // Allow some error margin due to rounding errors, eg due to frame + // interval not being an integer. + RunConstantUtilizationTest(1.0, 1.0, 0.01, kWindowSizeMs); +} + +TEST_F(EncoderOvershootDetectorTest, Undershoot) { + // Undershoot, reported utilization factor should be capped to 1.0 so + // that we don't incorrectly boost encoder bitrate during movement. + RunConstantUtilizationTest(0.5, 1.0, 0.00, kWindowSizeMs); +} + +TEST_F(EncoderOvershootDetectorTest, Overshoot) { + // Overshoot by 20%. + // Allow some error margin due to rounding errors. + RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs); +} + +TEST_F(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) { + // Overshoot by 20%, but vary framerate and bitrate. + // Allow some error margin due to rounding errors. + RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs); + target_framerate_fps_ /= 2; + RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2); + target_bitrate_ = DataRate::bps(target_bitrate_.bps() / 2); + RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2); +} + +TEST_F(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) { + // Overshoot by 10%, keep framerate and bitrate constant. + // Allow some error margin due to rounding errors. + RunConstantUtilizationTest(1.1, 1.1, 0.01, kWindowSizeMs); + // Change overshoot to 20%, run for half window and expect overshoot + // to be 15%. + RunConstantUtilizationTest(1.2, 1.15, 0.01, kWindowSizeMs / 2); + // Keep running at 20% overshoot, after window is full that should now + // be the reported overshoot. + RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2); +} + +TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) { + const int ideal_frame_size_bytes = + (target_bitrate_.bps() / target_framerate_fps_) / 8; + detector_.SetTargetRate(target_bitrate_, target_framerate_fps_, + rtc::TimeMillis()); + + // Test scenario with average bitrate matching the target bitrate, but + // with some utilization factor penalty as the frames can't be paced out + // on the network at the target rate. + // Insert a series of four frames: + // 1) 20% overshoot, not penalized as buffer if empty. + // 2) 20% overshoot, the 20% overshoot from the first frame is penalized. + // 3) 20% undershoot, negating the overshoot from the last frame. + // 4) 20% undershoot, no penalty. + // On average then utilization penalty is thus 5%. + + int64_t runtime_us = 0; + int i = 0; + while (runtime_us < kWindowSizeMs * rtc::kNumMicrosecsPerMillisec) { + runtime_us += rtc::kNumMicrosecsPerSec / target_framerate_fps_; + clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / target_framerate_fps_); + int frame_size_bytes = (i++ % 4 < 2) ? (ideal_frame_size_bytes * 120) / 100 + : (ideal_frame_size_bytes * 80) / 100; + detector_.OnEncodedFrame(frame_size_bytes, rtc::TimeMillis()); + } + + absl::optional utilization_factor = + detector_.GetUtilizationFactor(rtc::TimeMillis()); + EXPECT_NEAR(utilization_factor.value_or(-1), 1.05, 0.01); +} + +} // namespace webrtc diff --git a/video/end_to_end_tests/bandwidth_tests.cc b/video/end_to_end_tests/bandwidth_tests.cc index 15327a9fc6..17bb9853a7 100644 --- a/video/end_to_end_tests/bandwidth_tests.cc +++ b/video/end_to_end_tests/bandwidth_tests.cc @@ -266,6 +266,12 @@ TEST_F(BandwidthEndToEndTest, RembWithSendSideBwe) { } TEST_F(BandwidthEndToEndTest, ReportsSetEncoderRates) { + // If these fields trial are on, we get lower bitrates than expected by this + // test, due to the packetization overhead and encoder pushback. + webrtc::test::ScopedFieldTrials field_trials( + std::string(field_trial::GetFieldTrialString()) + + "WebRTC-SubtractPacketizationOverhead/Disabled/" + "WebRTC-VideoRateControl/bitrate_adjuster:false/"); class EncoderRateStatsTest : public test::EndToEndTest, public test::FakeEncoder { public: diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index 0585ef9626..8338d84205 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -2754,11 +2754,12 @@ TEST_F(VideoSendStreamTest, ReconfigureBitratesSetsEncoderBitratesCorrectly) { static const int kMaxBitrateKbps = 413; static const int kIncreasedStartBitrateKbps = 451; static const int kIncreasedMaxBitrateKbps = 597; - // If this field trial is on, we get lower bitrates than expected by this - // test, due to the packetization overhead. + // If these fields trial are on, we get lower bitrates than expected by this + // test, due to the packetization overhead and encoder pushback. webrtc::test::ScopedFieldTrials field_trials( std::string(field_trial::GetFieldTrialString()) + - "WebRTC-SubtractPacketizationOverhead/Disabled/"); + "WebRTC-SubtractPacketizationOverhead/Disabled/" + "WebRTC-VideoRateControl/bitrate_adjuster:false/"); class EncoderBitrateThresholdObserver : public test::SendTest, public test::FakeEncoder { diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index ea59b0aba0..f563f4fee4 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -26,6 +26,7 @@ #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" #include "rtc_base/experiments/quality_scaling_experiment.h" +#include "rtc_base/experiments/rate_control_settings.h" #include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/strings/string_builder.h" @@ -364,6 +365,7 @@ VideoStreamEncoder::VideoStreamEncoder( source_proxy_(new VideoSourceProxy(this)), sink_(nullptr), settings_(settings), + rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), video_sender_(Clock::GetRealTimeClock(), this), overuse_detector_(std::move(overuse_detector)), encoder_stats_observer_(encoder_stats_observer), @@ -449,7 +451,7 @@ void VideoStreamEncoder::SetSource( degradation_preference_ = degradation_preference; if (encoder_) - ConfigureQualityScaler(); + ConfigureQualityScaler(encoder_->GetEncoderInfo()); if (!IsFramerateScalingEnabled(degradation_preference) && max_framerate_ != -1) { @@ -645,6 +647,12 @@ void VideoStreamEncoder::ReconfigureEncoder() { field_trial::IsDisabled(kFrameDropperFieldTrial) || (num_layers > 1 && codec.mode == VideoCodecMode::kScreensharing); + VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo(); + if (rate_control_settings_.UseEncoderBitrateAdjuster()) { + bitrate_adjuster_ = absl::make_unique(codec); + bitrate_adjuster_->OnEncoderInfo(info); + } + if (rate_allocator_ && last_observed_bitrate_bps_ > 0) { // We have a new rate allocator instance and already configured target // bitrate. Update the rate allocation and notify observsers. @@ -672,12 +680,13 @@ void VideoStreamEncoder::ReconfigureEncoder() { max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); overuse_detector_->OnTargetFramerateUpdated(target_framerate); - ConfigureQualityScaler(); + ConfigureQualityScaler(info); } -void VideoStreamEncoder::ConfigureQualityScaler() { +void VideoStreamEncoder::ConfigureQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info) { RTC_DCHECK_RUN_ON(&encoder_queue_); - const auto scaling_settings = encoder_->GetEncoderInfo().scaling_settings; + const auto scaling_settings = encoder_info.scaling_settings; const bool quality_scaling_allowed = IsResolutionScalingEnabled(degradation_preference_) && scaling_settings.thresholds; @@ -840,6 +849,10 @@ VideoStreamEncoder::GetBitrateAllocationAndNotifyObserver( bitrate_observer_->OnBitrateAllocationUpdated(bitrate_allocation); } + if (bitrate_adjuster_) { + return bitrate_adjuster_->AdjustRateAllocation(bitrate_allocation, + framerate_fps); + } return bitrate_allocation; } @@ -987,7 +1000,21 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, if (info.implementation_name != encoder_info_.implementation_name) { encoder_stats_observer_->OnEncoderImplementationChanged( info.implementation_name); + if (bitrate_adjuster_) { + // Encoder implementation changed, reset overshoot detector states. + bitrate_adjuster_->Reset(); + } } + + if (bitrate_adjuster_) { + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + if (info.fps_allocation[si] != encoder_info_.fps_allocation[si]) { + bitrate_adjuster_->OnEncoderInfo(info); + break; + } + } + } + encoder_info_ = info; input_framerate_.Update(1u, clock_->TimeInMilliseconds()); @@ -1018,26 +1045,25 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); int64_t time_sent_us = rtc::TimeMicros(); - uint32_t timestamp = encoded_image.Timestamp(); - const int qp = encoded_image.qp_; - int64_t capture_time_us = - encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; + // We are only interested in propagating the meta-data about the image, not + // encoded data itself, to the post encode function. Since we cannot be sure + // the pointer will still be valid when run on the task queue, set it to null. + EncodedImage encoded_image_metadata = encoded_image; + encoded_image_metadata.set_buffer(nullptr, 0); - absl::optional encode_duration_us; - if (encoded_image.timing_.flags != VideoSendTiming::kInvalid) { - encode_duration_us.emplace( - // TODO(nisse): Maybe use capture_time_ms_ rather than encode_start_ms_? - rtc::kNumMicrosecsPerMillisec * - (encoded_image.timing_.encode_finish_ms - - encoded_image.timing_.encode_start_ms)); + int temporal_index = 0; + if (codec_specific_info) { + if (codec_specific_info->codecType == kVideoCodecVP9) { + temporal_index = codec_specific_info->codecSpecific.VP9.temporal_idx; + } else if (codec_specific_info->codecType == kVideoCodecVP8) { + temporal_index = codec_specific_info->codecSpecific.VP8.temporalIdx; + } + } + if (temporal_index == kNoTemporalIdx) { + temporal_index = 0; } - // Run post encode tasks, such as overuse detection and frame rate/drop - // stats for internal encoders. - const size_t frame_size = encoded_image.size(); - const bool keyframe = encoded_image._frameType == FrameType::kVideoFrameKey; - RunPostEncode(timestamp, time_sent_us, capture_time_us, encode_duration_us, - qp, frame_size, keyframe); + RunPostEncode(encoded_image_metadata, time_sent_us, temporal_index); if (result.error == Result::OK) { // In case of an internal encoder running on a separate thread, the @@ -1372,26 +1398,35 @@ VideoStreamEncoder::GetConstAdaptCounter() { return adapt_counters_[degradation_preference_]; } -void VideoStreamEncoder::RunPostEncode(uint32_t frame_timestamp, +void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image, int64_t time_sent_us, - int64_t capture_time_us, - absl::optional encode_durations_us, - int qp, - size_t frame_size_bytes, - bool keyframe) { + int temporal_index) { if (!encoder_queue_.IsCurrent()) { - encoder_queue_.PostTask([this, frame_timestamp, time_sent_us, qp, - capture_time_us, encode_durations_us, - frame_size_bytes, keyframe] { - RunPostEncode(frame_timestamp, time_sent_us, capture_time_us, - encode_durations_us, qp, frame_size_bytes, keyframe); - }); + encoder_queue_.PostTask( + [this, encoded_image, time_sent_us, temporal_index] { + RunPostEncode(encoded_image, time_sent_us, temporal_index); + }); return; } RTC_DCHECK_RUN_ON(&encoder_queue_); - if (frame_size_bytes > 0) { - frame_dropper_.Fill(frame_size_bytes, !keyframe); + + absl::optional encode_duration_us; + if (encoded_image.timing_.flags != VideoSendTiming::kInvalid) { + encode_duration_us = + // TODO(nisse): Maybe use capture_time_ms_ rather than encode_start_ms_? + rtc::kNumMicrosecsPerMillisec * + (encoded_image.timing_.encode_finish_ms - + encoded_image.timing_.encode_start_ms); + } + + // Run post encode tasks, such as overuse detection and frame rate/drop + // stats for internal encoders. + const size_t frame_size = encoded_image.size(); + const bool keyframe = encoded_image._frameType == FrameType::kVideoFrameKey; + + if (frame_size > 0) { + frame_dropper_.Fill(frame_size, !keyframe); } if (encoder_info_.has_internal_source) { @@ -1404,10 +1439,15 @@ void VideoStreamEncoder::RunPostEncode(uint32_t frame_timestamp, } } - overuse_detector_->FrameSent(frame_timestamp, time_sent_us, capture_time_us, - encode_durations_us); - if (quality_scaler_ && qp >= 0) - quality_scaler_->ReportQp(qp); + overuse_detector_->FrameSent( + encoded_image.Timestamp(), time_sent_us, + encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec, + encode_duration_us); + if (quality_scaler_ && encoded_image.qp_ >= 0) + quality_scaler_->ReportQp(encoded_image.qp_); + if (bitrate_adjuster_) { + bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index); + } } // Class holding adaptation information. diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 577ca5b562..feed55b9bf 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -29,9 +29,11 @@ #include "modules/video_coding/video_coding_impl.h" #include "rtc_base/critical_section.h" #include "rtc_base/event.h" +#include "rtc_base/experiments/rate_control_settings.h" #include "rtc_base/rate_statistics.h" #include "rtc_base/sequenced_task_checker.h" #include "rtc_base/task_queue.h" +#include "video/encoder_bitrate_adjuster.h" #include "video/overuse_frame_detector.h" namespace webrtc { @@ -106,7 +108,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, size_t max_data_payload_length); void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_); - void ConfigureQualityScaler(); + void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); // Implements VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; @@ -180,13 +182,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(&encoder_queue_); VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( AdaptReason reason) RTC_RUN_ON(&encoder_queue_); - void RunPostEncode(uint32_t frame_timestamp, + void RunPostEncode(EncodedImage encoded_image, int64_t time_sent_us, - int64_t capture_time_us, - absl::optional encode_durations_us, - int qp, - size_t frame_size_bytes, - bool keyframe); + int temporal_index); rtc::Event shutdown_event_; @@ -201,6 +199,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, const std::unique_ptr source_proxy_; EncoderSink* sink_; const VideoStreamEncoderSettings settings_; + const RateControlSettings rate_control_settings_; vcm::VideoSender video_sender_ RTC_GUARDED_BY(&encoder_queue_); const std::unique_ptr overuse_detector_ @@ -295,6 +294,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // the worker thread. std::atomic pending_frame_drops_; + std::unique_ptr bitrate_adjuster_ + RTC_GUARDED_BY(&encoder_queue_); + // All public methods are proxied to |encoder_queue_|. It must must be // destroyed first to make sure no tasks are run that use other members. rtc::TaskQueue encoder_queue_; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index fba99fe8db..68a30a170d 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -25,6 +25,7 @@ #include "rtc_base/fake_clock.h" #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" +#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/metrics.h" #include "system_wrappers/include/sleep.h" #include "test/encoder_settings.h" @@ -3300,7 +3301,13 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) { // Make encoder produce frames at double the expected bitrate during 3 seconds // of video, verify number of drops. Rate needs to be slightly changed in // order to force the rate to be reconfigured. - fake_encoder_.SimulateOvershoot(2.0); + double overshoot_factor = 2.0; + if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) { + // With bitrate adjuster, when need to overshoot even more to trigger + // frame dropping. + overshoot_factor *= 2; + } + fake_encoder_.SimulateOvershoot(overshoot_factor); video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps + 1000, 0, 0); num_dropped = 0; for (int i = 0; i < kNumFramesInRun; ++i) {