7ca375c8ca
The overshoot detector uses a simple pacer model to determine an estimate of how much the encoder is overusing the target bitrate. This utilization factor can then be adjuster for when configuring the actual target bitrate. Spatial layers (simulcast streams) are adjusted separately. Temporal layers are measured separately, but are combined into a single utilization factor per spatial layer. Bug: webrtc:10155 Change-Id: I8ea58dc6c4871e880553d7c22202f11cb2feb216 Reviewed-on: https://webrtc-review.googlesource.com/c/114886 Commit-Queue: Erik Språng <sprang@webrtc.org> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Cr-Commit-Position: refs/heads/master@{#26573}
381 lines
16 KiB
C++
381 lines
16 KiB
C++
/*
|
|
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "video/encoder_bitrate_adjuster.h"
|
|
|
|
#include <vector>
|
|
|
|
#include "absl/memory/memory.h"
|
|
#include "api/units/data_rate.h"
|
|
#include "rtc_base/fake_clock.h"
|
|
#include "rtc_base/numerics/safe_conversions.h"
|
|
#include "test/gtest.h"
|
|
|
|
namespace webrtc {
|
|
|
|
class EncoderBitrateAdjusterTest : public ::testing::Test {
|
|
public:
|
|
static constexpr int64_t kWindowSizeMs = 3000;
|
|
static constexpr int kDefaultBitrateBps = 300000;
|
|
static constexpr int kDefaultFrameRateFps = 30;
|
|
EncoderBitrateAdjusterTest()
|
|
: target_bitrate_(DataRate::bps(kDefaultBitrateBps)),
|
|
target_framerate_fps_(kDefaultFrameRateFps),
|
|
tl_pattern_idx_{} {}
|
|
|
|
protected:
|
|
void SetUpAdjuster(size_t num_spatial_layers,
|
|
size_t num_temporal_layers,
|
|
bool vp9_svc) {
|
|
// Initialize some default VideoCodec instance with the given number of
|
|
// layers.
|
|
if (vp9_svc) {
|
|
codec_.codecType = VideoCodecType::kVideoCodecVP9;
|
|
codec_.numberOfSimulcastStreams = 1;
|
|
codec_.VP9()->numberOfSpatialLayers = num_spatial_layers;
|
|
codec_.VP9()->numberOfTemporalLayers = num_temporal_layers;
|
|
for (size_t si = 0; si < num_spatial_layers; ++si) {
|
|
codec_.spatialLayers[si].minBitrate = 100 * (1 << si);
|
|
codec_.spatialLayers[si].targetBitrate = 200 * (1 << si);
|
|
codec_.spatialLayers[si].maxBitrate = 300 * (1 << si);
|
|
codec_.spatialLayers[si].active = true;
|
|
codec_.spatialLayers[si].numberOfTemporalLayers = num_temporal_layers;
|
|
}
|
|
} else {
|
|
codec_.codecType = VideoCodecType::kVideoCodecVP8;
|
|
codec_.numberOfSimulcastStreams = num_spatial_layers;
|
|
codec_.VP8()->numberOfTemporalLayers = num_temporal_layers;
|
|
for (size_t si = 0; si < num_spatial_layers; ++si) {
|
|
codec_.simulcastStream[si].minBitrate = 100 * (1 << si);
|
|
codec_.simulcastStream[si].targetBitrate = 200 * (1 << si);
|
|
codec_.simulcastStream[si].maxBitrate = 300 * (1 << si);
|
|
codec_.simulcastStream[si].active = true;
|
|
codec_.simulcastStream[si].numberOfTemporalLayers = num_temporal_layers;
|
|
}
|
|
}
|
|
|
|
for (size_t si = 0; si < num_spatial_layers; ++si) {
|
|
encoder_info_.fps_allocation[si].resize(num_temporal_layers);
|
|
double fraction = 1.0;
|
|
for (int ti = num_temporal_layers - 1; ti >= 0; --ti) {
|
|
encoder_info_.fps_allocation[si][ti] = static_cast<uint8_t>(
|
|
VideoEncoder::EncoderInfo::kMaxFramerateFraction * fraction + 0.5);
|
|
fraction /= 2.0;
|
|
}
|
|
}
|
|
|
|
adjuster_ = absl::make_unique<EncoderBitrateAdjuster>(codec_);
|
|
adjuster_->OnEncoderInfo(encoder_info_);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
}
|
|
|
|
void InsertFrames(std::vector<std::vector<double>> utilization_factors,
|
|
int64_t duration_ms) {
|
|
constexpr size_t kMaxFrameSize = 100000;
|
|
uint8_t buffer[kMaxFrameSize];
|
|
|
|
const int64_t start_us = rtc::TimeMicros();
|
|
while (rtc::TimeMicros() <
|
|
start_us + (duration_ms * rtc::kNumMicrosecsPerMillisec)) {
|
|
clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec /
|
|
target_framerate_fps_);
|
|
for (size_t si = 0; si < NumSpatialLayers(); ++si) {
|
|
const std::vector<int>& tl_pattern =
|
|
kTlPatterns[NumTemporalLayers(si) - 1];
|
|
const size_t ti =
|
|
tl_pattern[(tl_pattern_idx_[si]++) % tl_pattern.size()];
|
|
|
|
uint32_t layer_bitrate_bps =
|
|
current_adjusted_allocation_.GetBitrate(si, ti);
|
|
double layer_framerate_fps = target_framerate_fps_;
|
|
if (encoder_info_.fps_allocation[si].size() > ti) {
|
|
uint8_t layer_fps_fraction = encoder_info_.fps_allocation[si][ti];
|
|
if (ti > 0) {
|
|
// We're interested in the frame rate for this layer only, not
|
|
// cumulative frame rate.
|
|
layer_fps_fraction -= encoder_info_.fps_allocation[si][ti - 1];
|
|
}
|
|
layer_framerate_fps =
|
|
(target_framerate_fps_ * layer_fps_fraction) /
|
|
VideoEncoder::EncoderInfo::kMaxFramerateFraction;
|
|
}
|
|
double utilization_factor = 1.0;
|
|
if (utilization_factors.size() > si &&
|
|
utilization_factors[si].size() > ti) {
|
|
utilization_factor = utilization_factors[si][ti];
|
|
}
|
|
size_t frame_size_bytes = utilization_factor *
|
|
(layer_bitrate_bps / 8.0) /
|
|
layer_framerate_fps;
|
|
|
|
EncodedImage image(buffer, 0, kMaxFrameSize);
|
|
image.set_size(frame_size_bytes);
|
|
image.SetSpatialIndex(si);
|
|
adjuster_->OnEncodedFrame(image, ti);
|
|
}
|
|
}
|
|
}
|
|
|
|
size_t NumSpatialLayers() const {
|
|
if (codec_.codecType == VideoCodecType::kVideoCodecVP9) {
|
|
return codec_.VP9().numberOfSpatialLayers;
|
|
}
|
|
return codec_.numberOfSimulcastStreams;
|
|
}
|
|
|
|
size_t NumTemporalLayers(int spatial_index) {
|
|
if (codec_.codecType == VideoCodecType::kVideoCodecVP9) {
|
|
return codec_.spatialLayers[spatial_index].numberOfTemporalLayers;
|
|
}
|
|
return codec_.simulcastStream[spatial_index].numberOfTemporalLayers;
|
|
}
|
|
|
|
void ExpectNear(const VideoBitrateAllocation& expected_allocation,
|
|
const VideoBitrateAllocation& actual_allocation,
|
|
double allowed_error_fraction) {
|
|
for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
|
|
for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
|
|
if (expected_allocation.HasBitrate(si, ti)) {
|
|
EXPECT_TRUE(actual_allocation.HasBitrate(si, ti));
|
|
uint32_t expected_layer_bitrate_bps =
|
|
expected_allocation.GetBitrate(si, ti);
|
|
EXPECT_NEAR(expected_layer_bitrate_bps,
|
|
actual_allocation.GetBitrate(si, ti),
|
|
static_cast<uint32_t>(expected_layer_bitrate_bps *
|
|
allowed_error_fraction));
|
|
} else {
|
|
EXPECT_FALSE(actual_allocation.HasBitrate(si, ti));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
VideoBitrateAllocation MultiplyAllocation(
|
|
const VideoBitrateAllocation& allocation,
|
|
double factor) {
|
|
VideoBitrateAllocation multiplied_allocation;
|
|
for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
|
|
for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
|
|
if (allocation.HasBitrate(si, ti)) {
|
|
multiplied_allocation.SetBitrate(
|
|
si, ti,
|
|
static_cast<uint32_t>(factor * allocation.GetBitrate(si, ti) +
|
|
0.5));
|
|
}
|
|
}
|
|
}
|
|
return multiplied_allocation;
|
|
}
|
|
|
|
VideoCodec codec_;
|
|
VideoEncoder::EncoderInfo encoder_info_;
|
|
std::unique_ptr<EncoderBitrateAdjuster> adjuster_;
|
|
VideoBitrateAllocation current_input_allocation_;
|
|
VideoBitrateAllocation current_adjusted_allocation_;
|
|
rtc::ScopedFakeClock clock_;
|
|
DataRate target_bitrate_;
|
|
double target_framerate_fps_;
|
|
int tl_pattern_idx_[kMaxSpatialLayers];
|
|
|
|
const std::vector<int> kTlPatterns[kMaxTemporalStreams] = {
|
|
{0},
|
|
{0, 1},
|
|
{0, 2, 1, 2},
|
|
{0, 3, 2, 3, 1, 3, 2, 3}};
|
|
};
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, SingleLayerOptimal) {
|
|
// Single layer, well behaved encoder.
|
|
current_input_allocation_.SetBitrate(0, 0, 300000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 1, false);
|
|
InsertFrames({{1.0}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
// Adjusted allocation near input. Allow 1% error margin due to rounding
|
|
// errors etc.
|
|
ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, SingleLayerOveruse) {
|
|
// Single layer, well behaved encoder.
|
|
current_input_allocation_.SetBitrate(0, 0, 300000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 1, false);
|
|
InsertFrames({{1.2}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
// Adjusted allocation lowered by 20%.
|
|
ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.2),
|
|
current_adjusted_allocation_, 0.01);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, SingleLayerUnderuse) {
|
|
// Single layer, well behaved encoder.
|
|
current_input_allocation_.SetBitrate(0, 0, 300000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 1, false);
|
|
InsertFrames({{0.5}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
// Undershoot, adjusted should exactly match input.
|
|
ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.00);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersOptimalSize) {
|
|
// Three temporal layers, 60%/20%/20% bps distro, well behaved encoder.
|
|
current_input_allocation_.SetBitrate(0, 0, 180000);
|
|
current_input_allocation_.SetBitrate(0, 1, 60000);
|
|
current_input_allocation_.SetBitrate(0, 2, 60000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 3, false);
|
|
InsertFrames({{1.0, 1.0, 1.0}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersOvershoot) {
|
|
// Three temporal layers, 60%/20%/20% bps distro.
|
|
// 10% overshoot on all layers.
|
|
current_input_allocation_.SetBitrate(0, 0, 180000);
|
|
current_input_allocation_.SetBitrate(0, 1, 60000);
|
|
current_input_allocation_.SetBitrate(0, 2, 60000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 3, false);
|
|
InsertFrames({{1.1, 1.1, 1.1}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
// Adjusted allocation lowered by 10%.
|
|
ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.1),
|
|
current_adjusted_allocation_, 0.01);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersUndershoot) {
|
|
// Three temporal layers, 60%/20%/20% bps distro, undershoot all layers.
|
|
current_input_allocation_.SetBitrate(0, 0, 180000);
|
|
current_input_allocation_.SetBitrate(0, 1, 60000);
|
|
current_input_allocation_.SetBitrate(0, 2, 60000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 3, false);
|
|
InsertFrames({{0.8, 0.8, 0.8}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
// Adjusted allocation identical since we don't boost bitrates.
|
|
ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.0);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersSkewedOvershoot) {
|
|
// Three temporal layers, 60%/20%/20% bps distro.
|
|
// 10% overshoot on base layer, 20% on higher layers.
|
|
current_input_allocation_.SetBitrate(0, 0, 180000);
|
|
current_input_allocation_.SetBitrate(0, 1, 60000);
|
|
current_input_allocation_.SetBitrate(0, 2, 60000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 3, false);
|
|
InsertFrames({{1.1, 1.2, 1.2}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
// Expected overshoot is weighted by bitrate:
|
|
// (0.6 * 1.1 + 0.2 * 1.2 + 0.2 * 1.2) = 1.14
|
|
ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.14),
|
|
current_adjusted_allocation_, 0.01);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, FourTemporalLayersSkewedOvershoot) {
|
|
// Three temporal layers, 40%/30%/15%/15% bps distro.
|
|
// 10% overshoot on base layer, 20% on higher layers.
|
|
current_input_allocation_.SetBitrate(0, 0, 120000);
|
|
current_input_allocation_.SetBitrate(0, 1, 90000);
|
|
current_input_allocation_.SetBitrate(0, 2, 45000);
|
|
current_input_allocation_.SetBitrate(0, 3, 45000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 4, false);
|
|
InsertFrames({{1.1, 1.2, 1.2, 1.2}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
// Expected overshoot is weighted by bitrate:
|
|
// (0.4 * 1.1 + 0.3 * 1.2 + 0.15 * 1.2 + 0.15 * 1.2) = 1.16
|
|
ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.16),
|
|
current_adjusted_allocation_, 0.01);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersNonLayeredEncoder) {
|
|
// Three temporal layers, 60%/20%/20% bps allocation, 10% overshoot,
|
|
// encoder does not actually support temporal layers.
|
|
current_input_allocation_.SetBitrate(0, 0, 180000);
|
|
current_input_allocation_.SetBitrate(0, 1, 60000);
|
|
current_input_allocation_.SetBitrate(0, 2, 60000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 1, false);
|
|
InsertFrames({{1.1}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
// Expect the actual 10% overuse to be detected and the allocation to
|
|
// only contain the one entry.
|
|
VideoBitrateAllocation expected_allocation;
|
|
expected_allocation.SetBitrate(
|
|
0, 0,
|
|
static_cast<uint32_t>(current_input_allocation_.get_sum_bps() / 1.10));
|
|
ExpectNear(expected_allocation, current_adjusted_allocation_, 0.01);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, IgnoredStream) {
|
|
// Encoder with three temporal layers, but in a mode that does not support
|
|
// deterministic frame rate. Those are ignored, even if bitrate overshoots.
|
|
current_input_allocation_.SetBitrate(0, 0, 180000);
|
|
current_input_allocation_.SetBitrate(0, 1, 60000);
|
|
target_framerate_fps_ = 30;
|
|
SetUpAdjuster(1, 1, false);
|
|
encoder_info_.fps_allocation[0].clear();
|
|
adjuster_->OnEncoderInfo(encoder_info_);
|
|
|
|
InsertFrames({{1.1}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
|
|
// Values passed through.
|
|
ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.00);
|
|
}
|
|
|
|
TEST_F(EncoderBitrateAdjusterTest, DifferentSpatialOvershoots) {
|
|
// Two streams, both with three temporal layers.
|
|
// S0 has 5% overshoot, S1 has 25% overshoot.
|
|
current_input_allocation_.SetBitrate(0, 0, 180000);
|
|
current_input_allocation_.SetBitrate(0, 1, 60000);
|
|
current_input_allocation_.SetBitrate(0, 2, 60000);
|
|
current_input_allocation_.SetBitrate(1, 0, 400000);
|
|
current_input_allocation_.SetBitrate(1, 1, 150000);
|
|
current_input_allocation_.SetBitrate(1, 2, 150000);
|
|
target_framerate_fps_ = 30;
|
|
// Run twice, once configured as simulcast and once as VP9 SVC.
|
|
for (int i = 0; i < 2; ++i) {
|
|
SetUpAdjuster(2, 3, i == 0);
|
|
InsertFrames({{1.05, 1.05, 1.05}, {1.25, 1.25, 1.25}}, kWindowSizeMs);
|
|
current_adjusted_allocation_ = adjuster_->AdjustRateAllocation(
|
|
current_input_allocation_, target_framerate_fps_);
|
|
VideoBitrateAllocation expected_allocation;
|
|
for (size_t ti = 0; ti < 3; ++ti) {
|
|
expected_allocation.SetBitrate(
|
|
0, ti,
|
|
static_cast<uint32_t>(current_input_allocation_.GetBitrate(0, ti) /
|
|
1.05));
|
|
expected_allocation.SetBitrate(
|
|
1, ti,
|
|
static_cast<uint32_t>(current_input_allocation_.GetBitrate(1, ti) /
|
|
1.25));
|
|
}
|
|
ExpectNear(expected_allocation, current_adjusted_allocation_, 0.01);
|
|
}
|
|
}
|
|
|
|
} // namespace webrtc
|