Video: add new metric for VP9/AV1 hw encoding with softwareBRC
Record the RMSE of encoding bitrate diff from output stream to target bitrate set by WebRTC when the AV1/VP9 stream is removed. Design doc: https://docs.google.com/document/d/1tE7Fquj1U6CwZ01QxxjoCaJS2o0LFYyIxvstLDx0qB0/edit#heading=h.c11ebhaja9tg Bug: chromium:1358750 Change-Id: Ifa6f5aacfa8f6b0835f4609f102ae3ae1ee2897e Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/298684 Reviewed-by: Erik Språng <sprang@webrtc.org> Commit-Queue: Erik Språng <sprang@webrtc.org> Cr-Commit-Position: refs/heads/main@{#39907}
This commit is contained in:
parent
eba7cee1da
commit
d3416972dd
1
AUTHORS
1
AUTHORS
@ -133,6 +133,7 @@ Yura Yaroshevich <yura.yaroshevich@gmail.com>
|
||||
Yuriy Pavlyshak <yuriy@appear.in>
|
||||
Yusuke Suzuki <utatane.tea@gmail.com>
|
||||
Pengfei Han <hanpfei@gmail.com>
|
||||
Yingying Ma <yingying.ma@intel.com>
|
||||
# END individuals section.
|
||||
|
||||
# BEGIN organizations section.
|
||||
|
||||
@ -47,7 +47,9 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings)
|
||||
: utilize_bandwidth_headroom_(RateControlSettings::ParseFromFieldTrials()
|
||||
.BitrateAdjusterCanUseNetworkHeadroom()),
|
||||
frames_since_layout_change_(0),
|
||||
min_bitrates_bps_{} {
|
||||
min_bitrates_bps_{},
|
||||
codec_(codec_settings.codecType),
|
||||
codec_mode_(codec_settings.mode) {
|
||||
// TODO(https://crbug.com/webrtc/14891): If we want to support simulcast of
|
||||
// SVC streams, EncoderBitrateAdjuster needs to be updated to care about both
|
||||
// `simulcastStream` and `spatialLayers` at the same time.
|
||||
@ -90,7 +92,9 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
|
||||
++active_tls[si];
|
||||
if (!overshoot_detectors_[si][ti]) {
|
||||
overshoot_detectors_[si][ti] =
|
||||
std::make_unique<EncoderOvershootDetector>(kWindowSizeMs);
|
||||
std::make_unique<EncoderOvershootDetector>(
|
||||
kWindowSizeMs, codec_,
|
||||
codec_mode_ == VideoCodecMode::kScreensharing);
|
||||
frames_since_layout_change_ = 0;
|
||||
}
|
||||
} else if (overshoot_detectors_[si][ti]) {
|
||||
|
||||
@ -73,6 +73,12 @@ class EncoderBitrateAdjuster {
|
||||
|
||||
// Minimum bitrates allowed, per spatial layer.
|
||||
uint32_t min_bitrates_bps_[kMaxSpatialLayers];
|
||||
|
||||
// Codec type used for encoding.
|
||||
VideoCodecType codec_;
|
||||
|
||||
// Codec mode: { kRealtimeVideo, kScreensharing }.
|
||||
VideoCodecMode codec_mode_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
@ -11,6 +11,9 @@
|
||||
#include "video/encoder_overshoot_detector.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
|
||||
#include "system_wrappers/include/metrics.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
@ -20,7 +23,9 @@ namespace {
|
||||
static constexpr double kMaxMediaUnderrunFrames = 5.0;
|
||||
} // namespace
|
||||
|
||||
EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms)
|
||||
EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms,
|
||||
VideoCodecType codec,
|
||||
bool is_screenshare)
|
||||
: window_size_ms_(window_size_ms),
|
||||
time_last_update_ms_(-1),
|
||||
sum_network_utilization_factors_(0.0),
|
||||
@ -28,9 +33,16 @@ EncoderOvershootDetector::EncoderOvershootDetector(int64_t window_size_ms)
|
||||
target_bitrate_(DataRate::Zero()),
|
||||
target_framerate_fps_(0),
|
||||
network_buffer_level_bits_(0),
|
||||
media_buffer_level_bits_(0) {}
|
||||
media_buffer_level_bits_(0),
|
||||
codec_(codec),
|
||||
is_screenshare_(is_screenshare),
|
||||
frame_count_(0),
|
||||
sum_diff_kbps_squared_(0),
|
||||
sum_overshoot_percent_(0) {}
|
||||
|
||||
EncoderOvershootDetector::~EncoderOvershootDetector() = default;
|
||||
EncoderOvershootDetector::~EncoderOvershootDetector() {
|
||||
UpdateHistograms();
|
||||
}
|
||||
|
||||
void EncoderOvershootDetector::SetTargetRate(DataRate target_bitrate,
|
||||
double target_framerate_fps,
|
||||
@ -57,6 +69,7 @@ void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) {
|
||||
// bitrate.
|
||||
LeakBits(time_ms);
|
||||
|
||||
const int64_t frame_size_bits = bytes * 8;
|
||||
// Ideal size of a frame given the current rates.
|
||||
const int64_t ideal_frame_size_bits = IdealFrameSizeBits();
|
||||
if (ideal_frame_size_bits == 0) {
|
||||
@ -64,14 +77,22 @@ void EncoderOvershootDetector::OnEncodedFrame(size_t bytes, int64_t time_ms) {
|
||||
return;
|
||||
}
|
||||
|
||||
const double network_utilization_factor = HandleEncodedFrame(
|
||||
bytes * 8, ideal_frame_size_bits, time_ms, &network_buffer_level_bits_);
|
||||
const double media_utilization_factor = HandleEncodedFrame(
|
||||
bytes * 8, ideal_frame_size_bits, time_ms, &media_buffer_level_bits_);
|
||||
const double network_utilization_factor =
|
||||
HandleEncodedFrame(frame_size_bits, ideal_frame_size_bits, time_ms,
|
||||
&network_buffer_level_bits_);
|
||||
const double media_utilization_factor =
|
||||
HandleEncodedFrame(frame_size_bits, ideal_frame_size_bits, time_ms,
|
||||
&media_buffer_level_bits_);
|
||||
|
||||
sum_network_utilization_factors_ += network_utilization_factor;
|
||||
sum_media_utilization_factors_ += media_utilization_factor;
|
||||
|
||||
// Calculate the bitrate diff in kbps
|
||||
int64_t diff_kbits = (frame_size_bits - ideal_frame_size_bits) / 1000;
|
||||
sum_diff_kbps_squared_ += diff_kbits * diff_kbits;
|
||||
sum_overshoot_percent_ += diff_kbits * 100 * 1000 / ideal_frame_size_bits;
|
||||
++frame_count_;
|
||||
|
||||
utilization_factors_.emplace_back(network_utilization_factor,
|
||||
media_utilization_factor, time_ms);
|
||||
}
|
||||
@ -142,6 +163,10 @@ absl::optional<double> EncoderOvershootDetector::GetMediaRateUtilizationFactor(
|
||||
}
|
||||
|
||||
void EncoderOvershootDetector::Reset() {
|
||||
UpdateHistograms();
|
||||
sum_diff_kbps_squared_ = 0;
|
||||
frame_count_ = 0;
|
||||
sum_overshoot_percent_ = 0;
|
||||
time_last_update_ms_ = -1;
|
||||
utilization_factors_.clear();
|
||||
target_bitrate_ = DataRate::Zero();
|
||||
@ -201,4 +226,50 @@ void EncoderOvershootDetector::CullOldUpdates(int64_t time_ms) {
|
||||
}
|
||||
}
|
||||
|
||||
void EncoderOvershootDetector::UpdateHistograms() {
|
||||
if (frame_count_ == 0)
|
||||
return;
|
||||
|
||||
int64_t bitrate_rmse = std::sqrt(sum_diff_kbps_squared_ / frame_count_);
|
||||
int64_t average_overshoot_percent = sum_overshoot_percent_ / frame_count_;
|
||||
const std::string rmse_histogram_prefix =
|
||||
is_screenshare_ ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps."
|
||||
: "WebRTC.Video.RMSEOfEncodingBitrateInKbps.";
|
||||
const std::string overshoot_histogram_prefix =
|
||||
is_screenshare_ ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot."
|
||||
: "WebRTC.Video.EncodingBitrateOvershoot.";
|
||||
// index = 1 represents screensharing histograms recording.
|
||||
// index = 0 represents normal video histograms recording.
|
||||
const int index = is_screenshare_ ? 1 : 0;
|
||||
switch (codec_) {
|
||||
case VideoCodecType::kVideoCodecAV1:
|
||||
RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Av1",
|
||||
bitrate_rmse);
|
||||
RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Av1",
|
||||
average_overshoot_percent);
|
||||
break;
|
||||
case VideoCodecType::kVideoCodecVP9:
|
||||
RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Vp9",
|
||||
bitrate_rmse);
|
||||
RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Vp9",
|
||||
average_overshoot_percent);
|
||||
break;
|
||||
case VideoCodecType::kVideoCodecVP8:
|
||||
RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "Vp8",
|
||||
bitrate_rmse);
|
||||
RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "Vp8",
|
||||
average_overshoot_percent);
|
||||
break;
|
||||
case VideoCodecType::kVideoCodecH264:
|
||||
RTC_HISTOGRAMS_COUNTS_10000(index, rmse_histogram_prefix + "H264",
|
||||
bitrate_rmse);
|
||||
RTC_HISTOGRAMS_COUNTS_10000(index, overshoot_histogram_prefix + "H264",
|
||||
average_overshoot_percent);
|
||||
break;
|
||||
case VideoCodecType::kVideoCodecGeneric:
|
||||
case VideoCodecType::kVideoCodecMultiplex:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
@ -15,12 +15,15 @@
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/units/data_rate.h"
|
||||
#include "api/video_codecs/video_codec.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class EncoderOvershootDetector {
|
||||
public:
|
||||
explicit EncoderOvershootDetector(int64_t window_size_ms);
|
||||
explicit EncoderOvershootDetector(int64_t window_size_ms,
|
||||
VideoCodecType codec,
|
||||
bool is_screenshare);
|
||||
~EncoderOvershootDetector();
|
||||
|
||||
void SetTargetRate(DataRate target_bitrate,
|
||||
@ -64,6 +67,7 @@ class EncoderOvershootDetector {
|
||||
double media_utilization_factor;
|
||||
int64_t update_time_ms;
|
||||
};
|
||||
void UpdateHistograms();
|
||||
std::deque<BitrateUpdate> utilization_factors_;
|
||||
double sum_network_utilization_factors_;
|
||||
double sum_media_utilization_factors_;
|
||||
@ -71,6 +75,11 @@ class EncoderOvershootDetector {
|
||||
double target_framerate_fps_;
|
||||
int64_t network_buffer_level_bits_;
|
||||
int64_t media_buffer_level_bits_;
|
||||
VideoCodecType codec_;
|
||||
bool is_screenshare_;
|
||||
int64_t frame_count_;
|
||||
int64_t sum_diff_kbps_squared_;
|
||||
int64_t sum_overshoot_percent_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
@ -10,23 +10,58 @@
|
||||
|
||||
#include "video/encoder_overshoot_detector.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "api/units/data_rate.h"
|
||||
#include "rtc_base/fake_clock.h"
|
||||
#include "rtc_base/time_utils.h"
|
||||
#include "system_wrappers/include/metrics.h"
|
||||
#include "test/gtest.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class EncoderOvershootDetectorTest : public ::testing::Test {
|
||||
namespace {
|
||||
|
||||
using ::testing::TestWithParam;
|
||||
using ::testing::ValuesIn;
|
||||
|
||||
static std::string CodecTypeToHistogramSuffix(VideoCodecType codec) {
|
||||
switch (codec) {
|
||||
case kVideoCodecVP8:
|
||||
return "Vp8";
|
||||
case kVideoCodecVP9:
|
||||
return "Vp9";
|
||||
case kVideoCodecAV1:
|
||||
return "Av1";
|
||||
case kVideoCodecH264:
|
||||
return "H264";
|
||||
case kVideoCodecGeneric:
|
||||
return "Generic";
|
||||
case kVideoCodecMultiplex:
|
||||
return "Multiplex";
|
||||
}
|
||||
}
|
||||
|
||||
struct TestParams {
|
||||
VideoCodecType codec_type;
|
||||
bool is_screenshare;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
class EncoderOvershootDetectorTest : public TestWithParam<TestParams> {
|
||||
public:
|
||||
static constexpr int kDefaultBitrateBps = 300000;
|
||||
static constexpr double kDefaultFrameRateFps = 15;
|
||||
EncoderOvershootDetectorTest()
|
||||
: detector_(kWindowSizeMs),
|
||||
: detector_(kWindowSizeMs,
|
||||
GetParam().codec_type,
|
||||
GetParam().is_screenshare),
|
||||
target_bitrate_(DataRate::BitsPerSec(kDefaultBitrateBps)),
|
||||
target_framerate_fps_(kDefaultFrameRateFps) {}
|
||||
|
||||
protected:
|
||||
void SetUp() override { metrics::Reset(); }
|
||||
void RunConstantUtilizationTest(double actual_utilization_factor,
|
||||
double expected_utilization_factor,
|
||||
double allowed_error,
|
||||
@ -70,7 +105,7 @@ class EncoderOvershootDetectorTest : public ::testing::Test {
|
||||
double target_framerate_fps_;
|
||||
};
|
||||
|
||||
TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) {
|
||||
TEST_P(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) {
|
||||
const int frame_size_bytes = 1000;
|
||||
const int64_t time_interval_ms = 33;
|
||||
detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
|
||||
@ -86,26 +121,26 @@ TEST_F(EncoderOvershootDetectorTest, NoUtilizationIfNoRate) {
|
||||
detector_.GetNetworkRateUtilizationFactor(rtc::TimeMillis()).has_value());
|
||||
}
|
||||
|
||||
TEST_F(EncoderOvershootDetectorTest, OptimalSize) {
|
||||
TEST_P(EncoderOvershootDetectorTest, OptimalSize) {
|
||||
// Optimally behaved encoder.
|
||||
// Allow some error margin due to rounding errors, eg due to frame
|
||||
// interval not being an integer.
|
||||
RunConstantUtilizationTest(1.0, 1.0, 0.01, kWindowSizeMs);
|
||||
}
|
||||
|
||||
TEST_F(EncoderOvershootDetectorTest, Undershoot) {
|
||||
TEST_P(EncoderOvershootDetectorTest, Undershoot) {
|
||||
// Undershoot, reported utilization factor should be capped to 1.0 so
|
||||
// that we don't incorrectly boost encoder bitrate during movement.
|
||||
RunConstantUtilizationTest(0.5, 1.0, 0.00, kWindowSizeMs);
|
||||
}
|
||||
|
||||
TEST_F(EncoderOvershootDetectorTest, Overshoot) {
|
||||
TEST_P(EncoderOvershootDetectorTest, Overshoot) {
|
||||
// Overshoot by 20%.
|
||||
// Allow some error margin due to rounding errors.
|
||||
RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs);
|
||||
}
|
||||
|
||||
TEST_F(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) {
|
||||
TEST_P(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) {
|
||||
// Overshoot by 20%, but vary framerate and bitrate.
|
||||
// Allow some error margin due to rounding errors.
|
||||
RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs);
|
||||
@ -115,7 +150,7 @@ TEST_F(EncoderOvershootDetectorTest, ConstantOvershootVaryingRates) {
|
||||
RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2);
|
||||
}
|
||||
|
||||
TEST_F(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) {
|
||||
TEST_P(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) {
|
||||
// Overshoot by 10%, keep framerate and bitrate constant.
|
||||
// Allow some error margin due to rounding errors.
|
||||
RunConstantUtilizationTest(1.1, 1.1, 0.01, kWindowSizeMs);
|
||||
@ -127,7 +162,7 @@ TEST_F(EncoderOvershootDetectorTest, ConstantRateVaryingOvershoot) {
|
||||
RunConstantUtilizationTest(1.2, 1.2, 0.01, kWindowSizeMs / 2);
|
||||
}
|
||||
|
||||
TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) {
|
||||
TEST_P(EncoderOvershootDetectorTest, PartialOvershoot) {
|
||||
const int ideal_frame_size_bytes =
|
||||
(target_bitrate_.bps() / target_framerate_fps_) / 8;
|
||||
detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
|
||||
@ -163,4 +198,83 @@ TEST_F(EncoderOvershootDetectorTest, PartialOvershoot) {
|
||||
detector_.GetMediaRateUtilizationFactor(rtc::TimeMillis());
|
||||
EXPECT_NEAR(media_utilization_factor.value_or(-1), 1.00, 0.01);
|
||||
}
|
||||
|
||||
TEST_P(EncoderOvershootDetectorTest, RecordsZeroErrorMetricWithNoOvershoot) {
|
||||
DataSize ideal_frame_size =
|
||||
target_bitrate_ / Frequency::Hertz(target_framerate_fps_);
|
||||
detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
|
||||
rtc::TimeMillis());
|
||||
detector_.OnEncodedFrame(ideal_frame_size.bytes(), rtc::TimeMillis());
|
||||
detector_.Reset();
|
||||
|
||||
const VideoCodecType codec = GetParam().codec_type;
|
||||
const bool is_screenshare = GetParam().is_screenshare;
|
||||
const std::string rmse_histogram_prefix =
|
||||
is_screenshare ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps."
|
||||
: "WebRTC.Video.RMSEOfEncodingBitrateInKbps.";
|
||||
const std::string overshoot_histogram_prefix =
|
||||
is_screenshare ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot."
|
||||
: "WebRTC.Video.EncodingBitrateOvershoot.";
|
||||
// RMSE and overshoot percent = 0, since we used ideal frame size.
|
||||
EXPECT_METRIC_EQ(1, metrics::NumSamples(rmse_histogram_prefix +
|
||||
CodecTypeToHistogramSuffix(codec)));
|
||||
EXPECT_METRIC_EQ(
|
||||
1, metrics::NumEvents(
|
||||
rmse_histogram_prefix + CodecTypeToHistogramSuffix(codec), 0));
|
||||
|
||||
EXPECT_METRIC_EQ(1, metrics::NumSamples(overshoot_histogram_prefix +
|
||||
CodecTypeToHistogramSuffix(codec)));
|
||||
EXPECT_METRIC_EQ(1, metrics::NumEvents(overshoot_histogram_prefix +
|
||||
CodecTypeToHistogramSuffix(codec),
|
||||
0));
|
||||
}
|
||||
|
||||
TEST_P(EncoderOvershootDetectorTest,
|
||||
RecordScreenshareZeroMetricWithNoOvershoot) {
|
||||
DataSize ideal_frame_size =
|
||||
target_bitrate_ / Frequency::Hertz(target_framerate_fps_);
|
||||
// Use target frame size with 50% overshoot.
|
||||
DataSize target_frame_size = ideal_frame_size * 3 / 2;
|
||||
detector_.SetTargetRate(target_bitrate_, target_framerate_fps_,
|
||||
rtc::TimeMillis());
|
||||
detector_.OnEncodedFrame(target_frame_size.bytes(), rtc::TimeMillis());
|
||||
detector_.Reset();
|
||||
|
||||
const VideoCodecType codec = GetParam().codec_type;
|
||||
const bool is_screenshare = GetParam().is_screenshare;
|
||||
const std::string rmse_histogram_prefix =
|
||||
is_screenshare ? "WebRTC.Video.Screenshare.RMSEOfEncodingBitrateInKbps."
|
||||
: "WebRTC.Video.RMSEOfEncodingBitrateInKbps.";
|
||||
const std::string overshoot_histogram_prefix =
|
||||
is_screenshare ? "WebRTC.Video.Screenshare.EncodingBitrateOvershoot."
|
||||
: "WebRTC.Video.EncodingBitrateOvershoot.";
|
||||
// Use ideal_frame_size_kbits to represnt ideal_frame_size.bytes()*8/1000,
|
||||
// then rmse_in_kbps = ideal_frame_size_kbits/2
|
||||
// since we use target frame size with 50% overshoot.
|
||||
int64_t rmse_in_kbps = ideal_frame_size.bytes() * 8 / 1000 / 2;
|
||||
EXPECT_METRIC_EQ(1, metrics::NumSamples(rmse_histogram_prefix +
|
||||
CodecTypeToHistogramSuffix(codec)));
|
||||
EXPECT_METRIC_EQ(1, metrics::NumEvents(rmse_histogram_prefix +
|
||||
CodecTypeToHistogramSuffix(codec),
|
||||
rmse_in_kbps));
|
||||
// overshoot percent = 50, since we used ideal_frame_size * 3 / 2;
|
||||
EXPECT_METRIC_EQ(1, metrics::NumSamples(overshoot_histogram_prefix +
|
||||
CodecTypeToHistogramSuffix(codec)));
|
||||
EXPECT_METRIC_EQ(1, metrics::NumEvents(overshoot_histogram_prefix +
|
||||
CodecTypeToHistogramSuffix(codec),
|
||||
50));
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
PerCodecType,
|
||||
EncoderOvershootDetectorTest,
|
||||
ValuesIn<TestParams>({{VideoCodecType::kVideoCodecVP8, false},
|
||||
{VideoCodecType::kVideoCodecVP8, true},
|
||||
{VideoCodecType::kVideoCodecVP9, false},
|
||||
{VideoCodecType::kVideoCodecVP9, true},
|
||||
{VideoCodecType::kVideoCodecAV1, false},
|
||||
{VideoCodecType::kVideoCodecAV1, true},
|
||||
{VideoCodecType::kVideoCodecH264, false},
|
||||
{VideoCodecType::kVideoCodecH264, true}}));
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user