Implement QualityConvergenceMonitor
The quality convergence monitor will be used for screenshare streams to determine if encoded video frames have reached the target quality. This is a generalization of the static threshold that is currently used for VP8 in VideoStreamEncoder. Internal design document: go/qp-convergence-detection Bug: chromium:328598314 Change-Id: I13e32ee6efb54cbdb4e8a814c525087af8cd2759 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/355902 Commit-Queue: Johannes Kron <kron@webrtc.org> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Cr-Commit-Position: refs/heads/main@{#42566}
This commit is contained in:
Johannes Kron
WebRTC LUCI CQ
parent
84af278666
commit
6bbbc08747
@ -388,6 +388,8 @@ rtc_library("video_stream_encoder_impl") {
|
||||
"encoder_overshoot_detector.h",
|
||||
"frame_encode_metadata_writer.cc",
|
||||
"frame_encode_metadata_writer.h",
|
||||
"quality_convergence_monitor.cc",
|
||||
"quality_convergence_monitor.h",
|
||||
"video_source_sink_controller.cc",
|
||||
"video_source_sink_controller.h",
|
||||
"video_stream_encoder.cc",
|
||||
@ -771,6 +773,7 @@ if (rtc_include_tests) {
|
||||
"frame_decode_timing_unittest.cc",
|
||||
"frame_encode_metadata_writer_unittest.cc",
|
||||
"picture_id_tests.cc",
|
||||
"quality_convergence_monitor_unittest.cc",
|
||||
"quality_limitation_reason_tracker_unittest.cc",
|
||||
"quality_scaling_tests.cc",
|
||||
"receive_statistics_proxy_unittest.cc",
|
||||
|
||||
124
video/quality_convergence_monitor.cc
Normal file
124
video/quality_convergence_monitor.cc
Normal file
@ -0,0 +1,124 @@
|
||||
/*
|
||||
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include "video/quality_convergence_monitor.h"
|
||||
|
||||
#include <numeric>
|
||||
|
||||
#include "rtc_base/checks.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
QualityConvergenceMonitor::QualityConvergenceMonitor(const Parameters& params)
|
||||
: params_(params) {
|
||||
RTC_CHECK(
|
||||
!params_.dynamic_detection_enabled ||
|
||||
(params_.past_window_length > 0 && params_.recent_window_length > 0));
|
||||
}
|
||||
|
||||
// Adds the sample to the algorithms detection window and runs the following
|
||||
// convergence detection algorithm to determine if the time series of QP
|
||||
// values indicates that the encoded video has reached "target quality".
|
||||
//
|
||||
// Definitions
|
||||
//
|
||||
// - Let x[n] be the pixel data of a video frame.
|
||||
// - Let e[n] be the encoded representation of x[n].
|
||||
// - Let qp[n] be the corresponding QP value of the encoded video frame e[n].
|
||||
// - x[n] is a refresh frame if x[n] = x[n-1].
|
||||
// - qp_window is a list (or queue) of stored QP values, with size
|
||||
// L <= past_window_length + recent_window_length.
|
||||
// - qp_window can be partioned into:
|
||||
// qp_past = qp_window[ 0:end-recent_window_length ] and
|
||||
// qp_recent = qp_window[ -recent_window_length:end ].
|
||||
// - Let dynamic_qp_threshold be a maximum QP value for which convergence
|
||||
// is accepted.
|
||||
//
|
||||
// Algorithm
|
||||
//
|
||||
// For each encoded video frame e[n], take the corresponding qp[n] and do the
|
||||
// following:
|
||||
// 0. Check Static Threshold: if qp[n] < static_qp_threshold, return true.
|
||||
// 1. Check for Refresh Frame: If x[n] is not a refresh frame:
|
||||
// - Clear Q.
|
||||
// - Return false.
|
||||
// 2. Check Previous Convergence: If x[n] is a refresh frame AND true was
|
||||
// returned for x[n-1], return true.
|
||||
// 3. Update QP History: Append qp[n] to qp_window. If qp_window's length
|
||||
// exceeds past_window_length + recent_window_length, remove the first
|
||||
// element.
|
||||
// 4. Check for Sufficient Data: If L <= recent_window_length, return false.
|
||||
// 5. Calculate Average QP: Calculate avg(qp_past) and avg(ap_recent).
|
||||
// 6. Determine Convergence: If avg(qp_past) <= dynamic_qp_threshold AND
|
||||
// avg(qp_past) <= avg(qp_recent), return true. Otherwise, return false.
|
||||
//
|
||||
void QualityConvergenceMonitor::AddSample(int qp, bool is_refresh_frame) {
|
||||
// Invalid QP.
|
||||
if (qp < 0) {
|
||||
qp_window_.clear();
|
||||
at_target_quality_ = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// 0. Check static threshold.
|
||||
if (qp <= params_.static_qp_threshold) {
|
||||
at_target_quality_ = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Check for refresh frame and if dynamic detection is disabled.
|
||||
if (!is_refresh_frame || !params_.dynamic_detection_enabled) {
|
||||
qp_window_.clear();
|
||||
at_target_quality_ = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. Check previous convergence.
|
||||
RTC_CHECK(is_refresh_frame);
|
||||
if (at_target_quality_) {
|
||||
// No need to update state.
|
||||
return;
|
||||
}
|
||||
|
||||
// 3. Update QP history.
|
||||
qp_window_.push_back(qp);
|
||||
if (qp_window_.size() >
|
||||
params_.recent_window_length + params_.past_window_length) {
|
||||
qp_window_.pop_front();
|
||||
}
|
||||
|
||||
// 4. Check for sufficient data.
|
||||
if (qp_window_.size() <= params_.recent_window_length) {
|
||||
// No need to update state.
|
||||
RTC_CHECK(at_target_quality_ == false);
|
||||
return;
|
||||
}
|
||||
|
||||
// 5. Calculate average QP.
|
||||
float qp_past_average =
|
||||
std::accumulate(qp_window_.begin(),
|
||||
qp_window_.end() - params_.recent_window_length, 0.0) /
|
||||
(qp_window_.size() - params_.recent_window_length);
|
||||
float qp_recent_average =
|
||||
std::accumulate(qp_window_.end() - params_.recent_window_length,
|
||||
qp_window_.end(), 0.0) /
|
||||
params_.recent_window_length;
|
||||
// 6. Determine convergence.
|
||||
if (qp_past_average <= params_.dynamic_qp_threshold &&
|
||||
qp_past_average <= qp_recent_average) {
|
||||
at_target_quality_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool QualityConvergenceMonitor::AtTargetQuality() const {
|
||||
return at_target_quality_;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
70
video/quality_convergence_monitor.h
Normal file
70
video/quality_convergence_monitor.h
Normal file
@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#ifndef VIDEO_QUALITY_CONVERGENCE_MONITOR_H_
|
||||
#define VIDEO_QUALITY_CONVERGENCE_MONITOR_H_
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
|
||||
#include "api/video/video_codec_type.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
class QualityConvergenceMonitor {
|
||||
public:
|
||||
struct Parameters {
|
||||
// Static QP threshold. No history or even refresh-frame requirements to
|
||||
// determine that target quality is reached if the QP value is at or below
|
||||
// this threshold.
|
||||
int static_qp_threshold = 0;
|
||||
|
||||
// Determines if the dynamic threshold should be used for refresh frames.
|
||||
bool dynamic_detection_enabled = false;
|
||||
|
||||
// Window lengths of QP values to use when determining if refresh frames
|
||||
// have reached the target quality. The combined window length is
|
||||
// `past_window_length` + `recent_window_length`. The recent part of the
|
||||
// window contains the most recent samples. Once the recent buffer reaches
|
||||
// this length, new samples will pop the oldest samples in recent and move
|
||||
// them to the past buffer. The average of `QP_past` must be equal to or
|
||||
// less than the average of `QP_recent` to determine that target quality is
|
||||
// reached. See the implementation in `AddSample()`.
|
||||
size_t recent_window_length = 0;
|
||||
size_t past_window_length = 0;
|
||||
|
||||
// During dynamic detection, the average of `QP_past` must be less than or
|
||||
// equal to this threshold to determine that target quality is reached.
|
||||
int dynamic_qp_threshold = 0;
|
||||
};
|
||||
|
||||
explicit QualityConvergenceMonitor(const Parameters& params);
|
||||
|
||||
// Add the supplied `qp` value to the detection window.
|
||||
// `is_refresh_frame` must only be `true` if the corresponding
|
||||
// video frame is a refresh frame that is used to improve the visual quality.
|
||||
void AddSample(int qp, bool is_refresh_frame);
|
||||
|
||||
// Returns `true` if the algorithm has determined that the supplied QP values
|
||||
// have converged and reached the target quality.
|
||||
bool AtTargetQuality() const;
|
||||
|
||||
private:
|
||||
const Parameters params_;
|
||||
bool at_target_quality_ = false;
|
||||
|
||||
// Contains a window of QP values. New values are added at the back while old
|
||||
// values are popped from the front to maintain the configured window length.
|
||||
std::deque<int> qp_window_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // VIDEO_QUALITY_CONVERGENCE_MONITOR_H_
|
||||
196
video/quality_convergence_monitor_unittest.cc
Normal file
196
video/quality_convergence_monitor_unittest.cc
Normal file
@ -0,0 +1,196 @@
|
||||
|
||||
/*
|
||||
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
#include "video/quality_convergence_monitor.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "test/gtest.h"
|
||||
|
||||
namespace webrtc {
|
||||
namespace {
|
||||
|
||||
constexpr QualityConvergenceMonitor::Parameters kParametersOnlyStaticThreshold =
|
||||
{.static_qp_threshold = 13, .dynamic_detection_enabled = false};
|
||||
constexpr QualityConvergenceMonitor::Parameters
|
||||
kParametersWithDynamicDetection = {.static_qp_threshold = 13,
|
||||
.dynamic_detection_enabled = true,
|
||||
.recent_window_length = 3,
|
||||
.past_window_length = 9,
|
||||
.dynamic_qp_threshold = 24};
|
||||
|
||||
// Test the basics of the algorithm.
|
||||
|
||||
TEST(QualityConvergenceMonitorAlgorithm, StaticThreshold) {
|
||||
QualityConvergenceMonitor::Parameters p = kParametersOnlyStaticThreshold;
|
||||
auto monitor = std::make_unique<QualityConvergenceMonitor>(p);
|
||||
ASSERT_TRUE(monitor);
|
||||
|
||||
for (bool is_refresh_frame : {false, true}) {
|
||||
// Ramp down from 100. Not at target quality until qp <= static threshold.
|
||||
for (int qp = 100; qp > p.static_qp_threshold; --qp) {
|
||||
monitor->AddSample(qp, is_refresh_frame);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
monitor->AddSample(p.static_qp_threshold, is_refresh_frame);
|
||||
EXPECT_TRUE(monitor->AtTargetQuality());
|
||||
|
||||
// 100 samples just above the threshold is not at target quality.
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
monitor->AddSample(p.static_qp_threshold + 1, is_refresh_frame);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(QualityConvergenceMonitorAlgorithm,
|
||||
StaticThresholdWithDynamicDetectionEnabled) {
|
||||
QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection;
|
||||
auto monitor = std::make_unique<QualityConvergenceMonitor>(p);
|
||||
ASSERT_TRUE(monitor);
|
||||
|
||||
for (bool is_refresh_frame : {false, true}) {
|
||||
// Clear buffer.
|
||||
monitor->AddSample(-1, /*is_refresh_frame=*/false);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
|
||||
// Ramp down from 100. Not at target quality until qp <= static threshold.
|
||||
for (int qp = 100; qp > p.static_qp_threshold; --qp) {
|
||||
monitor->AddSample(qp, is_refresh_frame);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
// A single frame at the static QP threshold is considered to be at target
|
||||
// quality regardless of if it's a refresh frame or not.
|
||||
monitor->AddSample(p.static_qp_threshold, is_refresh_frame);
|
||||
EXPECT_TRUE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
// 100 samples just above the threshold is not at target quality if it's not a
|
||||
// refresh frame.
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
monitor->AddSample(p.static_qp_threshold + 1, /*is_refresh_frame=*/false);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(QualityConvergenceMonitorAlgorithm, ConvergenceAtDynamicThreshold) {
|
||||
QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection;
|
||||
auto monitor = std::make_unique<QualityConvergenceMonitor>(p);
|
||||
ASSERT_TRUE(monitor);
|
||||
|
||||
// `recent_window_length` + `past_window_length` refresh frames at the dynamic
|
||||
// threshold must mean we're at target quality.
|
||||
for (size_t i = 0; i < p.recent_window_length + p.past_window_length; ++i) {
|
||||
monitor->AddSample(p.dynamic_qp_threshold, /*is_refresh_frame=*/true);
|
||||
}
|
||||
EXPECT_TRUE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
TEST(QualityConvergenceMonitorAlgorithm, NoConvergenceAboveDynamicThreshold) {
|
||||
QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection;
|
||||
auto monitor = std::make_unique<QualityConvergenceMonitor>(p);
|
||||
ASSERT_TRUE(monitor);
|
||||
|
||||
// 100 samples just above the threshold must imply that we're not at target
|
||||
// quality.
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
monitor->AddSample(p.dynamic_qp_threshold + 1, /*is_refresh_frame=*/true);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(QualityConvergenceMonitorAlgorithm,
|
||||
MaintainAtTargetQualityForRefreshFrames) {
|
||||
QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection;
|
||||
auto monitor = std::make_unique<QualityConvergenceMonitor>(p);
|
||||
ASSERT_TRUE(monitor);
|
||||
|
||||
// `recent_window_length` + `past_window_length` refresh frames at the dynamic
|
||||
// threshold must mean we're at target quality.
|
||||
for (size_t i = 0; i < p.recent_window_length + p.past_window_length; ++i) {
|
||||
monitor->AddSample(p.dynamic_qp_threshold, /*is_refresh_frame=*/true);
|
||||
}
|
||||
EXPECT_TRUE(monitor->AtTargetQuality());
|
||||
|
||||
int qp = p.dynamic_qp_threshold;
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
monitor->AddSample(qp++, /*is_refresh_frame=*/true);
|
||||
EXPECT_TRUE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
// Reset state for first frame that is not a refresh frame.
|
||||
monitor->AddSample(qp, /*is_refresh_frame=*/false);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
// Test corner cases.
|
||||
|
||||
TEST(QualityConvergenceMonitorAlgorithm, SufficientData) {
|
||||
QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection;
|
||||
auto monitor = std::make_unique<QualityConvergenceMonitor>(p);
|
||||
ASSERT_TRUE(monitor);
|
||||
|
||||
// Less than `recent_window_length + 1` refresh frame QP values at the dynamic
|
||||
// threshold is not sufficient.
|
||||
for (size_t i = 0; i < p.recent_window_length; ++i) {
|
||||
monitor->AddSample(p.dynamic_qp_threshold, /*is_refresh_frame=*/true);
|
||||
// Not sufficient data
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
// However, `recent_window_length + 1` QP values are sufficient.
|
||||
monitor->AddSample(p.dynamic_qp_threshold, /*is_refresh_frame=*/true);
|
||||
EXPECT_TRUE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
TEST(QualityConvergenceMonitorAlgorithm,
|
||||
AtTargetIfQpPastLessThanOrEqualToQpRecent) {
|
||||
QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection;
|
||||
p.past_window_length = 3;
|
||||
p.recent_window_length = 3;
|
||||
auto monitor = std::make_unique<QualityConvergenceMonitor>(p);
|
||||
|
||||
// Sequence for which QP_past > QP_recent.
|
||||
for (int qp : {23, 21, 21, 21, 21, 22}) {
|
||||
monitor->AddSample(qp, /*is_refresh_frame=*/true);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
// Reset QP window.
|
||||
monitor->AddSample(-1, /*is_refresh_frame=*/false);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
|
||||
// Sequence for which one additional sample of 22 will make QP_past ==
|
||||
// QP_recent.
|
||||
for (int qp : {22, 21, 21, 21, 21}) {
|
||||
monitor->AddSample(qp, /*is_refresh_frame=*/true);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
monitor->AddSample(22, /*is_refresh_frame=*/true);
|
||||
EXPECT_TRUE(monitor->AtTargetQuality());
|
||||
|
||||
// Reset QP window.
|
||||
monitor->AddSample(-1, /*is_refresh_frame=*/false);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
|
||||
// Sequence for which one additional sample of 23 will make QP_past <
|
||||
// QP_recent.
|
||||
for (int qp : {22, 21, 21, 21, 21}) {
|
||||
monitor->AddSample(qp, /*is_refresh_frame=*/true);
|
||||
EXPECT_FALSE(monitor->AtTargetQuality());
|
||||
}
|
||||
monitor->AddSample(23, /*is_refresh_frame=*/true);
|
||||
EXPECT_TRUE(monitor->AtTargetQuality());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace webrtc
|
||||
Loading…
x
Reference in New Issue
Block a user