diff --git a/video/BUILD.gn b/video/BUILD.gn index 227febc4d8..a9773f6895 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -388,6 +388,8 @@ rtc_library("video_stream_encoder_impl") { "encoder_overshoot_detector.h", "frame_encode_metadata_writer.cc", "frame_encode_metadata_writer.h", + "quality_convergence_monitor.cc", + "quality_convergence_monitor.h", "video_source_sink_controller.cc", "video_source_sink_controller.h", "video_stream_encoder.cc", @@ -771,6 +773,7 @@ if (rtc_include_tests) { "frame_decode_timing_unittest.cc", "frame_encode_metadata_writer_unittest.cc", "picture_id_tests.cc", + "quality_convergence_monitor_unittest.cc", "quality_limitation_reason_tracker_unittest.cc", "quality_scaling_tests.cc", "receive_statistics_proxy_unittest.cc", diff --git a/video/quality_convergence_monitor.cc b/video/quality_convergence_monitor.cc new file mode 100644 index 0000000000..c05f0d3175 --- /dev/null +++ b/video/quality_convergence_monitor.cc @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/quality_convergence_monitor.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +QualityConvergenceMonitor::QualityConvergenceMonitor(const Parameters& params) + : params_(params) { + RTC_CHECK( + !params_.dynamic_detection_enabled || + (params_.past_window_length > 0 && params_.recent_window_length > 0)); +} + +// Adds the sample to the algorithms detection window and runs the following +// convergence detection algorithm to determine if the time series of QP +// values indicates that the encoded video has reached "target quality". +// +// Definitions +// +// - Let x[n] be the pixel data of a video frame. +// - Let e[n] be the encoded representation of x[n]. +// - Let qp[n] be the corresponding QP value of the encoded video frame e[n]. +// - x[n] is a refresh frame if x[n] = x[n-1]. +// - qp_window is a list (or queue) of stored QP values, with size +// L <= past_window_length + recent_window_length. +// - qp_window can be partioned into: +// qp_past = qp_window[ 0:end-recent_window_length ] and +// qp_recent = qp_window[ -recent_window_length:end ]. +// - Let dynamic_qp_threshold be a maximum QP value for which convergence +// is accepted. +// +// Algorithm +// +// For each encoded video frame e[n], take the corresponding qp[n] and do the +// following: +// 0. Check Static Threshold: if qp[n] < static_qp_threshold, return true. +// 1. Check for Refresh Frame: If x[n] is not a refresh frame: +// - Clear Q. +// - Return false. +// 2. Check Previous Convergence: If x[n] is a refresh frame AND true was +// returned for x[n-1], return true. +// 3. Update QP History: Append qp[n] to qp_window. If qp_window's length +// exceeds past_window_length + recent_window_length, remove the first +// element. +// 4. Check for Sufficient Data: If L <= recent_window_length, return false. +// 5. Calculate Average QP: Calculate avg(qp_past) and avg(ap_recent). +// 6. Determine Convergence: If avg(qp_past) <= dynamic_qp_threshold AND +// avg(qp_past) <= avg(qp_recent), return true. Otherwise, return false. +// +void QualityConvergenceMonitor::AddSample(int qp, bool is_refresh_frame) { + // Invalid QP. + if (qp < 0) { + qp_window_.clear(); + at_target_quality_ = false; + return; + } + + // 0. Check static threshold. + if (qp <= params_.static_qp_threshold) { + at_target_quality_ = true; + return; + } + + // 1. Check for refresh frame and if dynamic detection is disabled. + if (!is_refresh_frame || !params_.dynamic_detection_enabled) { + qp_window_.clear(); + at_target_quality_ = false; + return; + } + + // 2. Check previous convergence. + RTC_CHECK(is_refresh_frame); + if (at_target_quality_) { + // No need to update state. + return; + } + + // 3. Update QP history. + qp_window_.push_back(qp); + if (qp_window_.size() > + params_.recent_window_length + params_.past_window_length) { + qp_window_.pop_front(); + } + + // 4. Check for sufficient data. + if (qp_window_.size() <= params_.recent_window_length) { + // No need to update state. + RTC_CHECK(at_target_quality_ == false); + return; + } + + // 5. Calculate average QP. + float qp_past_average = + std::accumulate(qp_window_.begin(), + qp_window_.end() - params_.recent_window_length, 0.0) / + (qp_window_.size() - params_.recent_window_length); + float qp_recent_average = + std::accumulate(qp_window_.end() - params_.recent_window_length, + qp_window_.end(), 0.0) / + params_.recent_window_length; + // 6. Determine convergence. + if (qp_past_average <= params_.dynamic_qp_threshold && + qp_past_average <= qp_recent_average) { + at_target_quality_ = true; + } +} + +bool QualityConvergenceMonitor::AtTargetQuality() const { + return at_target_quality_; +} + +} // namespace webrtc diff --git a/video/quality_convergence_monitor.h b/video/quality_convergence_monitor.h new file mode 100644 index 0000000000..6bb3acbad5 --- /dev/null +++ b/video/quality_convergence_monitor.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_QUALITY_CONVERGENCE_MONITOR_H_ +#define VIDEO_QUALITY_CONVERGENCE_MONITOR_H_ + +#include +#include + +#include "api/video/video_codec_type.h" + +namespace webrtc { + +class QualityConvergenceMonitor { + public: + struct Parameters { + // Static QP threshold. No history or even refresh-frame requirements to + // determine that target quality is reached if the QP value is at or below + // this threshold. + int static_qp_threshold = 0; + + // Determines if the dynamic threshold should be used for refresh frames. + bool dynamic_detection_enabled = false; + + // Window lengths of QP values to use when determining if refresh frames + // have reached the target quality. The combined window length is + // `past_window_length` + `recent_window_length`. The recent part of the + // window contains the most recent samples. Once the recent buffer reaches + // this length, new samples will pop the oldest samples in recent and move + // them to the past buffer. The average of `QP_past` must be equal to or + // less than the average of `QP_recent` to determine that target quality is + // reached. See the implementation in `AddSample()`. + size_t recent_window_length = 0; + size_t past_window_length = 0; + + // During dynamic detection, the average of `QP_past` must be less than or + // equal to this threshold to determine that target quality is reached. + int dynamic_qp_threshold = 0; + }; + + explicit QualityConvergenceMonitor(const Parameters& params); + + // Add the supplied `qp` value to the detection window. + // `is_refresh_frame` must only be `true` if the corresponding + // video frame is a refresh frame that is used to improve the visual quality. + void AddSample(int qp, bool is_refresh_frame); + + // Returns `true` if the algorithm has determined that the supplied QP values + // have converged and reached the target quality. + bool AtTargetQuality() const; + + private: + const Parameters params_; + bool at_target_quality_ = false; + + // Contains a window of QP values. New values are added at the back while old + // values are popped from the front to maintain the configured window length. + std::deque qp_window_; +}; + +} // namespace webrtc + +#endif // VIDEO_QUALITY_CONVERGENCE_MONITOR_H_ diff --git a/video/quality_convergence_monitor_unittest.cc b/video/quality_convergence_monitor_unittest.cc new file mode 100644 index 0000000000..fe5fd97944 --- /dev/null +++ b/video/quality_convergence_monitor_unittest.cc @@ -0,0 +1,196 @@ + +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "video/quality_convergence_monitor.h" + +#include + +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr QualityConvergenceMonitor::Parameters kParametersOnlyStaticThreshold = + {.static_qp_threshold = 13, .dynamic_detection_enabled = false}; +constexpr QualityConvergenceMonitor::Parameters + kParametersWithDynamicDetection = {.static_qp_threshold = 13, + .dynamic_detection_enabled = true, + .recent_window_length = 3, + .past_window_length = 9, + .dynamic_qp_threshold = 24}; + +// Test the basics of the algorithm. + +TEST(QualityConvergenceMonitorAlgorithm, StaticThreshold) { + QualityConvergenceMonitor::Parameters p = kParametersOnlyStaticThreshold; + auto monitor = std::make_unique(p); + ASSERT_TRUE(monitor); + + for (bool is_refresh_frame : {false, true}) { + // Ramp down from 100. Not at target quality until qp <= static threshold. + for (int qp = 100; qp > p.static_qp_threshold; --qp) { + monitor->AddSample(qp, is_refresh_frame); + EXPECT_FALSE(monitor->AtTargetQuality()); + } + + monitor->AddSample(p.static_qp_threshold, is_refresh_frame); + EXPECT_TRUE(monitor->AtTargetQuality()); + + // 100 samples just above the threshold is not at target quality. + for (int i = 0; i < 100; ++i) { + monitor->AddSample(p.static_qp_threshold + 1, is_refresh_frame); + EXPECT_FALSE(monitor->AtTargetQuality()); + } + } +} + +TEST(QualityConvergenceMonitorAlgorithm, + StaticThresholdWithDynamicDetectionEnabled) { + QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection; + auto monitor = std::make_unique(p); + ASSERT_TRUE(monitor); + + for (bool is_refresh_frame : {false, true}) { + // Clear buffer. + monitor->AddSample(-1, /*is_refresh_frame=*/false); + EXPECT_FALSE(monitor->AtTargetQuality()); + + // Ramp down from 100. Not at target quality until qp <= static threshold. + for (int qp = 100; qp > p.static_qp_threshold; --qp) { + monitor->AddSample(qp, is_refresh_frame); + EXPECT_FALSE(monitor->AtTargetQuality()); + } + + // A single frame at the static QP threshold is considered to be at target + // quality regardless of if it's a refresh frame or not. + monitor->AddSample(p.static_qp_threshold, is_refresh_frame); + EXPECT_TRUE(monitor->AtTargetQuality()); + } + + // 100 samples just above the threshold is not at target quality if it's not a + // refresh frame. + for (int i = 0; i < 100; ++i) { + monitor->AddSample(p.static_qp_threshold + 1, /*is_refresh_frame=*/false); + EXPECT_FALSE(monitor->AtTargetQuality()); + } +} + +TEST(QualityConvergenceMonitorAlgorithm, ConvergenceAtDynamicThreshold) { + QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection; + auto monitor = std::make_unique(p); + ASSERT_TRUE(monitor); + + // `recent_window_length` + `past_window_length` refresh frames at the dynamic + // threshold must mean we're at target quality. + for (size_t i = 0; i < p.recent_window_length + p.past_window_length; ++i) { + monitor->AddSample(p.dynamic_qp_threshold, /*is_refresh_frame=*/true); + } + EXPECT_TRUE(monitor->AtTargetQuality()); +} + +TEST(QualityConvergenceMonitorAlgorithm, NoConvergenceAboveDynamicThreshold) { + QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection; + auto monitor = std::make_unique(p); + ASSERT_TRUE(monitor); + + // 100 samples just above the threshold must imply that we're not at target + // quality. + for (int i = 0; i < 100; ++i) { + monitor->AddSample(p.dynamic_qp_threshold + 1, /*is_refresh_frame=*/true); + EXPECT_FALSE(monitor->AtTargetQuality()); + } +} + +TEST(QualityConvergenceMonitorAlgorithm, + MaintainAtTargetQualityForRefreshFrames) { + QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection; + auto monitor = std::make_unique(p); + ASSERT_TRUE(monitor); + + // `recent_window_length` + `past_window_length` refresh frames at the dynamic + // threshold must mean we're at target quality. + for (size_t i = 0; i < p.recent_window_length + p.past_window_length; ++i) { + monitor->AddSample(p.dynamic_qp_threshold, /*is_refresh_frame=*/true); + } + EXPECT_TRUE(monitor->AtTargetQuality()); + + int qp = p.dynamic_qp_threshold; + for (int i = 0; i < 100; ++i) { + monitor->AddSample(qp++, /*is_refresh_frame=*/true); + EXPECT_TRUE(monitor->AtTargetQuality()); + } + + // Reset state for first frame that is not a refresh frame. + monitor->AddSample(qp, /*is_refresh_frame=*/false); + EXPECT_FALSE(monitor->AtTargetQuality()); +} + +// Test corner cases. + +TEST(QualityConvergenceMonitorAlgorithm, SufficientData) { + QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection; + auto monitor = std::make_unique(p); + ASSERT_TRUE(monitor); + + // Less than `recent_window_length + 1` refresh frame QP values at the dynamic + // threshold is not sufficient. + for (size_t i = 0; i < p.recent_window_length; ++i) { + monitor->AddSample(p.dynamic_qp_threshold, /*is_refresh_frame=*/true); + // Not sufficient data + EXPECT_FALSE(monitor->AtTargetQuality()); + } + + // However, `recent_window_length + 1` QP values are sufficient. + monitor->AddSample(p.dynamic_qp_threshold, /*is_refresh_frame=*/true); + EXPECT_TRUE(monitor->AtTargetQuality()); +} + +TEST(QualityConvergenceMonitorAlgorithm, + AtTargetIfQpPastLessThanOrEqualToQpRecent) { + QualityConvergenceMonitor::Parameters p = kParametersWithDynamicDetection; + p.past_window_length = 3; + p.recent_window_length = 3; + auto monitor = std::make_unique(p); + + // Sequence for which QP_past > QP_recent. + for (int qp : {23, 21, 21, 21, 21, 22}) { + monitor->AddSample(qp, /*is_refresh_frame=*/true); + EXPECT_FALSE(monitor->AtTargetQuality()); + } + + // Reset QP window. + monitor->AddSample(-1, /*is_refresh_frame=*/false); + EXPECT_FALSE(monitor->AtTargetQuality()); + + // Sequence for which one additional sample of 22 will make QP_past == + // QP_recent. + for (int qp : {22, 21, 21, 21, 21}) { + monitor->AddSample(qp, /*is_refresh_frame=*/true); + EXPECT_FALSE(monitor->AtTargetQuality()); + } + monitor->AddSample(22, /*is_refresh_frame=*/true); + EXPECT_TRUE(monitor->AtTargetQuality()); + + // Reset QP window. + monitor->AddSample(-1, /*is_refresh_frame=*/false); + EXPECT_FALSE(monitor->AtTargetQuality()); + + // Sequence for which one additional sample of 23 will make QP_past < + // QP_recent. + for (int qp : {22, 21, 21, 21, 21}) { + monitor->AddSample(qp, /*is_refresh_frame=*/true); + EXPECT_FALSE(monitor->AtTargetQuality()); + } + monitor->AddSample(23, /*is_refresh_frame=*/true); + EXPECT_TRUE(monitor->AtTargetQuality()); +} + +} // namespace +} // namespace webrtc