diff --git a/modules/audio_processing/agc2/BUILD.gn b/modules/audio_processing/agc2/BUILD.gn index f7d7842c07..d39e3279b4 100644 --- a/modules/audio_processing/agc2/BUILD.gn +++ b/modules/audio_processing/agc2/BUILD.gn @@ -95,25 +95,6 @@ rtc_library("clipping_predictor") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } -rtc_library("clipping_predictor_evaluator") { - visibility = [ - "../agc:agc", - "./*", - ] - - sources = [ - "clipping_predictor_evaluator.cc", - "clipping_predictor_evaluator.h", - ] - - deps = [ - "../../../rtc_base:checks", - "../../../rtc_base:logging", - ] - - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] -} - rtc_source_set("common") { sources = [ "agc2_common.h" ] } @@ -331,7 +312,6 @@ rtc_library("fixed_digital_unittests") { rtc_library("input_volume_controller_unittests") { testonly = true sources = [ - "clipping_predictor_evaluator_unittest.cc", "clipping_predictor_level_buffer_unittest.cc", "clipping_predictor_unittest.cc", "speech_probability_buffer_unittest.cc", @@ -341,7 +321,6 @@ rtc_library("input_volume_controller_unittests") { deps = [ ":clipping_predictor", - ":clipping_predictor_evaluator", ":gain_map", ":input_volume_controller", "../../../rtc_base:checks", diff --git a/modules/audio_processing/agc2/clipping_predictor_evaluator.cc b/modules/audio_processing/agc2/clipping_predictor_evaluator.cc deleted file mode 100644 index 0bda41bd75..0000000000 --- a/modules/audio_processing/agc2/clipping_predictor_evaluator.cc +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/clipping_predictor_evaluator.h" - -#include - -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" - -namespace webrtc { -namespace { - -// Returns the index of the oldest item in the ring buffer for a non-empty -// ring buffer with give `size`, `tail` index and `capacity`. -int OldestExpectedDetectionIndex(int size, int tail, int capacity) { - RTC_DCHECK_GT(size, 0); - return tail - size + (tail < size ? capacity : 0); -} - -} // namespace - -ClippingPredictorEvaluator::ClippingPredictorEvaluator(int history_size) - : history_size_(history_size), - ring_buffer_capacity_(history_size + 1), - ring_buffer_(ring_buffer_capacity_) { - RTC_DCHECK_GT(history_size_, 0); - Reset(); - counters_.true_positives = 0; - counters_.true_negatives = 0; - counters_.false_positives = 0; - counters_.false_negatives = 0; -} - -ClippingPredictorEvaluator::~ClippingPredictorEvaluator() = default; - -absl::optional ClippingPredictorEvaluator::Observe( - bool clipping_detected, - bool clipping_predicted) { - RTC_DCHECK_GE(ring_buffer_size_, 0); - RTC_DCHECK_LE(ring_buffer_size_, ring_buffer_capacity_); - RTC_DCHECK_GE(ring_buffer_tail_, 0); - RTC_DCHECK_LT(ring_buffer_tail_, ring_buffer_capacity_); - - DecreaseTimesToLive(); - // Clipping is expected if there are expected detections regardless of - // whether all the expected detections have been previously matched - i.e., - // `ExpectedDetection::detected` is true. - const bool clipping_expected = ring_buffer_size_ > 0; - - absl::optional prediction_interval; - if (clipping_expected && clipping_detected) { - prediction_interval = FindEarliestPredictionInterval(); - // Add a true positive for each unexpired expected detection. - const int num_modified_items = MarkExpectedDetectionAsDetected(); - counters_.true_positives += num_modified_items; - RTC_DCHECK(prediction_interval.has_value() || num_modified_items == 0); - RTC_DCHECK(!prediction_interval.has_value() || num_modified_items > 0); - } else if (clipping_expected && !clipping_detected) { - // Add a false positive if there is one expected detection that has expired - // and that has never been matched before. Note that there is at most one - // unmatched expired detection. - if (HasExpiredUnmatchedExpectedDetection()) { - counters_.false_positives++; - } - } else if (!clipping_expected && clipping_detected) { - counters_.false_negatives++; - } else { - RTC_DCHECK(!clipping_expected && !clipping_detected); - counters_.true_negatives++; - } - - if (clipping_predicted) { - // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. - Push(/*expected_detection=*/{/*ttl=*/history_size_, /*detected=*/false}); - } - - return prediction_interval; -} - -void ClippingPredictorEvaluator::RemoveExpectations() { - // Empty the ring buffer of expected detections. - ring_buffer_tail_ = 0; - ring_buffer_size_ = 0; -} - -void ClippingPredictorEvaluator::Reset() { - counters_.true_positives = 0; - counters_.true_negatives = 0; - counters_.false_positives = 0; - counters_.false_negatives = 0; - RemoveExpectations(); -} - -// Cost: O(1). -void ClippingPredictorEvaluator::Push(ExpectedDetection value) { - ring_buffer_[ring_buffer_tail_] = value; - ring_buffer_tail_++; - if (ring_buffer_tail_ == ring_buffer_capacity_) { - ring_buffer_tail_ = 0; - } - ring_buffer_size_ = std::min(ring_buffer_capacity_, ring_buffer_size_ + 1); -} - -// Cost: O(N). -void ClippingPredictorEvaluator::DecreaseTimesToLive() { - bool expired_found = false; - for (int i = ring_buffer_tail_ - ring_buffer_size_; i < ring_buffer_tail_; - ++i) { - int index = i >= 0 ? i : ring_buffer_capacity_ + i; - RTC_DCHECK_GE(index, 0); - RTC_DCHECK_LT(index, ring_buffer_.size()); - RTC_DCHECK_GE(ring_buffer_[index].ttl, 0); - if (ring_buffer_[index].ttl == 0) { - RTC_DCHECK(!expired_found) - << "There must be at most one expired item in the ring buffer."; - expired_found = true; - RTC_DCHECK_EQ(index, OldestExpectedDetectionIndex(ring_buffer_size_, - ring_buffer_tail_, - ring_buffer_capacity_)) - << "The expired item must be the oldest in the ring buffer."; - } - ring_buffer_[index].ttl--; - } - if (expired_found) { - ring_buffer_size_--; - } -} - -// Cost: O(N). -absl::optional ClippingPredictorEvaluator::FindEarliestPredictionInterval() - const { - absl::optional prediction_interval; - for (int i = ring_buffer_tail_ - ring_buffer_size_; i < ring_buffer_tail_; - ++i) { - int index = i >= 0 ? i : ring_buffer_capacity_ + i; - RTC_DCHECK_GE(index, 0); - RTC_DCHECK_LT(index, ring_buffer_.size()); - if (!ring_buffer_[index].detected) { - prediction_interval = std::max(prediction_interval.value_or(0), - history_size_ - ring_buffer_[index].ttl); - } - } - return prediction_interval; -} - -// Cost: O(N). -int ClippingPredictorEvaluator::MarkExpectedDetectionAsDetected() { - int num_modified_items = 0; - for (int i = ring_buffer_tail_ - ring_buffer_size_; i < ring_buffer_tail_; - ++i) { - int index = i >= 0 ? i : ring_buffer_capacity_ + i; - RTC_DCHECK_GE(index, 0); - RTC_DCHECK_LT(index, ring_buffer_.size()); - if (!ring_buffer_[index].detected) { - num_modified_items++; - } - ring_buffer_[index].detected = true; - } - return num_modified_items; -} - -// Cost: O(1). -bool ClippingPredictorEvaluator::HasExpiredUnmatchedExpectedDetection() const { - if (ring_buffer_size_ == 0) { - return false; - } - // If an expired item, that is `ttl` equal to 0, exists, it must be the - // oldest. - const int oldest_index = OldestExpectedDetectionIndex( - ring_buffer_size_, ring_buffer_tail_, ring_buffer_capacity_); - RTC_DCHECK_GE(oldest_index, 0); - RTC_DCHECK_LT(oldest_index, ring_buffer_.size()); - return ring_buffer_[oldest_index].ttl == 0 && - !ring_buffer_[oldest_index].detected; -} - -absl::optional ComputeClippingPredictionMetrics( - const ClippingPredictionCounters& counters) { - RTC_DCHECK_GE(counters.true_positives, 0); - RTC_DCHECK_GE(counters.true_negatives, 0); - RTC_DCHECK_GE(counters.false_positives, 0); - RTC_DCHECK_GE(counters.false_negatives, 0); - if (counters.true_positives == 0) { - // Both precision and recall are zero in this case and hence the F1 score - // is undefined. - return absl::nullopt; - } - int precision_denominator = - counters.true_positives + counters.false_positives; - int recall_denominator = counters.true_positives + counters.false_negatives; - if (precision_denominator == 0 || recall_denominator == 0) { - // Both precision and recall must be defined. - return absl::nullopt; - } - ClippingPredictionMetrics metrics; - float true_positives = counters.true_positives; - metrics.precision = true_positives / precision_denominator; - metrics.recall = true_positives / recall_denominator; - float f1_score_denominator = metrics.precision + metrics.recall; - RTC_DCHECK_GT(f1_score_denominator, 0.0f); - metrics.f1_score = - 2 * metrics.precision * metrics.recall / f1_score_denominator; - return metrics; -} - -} // namespace webrtc diff --git a/modules/audio_processing/agc2/clipping_predictor_evaluator.h b/modules/audio_processing/agc2/clipping_predictor_evaluator.h deleted file mode 100644 index 084ae90071..0000000000 --- a/modules/audio_processing/agc2/clipping_predictor_evaluator.h +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_EVALUATOR_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_EVALUATOR_H_ - -#include - -#include "absl/types/optional.h" - -namespace webrtc { - -// Clipping prediction counters. -struct ClippingPredictionCounters { - int true_positives; // TP. - int true_negatives; // TN. - int false_positives; // FP. - int false_negatives; // FN. -}; - -// Counts true/false positives/negatives while observing sequences of flag pairs -// that indicate whether clipping has been detected and/or if clipping is -// predicted. When a true positive is found measures the time interval between -// prediction and detection events. -// After a prediction is observed and for a period equal to -// `history_size` calls to `Observe()`, one or more detections are expected. If -// the expectation is met, a true positive is added and the time interval -// between the earliest prediction and the detection is recorded; otherwise, -// when the deadline is reached, a false positive is added. Note that one -// detection matches all the expected detections that have not expired - i.e., -// one detection counts as multiple true positives. -// If a detection is observed, but no prediction has been observed over the past -// `history_size` calls to `Observe()`, then a false negative is added; -// otherwise, a true negative is added. -class ClippingPredictorEvaluator { - public: - // Ctor. `history_size` indicates how long to wait for a call to `Observe()` - // having `clipping_detected` set to true from the time clipping is predicted. - explicit ClippingPredictorEvaluator(int history_size); - ClippingPredictorEvaluator(const ClippingPredictorEvaluator&) = delete; - ClippingPredictorEvaluator& operator=(const ClippingPredictorEvaluator&) = - delete; - ~ClippingPredictorEvaluator(); - - // Observes whether clipping has been detected and/or if clipping is - // predicted. When predicted one or more detections are expected in the next - // `history_size_` calls of `Observe()`. When true positives are found returns - // the prediction interval between the earliest prediction and the detection. - absl::optional Observe(bool clipping_detected, bool clipping_predicted); - - // Removes any expectation recently set after a call to `Observe()` having - // `clipping_predicted` set to true. Counters won't be reset. - void RemoveExpectations(); - - // Resets counters and removes any expectation (see `RemoveExpectations()`). - void Reset(); - - ClippingPredictionCounters counters() const { return counters_; } - - private: - const int history_size_; - - // State of a detection expected to be observed after a prediction. - struct ExpectedDetection { - // Time to live (TTL); remaining number of `Observe()` calls to match a call - // having `clipping_detected` set to true. - int ttl; - // True if an `Observe()` call having `clipping_detected` set to true has - // been observed. - bool detected; - }; - // Ring buffer of expected detections. - const int ring_buffer_capacity_; - std::vector ring_buffer_; - int ring_buffer_tail_; - int ring_buffer_size_; - - // Pushes `expected_detection` into `expected_matches_ring_buffer_`. - void Push(ExpectedDetection expected_detection); - // Decreased the TTLs in `expected_matches_ring_buffer_` and removes expired - // items. - void DecreaseTimesToLive(); - // Returns the prediction interval for the earliest unexpired expected - // detection if any. - absl::optional FindEarliestPredictionInterval() const; - // Marks all the items in `expected_matches_ring_buffer_` as `detected` and - // returns the number of updated items. - int MarkExpectedDetectionAsDetected(); - // Returns true if `expected_matches_ring_buffer_` has an item having `ttl` - // equal to 0 (expired) and `detected` equal to false (unmatched). - bool HasExpiredUnmatchedExpectedDetection() const; - - // Counters. - ClippingPredictionCounters counters_; -}; - -// Clipping prediction metrics derived from the clipping prediction counters. -struct ClippingPredictionMetrics { - // Precision (P) is defined as TP / (TP + FP). - float precision; - // Recall (R) is defined as TP / (TP + FN). - float recall; - // The F1 score is defined as 2 * P * R / (P + R). - float f1_score; -}; - -// Derives clipping prediction metrics from the true/false positives/negatives -// `counters`. Returns an unspecified value if one or more metrics are not -// defined. -absl::optional ComputeClippingPredictionMetrics( - const ClippingPredictionCounters& counters); - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_EVALUATOR_H_ diff --git a/modules/audio_processing/agc2/clipping_predictor_evaluator_unittest.cc b/modules/audio_processing/agc2/clipping_predictor_evaluator_unittest.cc deleted file mode 100644 index 8337d44bc3..0000000000 --- a/modules/audio_processing/agc2/clipping_predictor_evaluator_unittest.cc +++ /dev/null @@ -1,763 +0,0 @@ -/* - * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/clipping_predictor_evaluator.h" - -#include -#include -#include -#include - -#include "absl/types/optional.h" -#include "rtc_base/numerics/safe_conversions.h" -#include "rtc_base/random.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace { - -using testing::Eq; -using testing::Field; -using testing::Optional; - -constexpr bool kDetected = true; -constexpr bool kNotDetected = false; - -constexpr bool kPredicted = true; -constexpr bool kNotPredicted = false; - -ClippingPredictionCounters operator-(const ClippingPredictionCounters& lhs, - const ClippingPredictionCounters& rhs) { - return { - lhs.true_positives - rhs.true_positives, - lhs.true_negatives - rhs.true_negatives, - lhs.false_positives - rhs.false_positives, - lhs.false_negatives - rhs.false_negatives, - }; -} - -// Checks the metrics after init - i.e., no call to `Observe()`. -TEST(ClippingPredictionEvalTest, Init) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -class ClippingPredictorEvaluatorParameterization - : public ::testing::TestWithParam> { - protected: - uint64_t seed() const { - return rtc::checked_cast(std::get<0>(GetParam())); - } - int history_size() const { return std::get<1>(GetParam()); } -}; - -// Checks that after each call to `Observe()` at most one metric changes. -TEST_P(ClippingPredictorEvaluatorParameterization, AtMostOneMetricChanges) { - constexpr int kNumCalls = 123; - Random random_generator(seed()); - ClippingPredictorEvaluator evaluator(history_size()); - - for (int i = 0; i < kNumCalls; ++i) { - SCOPED_TRACE(i); - // Read metrics before `Observe()` is called. - const auto pre = evaluator.counters(); - // `Observe()` a random observation. - bool clipping_detected = random_generator.Rand(); - bool clipping_predicted = random_generator.Rand(); - evaluator.Observe(clipping_detected, clipping_predicted); - - // Check that at most one metric has changed. - const auto post = evaluator.counters(); - int num_changes = 0; - num_changes += pre.true_positives == post.true_positives ? 0 : 1; - num_changes += pre.true_negatives == post.true_negatives ? 0 : 1; - num_changes += pre.false_positives == post.false_positives ? 0 : 1; - num_changes += pre.false_negatives == post.false_negatives ? 0 : 1; - EXPECT_GE(num_changes, 0); - EXPECT_LE(num_changes, 1); - } -} - -// Checks that after each call to `Observe()` each metric either remains -// unchanged or grows. -TEST_P(ClippingPredictorEvaluatorParameterization, MetricsAreWeaklyMonotonic) { - constexpr int kNumCalls = 123; - Random random_generator(seed()); - ClippingPredictorEvaluator evaluator(history_size()); - - for (int i = 0; i < kNumCalls; ++i) { - SCOPED_TRACE(i); - // Read metrics before `Observe()` is called. - const auto pre = evaluator.counters(); - // `Observe()` a random observation. - bool clipping_detected = random_generator.Rand(); - bool clipping_predicted = random_generator.Rand(); - evaluator.Observe(clipping_detected, clipping_predicted); - - // Check that metrics are weakly monotonic. - const auto post = evaluator.counters(); - EXPECT_GE(post.true_positives, pre.true_positives); - EXPECT_GE(post.true_negatives, pre.true_negatives); - EXPECT_GE(post.false_positives, pre.false_positives); - EXPECT_GE(post.false_negatives, pre.false_negatives); - } -} - -// Checks that after each call to `Observe()` the growth speed of each metrics -// is bounded. -TEST_P(ClippingPredictorEvaluatorParameterization, BoundedMetricsGrowth) { - constexpr int kNumCalls = 123; - Random random_generator(seed()); - ClippingPredictorEvaluator evaluator(history_size()); - - for (int i = 0; i < kNumCalls; ++i) { - SCOPED_TRACE(i); - // Read metrics before `Observe()` is called. - const auto pre = evaluator.counters(); - // `Observe()` a random observation. - bool clipping_detected = random_generator.Rand(); - bool clipping_predicted = random_generator.Rand(); - evaluator.Observe(clipping_detected, clipping_predicted); - - const auto diff = evaluator.counters() - pre; - // Check that TPs grow by at most `history_size() + 1`. Such an upper bound - // is reached when multiple predictions are matched by a single detection. - EXPECT_LE(diff.true_positives, history_size() + 1); - // Check that TNs, FPs and FNs grow by at most one. - EXPECT_LE(diff.true_negatives, 1); - EXPECT_LE(diff.false_positives, 1); - EXPECT_LE(diff.false_negatives, 1); - } -} - -// Checks that `Observe()` returns a prediction interval if and only if one or -// more true positives are found. -TEST_P(ClippingPredictorEvaluatorParameterization, - PredictionIntervalIfAndOnlyIfTruePositives) { - constexpr int kNumCalls = 123; - Random random_generator(seed()); - ClippingPredictorEvaluator evaluator(history_size()); - - for (int i = 0; i < kNumCalls; ++i) { - SCOPED_TRACE(i); - // Read true positives before `Observe()` is called. - const int last_tp = evaluator.counters().true_positives; - // `Observe()` a random observation. - bool clipping_detected = random_generator.Rand(); - bool clipping_predicted = random_generator.Rand(); - absl::optional prediction_interval = - evaluator.Observe(clipping_detected, clipping_predicted); - - // Check that the prediction interval is returned when a true positive is - // found. - if (evaluator.counters().true_positives == last_tp) { - EXPECT_FALSE(prediction_interval.has_value()); - } else { - EXPECT_TRUE(prediction_interval.has_value()); - } - } -} - -INSTANTIATE_TEST_SUITE_P( - ClippingPredictionEvalTest, - ClippingPredictorEvaluatorParameterization, - ::testing::Combine(::testing::Values(4, 8, 15, 16, 23, 42), - ::testing::Values(1, 10, 21))); - -// Checks that after initialization, when no detection is expected, -// observing no detection and no prediction produces a true negative. -TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectNoPredictAfterInit) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - - evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 1); - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -// Checks that after initialization, when no detection is expected, -// observing no detection and prediction produces a true negative. -TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectPredictAfterInit) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - - evaluator.Observe(kNotDetected, kPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 1); - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -// Checks that after initialization, when no detection is expected, -// observing a detection and no prediction produces a false negative. -TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectNoPredictAfterInit) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 1); -} - -// Checks that after initialization, when no detection is expected, -// simultaneously observing a detection and a prediction produces a false -// negative. -TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectPredictAfterInit) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - - evaluator.Observe(kDetected, kPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 1); -} - -// Checks that, after removing existing expectations, observing no detection and -// no prediction produces a true negative. -TEST(ClippingPredictionEvalTest, - TrueNegativeWithNoDetectNoPredictAfterRemoveExpectations) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - - // Set an expectation, then remove it. - evaluator.Observe(kNotDetected, kPredicted); - evaluator.RemoveExpectations(); - const auto pre = evaluator.counters(); - - evaluator.Observe(kNotDetected, kNotPredicted); - const auto diff = evaluator.counters() - pre; - EXPECT_EQ(diff.true_positives, 0); - EXPECT_EQ(diff.true_negatives, 1); - EXPECT_EQ(diff.false_positives, 0); - EXPECT_EQ(diff.false_negatives, 0); -} - -// Checks that, after removing existing expectations, observing no detection and -// a prediction produces a true negative. -TEST(ClippingPredictionEvalTest, - TrueNegativeWithNoDetectPredictAfterRemoveExpectations) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - - // Set an expectation, then remove it. - evaluator.Observe(kNotDetected, kPredicted); - evaluator.RemoveExpectations(); - const auto pre = evaluator.counters(); - - evaluator.Observe(kNotDetected, kPredicted); - const auto diff = evaluator.counters() - pre; - EXPECT_EQ(diff.true_positives, 0); - EXPECT_EQ(diff.true_negatives, 1); - EXPECT_EQ(diff.false_positives, 0); - EXPECT_EQ(diff.false_negatives, 0); -} - -// Checks that, after removing existing expectations, observing a detection and -// no prediction produces a false negative. -TEST(ClippingPredictionEvalTest, - FalseNegativeWithDetectNoPredictAfterRemoveExpectations) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - - // Set an expectation, then remove it. - evaluator.Observe(kNotDetected, kPredicted); - evaluator.RemoveExpectations(); - const auto pre = evaluator.counters(); - - evaluator.Observe(kDetected, kNotPredicted); - const auto diff = evaluator.counters() - pre; - EXPECT_EQ(diff.true_positives, 0); - EXPECT_EQ(diff.true_negatives, 0); - EXPECT_EQ(diff.false_positives, 0); - EXPECT_EQ(diff.false_negatives, 1); -} - -// Checks that, after removing existing expectations, simultaneously observing a -// detection and a prediction produces a false negative. -TEST(ClippingPredictionEvalTest, - FalseNegativeWithDetectPredictAfterRemoveExpectations) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - - // Set an expectation, then remove it. - evaluator.Observe(kNotDetected, kPredicted); - evaluator.RemoveExpectations(); - const auto pre = evaluator.counters(); - - evaluator.Observe(kDetected, kPredicted); - const auto diff = evaluator.counters() - pre; - EXPECT_EQ(diff.false_negatives, 1); - EXPECT_EQ(diff.true_positives, 0); - EXPECT_EQ(diff.true_negatives, 0); - EXPECT_EQ(diff.false_positives, 0); -} - -// Checks that the evaluator detects true negatives when clipping is neither -// predicted nor detected. -TEST(ClippingPredictionEvalTest, TrueNegativesWhenNeverDetectedOrPredicted) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_negatives, 4); -} - -// Checks that, until the observation period expires, the evaluator does not -// count a false positive when clipping is predicted and not detected. -TEST(ClippingPredictionEvalTest, PredictedOnceAndNeverDetectedBeforeDeadline) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); - evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); - evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 1); -} - -// Checks that, after the observation period expires, the evaluator detects a -// false positive when clipping is predicted and detected. -TEST(ClippingPredictionEvalTest, PredictedOnceButDetectedAfterDeadline) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 1); -} - -// Checks that a prediction followed by a detection counts as true positive. -TEST(ClippingPredictionEvalTest, PredictedOnceAndThenImmediatelyDetected) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 1); - EXPECT_EQ(evaluator.counters().false_positives, 0); -} - -// Checks that a prediction followed by a delayed detection counts as true -// positive if the delay is within the observation period. -TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedBeforeDeadline) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 1); - EXPECT_EQ(evaluator.counters().false_positives, 0); -} - -// Checks that a prediction followed by a delayed detection counts as true -// positive if the delay equals the observation period. -TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedAtDeadline) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 1); - EXPECT_EQ(evaluator.counters().false_positives, 0); -} - -// Checks that a prediction followed by a multiple adjacent detections within -// the deadline counts as a single true positive and that, after the deadline, -// a detection counts as a false negative. -TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedMultipleTimes) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - // Multiple detections. - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 1); - EXPECT_EQ(evaluator.counters().false_negatives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 0); - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 1); - EXPECT_EQ(evaluator.counters().false_negatives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 0); - // A detection outside of the observation period counts as false negative. - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 1); - EXPECT_EQ(evaluator.counters().false_negatives, 1); - EXPECT_EQ(evaluator.counters().false_positives, 0); -} - -// Checks that when clipping is predicted multiple times, a prediction that is -// observed too early counts as a false positive, whereas the other predictions -// that are matched to a detection count as true positives. -TEST(ClippingPredictionEvalTest, - PredictedMultipleTimesAndDetectedOnceAfterDeadline) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); // ---+ - evaluator.Observe(kNotDetected, kPredicted); // | - evaluator.Observe(kNotDetected, kPredicted); // | - evaluator.Observe(kNotDetected, kPredicted); // <--+ Not matched. - // The time to match a detection after the first prediction expired. - EXPECT_EQ(evaluator.counters().false_positives, 1); - evaluator.Observe(kDetected, kNotPredicted); - // The detection above does not match the first prediction because it happened - // after the deadline of the 1st prediction. - EXPECT_EQ(evaluator.counters().false_positives, 1); - // However, the detection matches all the other predictions. - EXPECT_EQ(evaluator.counters().true_positives, 3); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -// Checks that multiple consecutive predictions match the first detection -// observed before the expected detection deadline expires. -TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndDetectedOnce) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); // --+ - evaluator.Observe(kNotDetected, kPredicted); // | --+ - evaluator.Observe(kNotDetected, kPredicted); // | | --+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ <-+ - EXPECT_EQ(evaluator.counters().true_positives, 3); - // The following observations do not generate any true negatives as they - // belong to the observation period of the last prediction - for which a - // detection has already been matched. - const int true_negatives = evaluator.counters().true_negatives; - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_negatives, true_negatives); - - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -// Checks that multiple consecutive predictions match the multiple detections -// observed before the expected detection deadline expires. -TEST(ClippingPredictionEvalTest, - PredictedMultipleTimesAndDetectedMultipleTimes) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); // --+ - evaluator.Observe(kNotDetected, kPredicted); // | --+ - evaluator.Observe(kNotDetected, kPredicted); // | | --+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ <-+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ - EXPECT_EQ(evaluator.counters().true_positives, 3); - // The following observation does not generate a true negative as it belongs - // to the observation period of the last prediction - for which two detections - // have already been matched. - const int true_negatives = evaluator.counters().true_negatives; - evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().true_negatives, true_negatives); - - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -// Checks that multiple consecutive predictions match all the detections -// observed before the expected detection deadline expires. -TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndAllDetected) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); // --+ - evaluator.Observe(kNotDetected, kPredicted); // | --+ - evaluator.Observe(kNotDetected, kPredicted); // | | --+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ <-+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ - EXPECT_EQ(evaluator.counters().true_positives, 3); - - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -// Checks that multiple non-consecutive predictions match all the detections -// observed before the expected detection deadline expires. -TEST(ClippingPredictionEvalTest, PredictedMultipleTimesWithGapAndAllDetected) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); // --+ - evaluator.Observe(kNotDetected, kNotPredicted); // | - evaluator.Observe(kNotDetected, kPredicted); // | --+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ - evaluator.Observe(kDetected, kNotPredicted); // <-+ - EXPECT_EQ(evaluator.counters().true_positives, 2); - - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -class ClippingPredictorEvaluatorPredictionIntervalParameterization - : public ::testing::TestWithParam> { - protected: - int num_extra_observe_calls() const { return std::get<0>(GetParam()); } - int history_size() const { return std::get<1>(GetParam()); } -}; - -// Checks that the minimum prediction interval is returned if clipping is -// correctly predicted just before clipping is detected - i.e., smallest -// anticipation. -TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, - MinimumPredictionInterval) { - ClippingPredictorEvaluator evaluator(history_size()); - for (int i = 0; i < num_extra_observe_calls(); ++i) { - EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt); - } - EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); - EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(1))); -} - -// Checks that a prediction interval between the minimum and the maximum is -// returned if clipping is correctly predicted before it is detected but not as -// early as possible. -TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, - IntermediatePredictionInterval) { - ClippingPredictorEvaluator evaluator(history_size()); - for (int i = 0; i < num_extra_observe_calls(); ++i) { - EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt); - } - EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); - EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); - EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); - EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(3))); -} - -// Checks that the maximum prediction interval is returned if clipping is -// correctly predicted as early as possible. -TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, - MaximumPredictionInterval) { - ClippingPredictorEvaluator evaluator(history_size()); - for (int i = 0; i < num_extra_observe_calls(); ++i) { - EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt); - } - for (int i = 0; i < history_size(); ++i) { - EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); - } - EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), - Optional(Eq(history_size()))); -} - -// Checks that `Observe()` returns the prediction interval as soon as a true -// positive is found and never again while ongoing detections are matched to a -// previously observed prediction. -TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, - PredictionIntervalReturnedOnce) { - ASSERT_LT(num_extra_observe_calls(), history_size()); - ClippingPredictorEvaluator evaluator(history_size()); - // Observe predictions before detection. - for (int i = 0; i < num_extra_observe_calls(); ++i) { - EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); - } - // Observe a detection. - absl::optional prediction_interval = - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_TRUE(prediction_interval.has_value()); - // `Observe()` does not return a prediction interval anymore during ongoing - // detections observed while a detection is still expected. - for (int i = 0; i < history_size(); ++i) { - EXPECT_EQ(evaluator.Observe(kDetected, kNotPredicted), absl::nullopt); - } -} - -INSTANTIATE_TEST_SUITE_P( - ClippingPredictionEvalTest, - ClippingPredictorEvaluatorPredictionIntervalParameterization, - ::testing::Combine(::testing::Values(1, 3, 5), ::testing::Values(7, 11))); - -// Checks that, when a detection is expected, the expectation is not removed -// before the detection deadline expires unless `RemoveExpectations()` is -// called. -TEST(ClippingPredictionEvalTest, NoFalsePositivesAfterRemoveExpectations) { - constexpr int kHistorySize = 2; - - // Case 1: `RemoveExpectations()` is NOT called. - ClippingPredictorEvaluator e1(kHistorySize); - e1.Observe(kNotDetected, kPredicted); - ASSERT_EQ(e1.counters().true_negatives, 1); - e1.Observe(kNotDetected, kNotPredicted); - e1.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(e1.counters().true_positives, 0); - EXPECT_EQ(e1.counters().true_negatives, 1); - EXPECT_EQ(e1.counters().false_positives, 1); - EXPECT_EQ(e1.counters().false_negatives, 0); - - // Case 2: `RemoveExpectations()` is called. - ClippingPredictorEvaluator e2(kHistorySize); - e2.Observe(kNotDetected, kPredicted); - ASSERT_EQ(e2.counters().true_negatives, 1); - e2.RemoveExpectations(); - e2.Observe(kNotDetected, kNotPredicted); - e2.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(e2.counters().true_positives, 0); - EXPECT_EQ(e2.counters().true_negatives, 3); - EXPECT_EQ(e2.counters().false_positives, 0); - EXPECT_EQ(e2.counters().false_negatives, 0); -} - -class ComputeClippingPredictionMetricsParameterization - : public ::testing::TestWithParam { - protected: - int true_negatives() const { return GetParam(); } -}; - -// Checks that `ComputeClippingPredictionMetrics()` does not return metrics if -// precision cannot be defined - i.e., TP + FP is zero. -TEST_P(ComputeClippingPredictionMetricsParameterization, - NoMetricsWithUndefinedPrecision) { - EXPECT_EQ(ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/0, - /*true_negatives=*/true_negatives(), - /*false_positives=*/0, - /*false_negatives=*/0}), - absl::nullopt); - EXPECT_EQ(ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/0, - /*true_negatives=*/true_negatives(), - /*false_positives=*/0, - /*false_negatives=*/1}), - absl::nullopt); -} - -// Checks that `ComputeClippingPredictionMetrics()` does not return metrics if -// recall cannot be defined - i.e., TP + FN is zero. -TEST_P(ComputeClippingPredictionMetricsParameterization, - NoMetricsWithUndefinedRecall) { - EXPECT_EQ(ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/0, - /*true_negatives=*/true_negatives(), - /*false_positives=*/0, - /*false_negatives=*/0}), - absl::nullopt); - EXPECT_EQ(ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/0, - /*true_negatives=*/true_negatives(), - /*false_positives=*/1, - /*false_negatives=*/0}), - absl::nullopt); -} - -// Checks that `ComputeClippingPredictionMetrics()` does not return metrics if -// the F1 score cannot be defined - i.e., P + R is zero. -TEST_P(ComputeClippingPredictionMetricsParameterization, - NoMetricsWithUndefinedF1Score) { - EXPECT_EQ(ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/0, - /*true_negatives=*/true_negatives(), - /*false_positives=*/1, - /*false_negatives=*/1}), - absl::nullopt); -} - -// Checks that the highest precision is reached when there are no false -// positives. -TEST_P(ComputeClippingPredictionMetricsParameterization, HighestPrecision) { - EXPECT_THAT(ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/1, - /*true_negatives=*/true_negatives(), - /*false_positives=*/0, - /*false_negatives=*/1}), - Optional(Field(&ClippingPredictionMetrics::precision, Eq(1.0f)))); -} - -// Checks that the highest recall is reached when there are no false -// negatives. -TEST_P(ComputeClippingPredictionMetricsParameterization, HighestRecall) { - EXPECT_THAT(ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/1, - /*true_negatives=*/true_negatives(), - /*false_positives=*/1, - /*false_negatives=*/0}), - Optional(Field(&ClippingPredictionMetrics::recall, Eq(1.0f)))); -} - -// Checks that 50% precision and 50% recall is reached when the number of true -// positives, false positives and false negatives are the same. -TEST_P(ComputeClippingPredictionMetricsParameterization, - PrecisionAndRecall50Percent) { - absl::optional metrics = - ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/42, - /*true_negatives=*/true_negatives(), - /*false_positives=*/42, - /*false_negatives=*/42}); - ASSERT_TRUE(metrics.has_value()); - EXPECT_EQ(metrics->precision, 0.5f); - EXPECT_EQ(metrics->recall, 0.5f); - EXPECT_EQ(metrics->f1_score, 0.5f); -} - -// Checks that the highest precision, recall and F1 score are jointly reached -// when there are no false positives and no false negatives. -TEST_P(ComputeClippingPredictionMetricsParameterization, - HighestPrecisionRecallF1Score) { - absl::optional metrics = - ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/123, - /*true_negatives=*/true_negatives(), - /*false_positives=*/0, - /*false_negatives=*/0}); - ASSERT_TRUE(metrics.has_value()); - EXPECT_EQ(metrics->precision, 1.0f); - EXPECT_EQ(metrics->recall, 1.0f); - EXPECT_EQ(metrics->f1_score, 1.0f); -} - -// Checks that precision is lower than recall when there are more false -// positives than false negatives. -TEST_P(ComputeClippingPredictionMetricsParameterization, - PrecisionLowerThanRecall) { - absl::optional metrics = - ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/1, - /*true_negatives=*/true_negatives(), - /*false_positives=*/8, - /*false_negatives=*/1}); - ASSERT_TRUE(metrics.has_value()); - EXPECT_LT(metrics->precision, metrics->recall); -} - -// Checks that precision is greater than recall when there are less false -// positives than false negatives. -TEST_P(ComputeClippingPredictionMetricsParameterization, - PrecisionGreaterThanRecall) { - absl::optional metrics = - ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/1, - /*true_negatives=*/true_negatives(), - /*false_positives=*/1, - /*false_negatives=*/8}); - ASSERT_TRUE(metrics.has_value()); - EXPECT_GT(metrics->precision, metrics->recall); -} - -// Checks that swapping precision and recall does not change the F1 score. -TEST_P(ComputeClippingPredictionMetricsParameterization, SameF1Score) { - absl::optional m1 = - ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/1, - /*true_negatives=*/true_negatives(), - /*false_positives=*/8, - /*false_negatives=*/1}); - absl::optional m2 = - ComputeClippingPredictionMetrics( - /*counters=*/{/*true_positives=*/1, - /*true_negatives=*/true_negatives(), - /*false_positives=*/1, - /*false_negatives=*/8}); - // Preconditions. - ASSERT_TRUE(m1.has_value()); - ASSERT_TRUE(m2.has_value()); - ASSERT_EQ(m1->precision, m2->recall); - ASSERT_EQ(m1->recall, m2->precision); - // Same F1 score. - EXPECT_EQ(m1->f1_score, m2->f1_score); -} - -INSTANTIATE_TEST_SUITE_P(ClippingPredictionEvalTest, - ComputeClippingPredictionMetricsParameterization, - ::testing::Values(0, 1, 11)); - -} // namespace -} // namespace webrtc