From f6c6b4a831e8a495d3ec656f60f4aab708a46876 Mon Sep 17 00:00:00 2001 From: Alessio Bazzica Date: Tue, 7 Sep 2021 16:23:09 +0200 Subject: [PATCH] ClippingPredictorEvaluator: predictions only match future detections To focus on the ability to predict clipping, the clipping predictor evaluator doesn't increment the true positive count anymore when a prediction is simultaneously observed with a detection. Note that `WebRTC.Audio.Agc.ClippingPredictor.F1Score` is still used to log the F1 score - i.e., the histogram hasn't been renamed. Bug: webrtc:12774 Change-Id: Ia987e568a6df2a3ddba7fa1b5697d6feda22d20c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/231233 Commit-Queue: Alessio Bazzica Reviewed-by: Hanna Silen Cr-Commit-Position: refs/heads/main@{#34942} --- .../agc/agc_manager_direct.cc | 2 +- .../agc/clipping_predictor_evaluator.cc | 20 +- .../agc/clipping_predictor_evaluator.h | 3 + .../clipping_predictor_evaluator_unittest.cc | 362 ++++++++++-------- 4 files changed, 215 insertions(+), 172 deletions(-) diff --git a/modules/audio_processing/agc/agc_manager_direct.cc b/modules/audio_processing/agc/agc_manager_direct.cc index 22a097e81a..bda1cae327 100644 --- a/modules/audio_processing/agc/agc_manager_direct.cc +++ b/modules/audio_processing/agc/agc_manager_direct.cc @@ -629,7 +629,7 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, frames_since_clipped_ = 0; if (!!clipping_predictor_) { clipping_predictor_->Reset(); - clipping_predictor_evaluator_.Reset(); + clipping_predictor_evaluator_.RemoveExpectations(); } } AggregateChannelLevels(); diff --git a/modules/audio_processing/agc/clipping_predictor_evaluator.cc b/modules/audio_processing/agc/clipping_predictor_evaluator.cc index f211d6e8e3..ed7198d119 100644 --- a/modules/audio_processing/agc/clipping_predictor_evaluator.cc +++ b/modules/audio_processing/agc/clipping_predictor_evaluator.cc @@ -50,10 +50,6 @@ absl::optional ClippingPredictorEvaluator::Observe( RTC_DCHECK_LT(ring_buffer_tail_, ring_buffer_capacity_); DecreaseTimesToLive(); - if (clipping_predicted) { - // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. - Push(/*expected_detection=*/{/*ttl=*/history_size_, /*detected=*/false}); - } // Clipping is expected if there are expected detections regardless of // whether all the expected detections have been previously matched - i.e., // `ExpectedDetection::detected` is true. @@ -80,15 +76,29 @@ absl::optional ClippingPredictorEvaluator::Observe( RTC_DCHECK(!clipping_expected && !clipping_detected); counters_.true_negatives++; } + + if (clipping_predicted) { + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + Push(/*expected_detection=*/{/*ttl=*/history_size_, /*detected=*/false}); + } + return prediction_interval; } -void ClippingPredictorEvaluator::Reset() { +void ClippingPredictorEvaluator::RemoveExpectations() { // Empty the ring buffer of expected detections. ring_buffer_tail_ = 0; ring_buffer_size_ = 0; } +void ClippingPredictorEvaluator::Reset() { + counters_.true_positives = 0; + counters_.true_negatives = 0; + counters_.false_positives = 0; + counters_.false_negatives = 0; + RemoveExpectations(); +} + // Cost: O(1). void ClippingPredictorEvaluator::Push(ExpectedDetection value) { ring_buffer_[ring_buffer_tail_] = value; diff --git a/modules/audio_processing/agc/clipping_predictor_evaluator.h b/modules/audio_processing/agc/clipping_predictor_evaluator.h index 3a4ebd5095..348f753493 100644 --- a/modules/audio_processing/agc/clipping_predictor_evaluator.h +++ b/modules/audio_processing/agc/clipping_predictor_evaluator.h @@ -57,6 +57,9 @@ class ClippingPredictorEvaluator { // Removes any expectation recently set after a call to `Observe()` having // `clipping_predicted` set to true. Counters won't be reset. + void RemoveExpectations(); + + // Resets counters and removes any expectation (see `RemoveExpectations()`). void Reset(); ClippingPredictionCounters counters() const { return counters_; } diff --git a/modules/audio_processing/agc/clipping_predictor_evaluator_unittest.cc b/modules/audio_processing/agc/clipping_predictor_evaluator_unittest.cc index caebe8f99e..b2d2797ca5 100644 --- a/modules/audio_processing/agc/clipping_predictor_evaluator_unittest.cc +++ b/modules/audio_processing/agc/clipping_predictor_evaluator_unittest.cc @@ -34,12 +34,14 @@ constexpr bool kNotDetected = false; constexpr bool kPredicted = true; constexpr bool kNotPredicted = false; -int SumTrueFalsePositivesNegatives( - const ClippingPredictorEvaluator& evaluator) { - return evaluator.counters().true_positives + - evaluator.counters().true_negatives + - evaluator.counters().false_positives + - evaluator.counters().false_negatives; +ClippingPredictionCounters operator-(const ClippingPredictionCounters& lhs, + const ClippingPredictionCounters& rhs) { + return { + lhs.true_positives - rhs.true_positives, + lhs.true_negatives - rhs.true_negatives, + lhs.false_positives - rhs.false_positives, + lhs.false_negatives - rhs.false_negatives, + }; } // Checks the metrics after init - i.e., no call to `Observe()`. @@ -69,21 +71,19 @@ TEST_P(ClippingPredictorEvaluatorParameterization, AtMostOneMetricChanges) { for (int i = 0; i < kNumCalls; ++i) { SCOPED_TRACE(i); // Read metrics before `Observe()` is called. - const int last_tp = evaluator.counters().true_positives; - const int last_tn = evaluator.counters().true_negatives; - const int last_fp = evaluator.counters().false_positives; - const int last_fn = evaluator.counters().false_negatives; + const auto pre = evaluator.counters(); // `Observe()` a random observation. bool clipping_detected = random_generator.Rand(); bool clipping_predicted = random_generator.Rand(); evaluator.Observe(clipping_detected, clipping_predicted); // Check that at most one metric has changed. + const auto post = evaluator.counters(); int num_changes = 0; - num_changes += last_tp == evaluator.counters().true_positives ? 0 : 1; - num_changes += last_tn == evaluator.counters().true_negatives ? 0 : 1; - num_changes += last_fp == evaluator.counters().false_positives ? 0 : 1; - num_changes += last_fn == evaluator.counters().false_negatives ? 0 : 1; + num_changes += pre.true_positives == post.true_positives ? 0 : 1; + num_changes += pre.true_negatives == post.true_negatives ? 0 : 1; + num_changes += pre.false_positives == post.false_positives ? 0 : 1; + num_changes += pre.false_negatives == post.false_negatives ? 0 : 1; EXPECT_GE(num_changes, 0); EXPECT_LE(num_changes, 1); } @@ -99,20 +99,18 @@ TEST_P(ClippingPredictorEvaluatorParameterization, MetricsAreWeaklyMonotonic) { for (int i = 0; i < kNumCalls; ++i) { SCOPED_TRACE(i); // Read metrics before `Observe()` is called. - const int last_tp = evaluator.counters().true_positives; - const int last_tn = evaluator.counters().true_negatives; - const int last_fp = evaluator.counters().false_positives; - const int last_fn = evaluator.counters().false_negatives; + const auto pre = evaluator.counters(); // `Observe()` a random observation. bool clipping_detected = random_generator.Rand(); bool clipping_predicted = random_generator.Rand(); evaluator.Observe(clipping_detected, clipping_predicted); // Check that metrics are weakly monotonic. - EXPECT_GE(evaluator.counters().true_positives, last_tp); - EXPECT_GE(evaluator.counters().true_negatives, last_tn); - EXPECT_GE(evaluator.counters().false_positives, last_fp); - EXPECT_GE(evaluator.counters().false_negatives, last_fn); + const auto post = evaluator.counters(); + EXPECT_GE(post.true_positives, pre.true_positives); + EXPECT_GE(post.true_negatives, pre.true_negatives); + EXPECT_GE(post.false_positives, pre.false_positives); + EXPECT_GE(post.false_negatives, pre.false_negatives); } } @@ -126,23 +124,20 @@ TEST_P(ClippingPredictorEvaluatorParameterization, BoundedMetricsGrowth) { for (int i = 0; i < kNumCalls; ++i) { SCOPED_TRACE(i); // Read metrics before `Observe()` is called. - const int last_tp = evaluator.counters().true_positives; - const int last_tn = evaluator.counters().true_negatives; - const int last_fp = evaluator.counters().false_positives; - const int last_fn = evaluator.counters().false_negatives; + const auto pre = evaluator.counters(); // `Observe()` a random observation. bool clipping_detected = random_generator.Rand(); bool clipping_predicted = random_generator.Rand(); evaluator.Observe(clipping_detected, clipping_predicted); + const auto diff = evaluator.counters() - pre; // Check that TPs grow by at most `history_size() + 1`. Such an upper bound // is reached when multiple predictions are matched by a single detection. - EXPECT_LE(evaluator.counters().true_positives - last_tp, - history_size() + 1); - // Check that TNs, FPs and FNs grow by at most one. `max_growth`. - EXPECT_LE(evaluator.counters().true_negatives - last_tn, 1); - EXPECT_LE(evaluator.counters().false_positives - last_fp, 1); - EXPECT_LE(evaluator.counters().false_negatives - last_fn, 1); + EXPECT_LE(diff.true_positives, history_size() + 1); + // Check that TNs, FPs and FNs grow by at most one. + EXPECT_LE(diff.true_negatives, 1); + EXPECT_LE(diff.false_positives, 1); + EXPECT_LE(diff.false_negatives, 1); } } @@ -180,90 +175,144 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine(::testing::Values(4, 8, 15, 16, 23, 42), ::testing::Values(1, 10, 21))); -// Checks that, observing a detection and a prediction after init, produces a -// true positive. -TEST(ClippingPredictionEvalTest, OneTruePositiveAfterInit) { +// Checks that after initialization, when no detection is expected, +// observing no detection and no prediction produces a true negative. +TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectNoPredictAfterInit) { ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kDetected, kPredicted); - EXPECT_EQ(evaluator.counters().true_positives, 1); - - EXPECT_EQ(evaluator.counters().true_negatives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -// Checks that, observing a detection but no prediction after init, produces a -// false negative. -TEST(ClippingPredictionEvalTest, OneFalseNegativeAfterInit) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_negatives, 1); + evaluator.Observe(kNotDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 0); -} - -// Checks that, observing no detection but a prediction after init, produces a -// false positive after expiration. -TEST(ClippingPredictionEvalTest, OneFalsePositiveAfterInit) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 1); - - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); -} - -// Checks that, observing no detection and no prediction after init, produces a -// true negative. -TEST(ClippingPredictionEvalTest, OneTrueNegativeAfterInit) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_negatives, 1); - - EXPECT_EQ(evaluator.counters().true_positives, 0); EXPECT_EQ(evaluator.counters().false_positives, 0); EXPECT_EQ(evaluator.counters().false_negatives, 0); } +// Checks that after initialization, when no detection is expected, +// observing no detection and prediction produces a true negative. +TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectPredictAfterInit) { + ClippingPredictorEvaluator evaluator(/*history_size=*/3); + + evaluator.Observe(kNotDetected, kPredicted); + EXPECT_EQ(evaluator.counters().true_positives, 0); + EXPECT_EQ(evaluator.counters().true_negatives, 1); + EXPECT_EQ(evaluator.counters().false_positives, 0); + EXPECT_EQ(evaluator.counters().false_negatives, 0); +} + +// Checks that after initialization, when no detection is expected, +// observing a detection and no prediction produces a false negative. +TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectNoPredictAfterInit) { + ClippingPredictorEvaluator evaluator(/*history_size=*/3); + + evaluator.Observe(kDetected, kNotPredicted); + EXPECT_EQ(evaluator.counters().true_positives, 0); + EXPECT_EQ(evaluator.counters().true_negatives, 0); + EXPECT_EQ(evaluator.counters().false_positives, 0); + EXPECT_EQ(evaluator.counters().false_negatives, 1); +} + +// Checks that after initialization, when no detection is expected, +// simultaneously observing a detection and a prediction produces a false +// negative. +TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectPredictAfterInit) { + ClippingPredictorEvaluator evaluator(/*history_size=*/3); + + evaluator.Observe(kDetected, kPredicted); + EXPECT_EQ(evaluator.counters().true_positives, 0); + EXPECT_EQ(evaluator.counters().true_negatives, 0); + EXPECT_EQ(evaluator.counters().false_positives, 0); + EXPECT_EQ(evaluator.counters().false_negatives, 1); +} + +// Checks that, after removing existing expectations, observing no detection and +// no prediction produces a true negative. +TEST(ClippingPredictionEvalTest, + TrueNegativeWithNoDetectNoPredictAfterRemoveExpectations) { + ClippingPredictorEvaluator evaluator(/*history_size=*/3); + + // Set an expectation, then remove it. + evaluator.Observe(kNotDetected, kPredicted); + evaluator.RemoveExpectations(); + const auto pre = evaluator.counters(); + + evaluator.Observe(kNotDetected, kNotPredicted); + const auto diff = evaluator.counters() - pre; + EXPECT_EQ(diff.true_positives, 0); + EXPECT_EQ(diff.true_negatives, 1); + EXPECT_EQ(diff.false_positives, 0); + EXPECT_EQ(diff.false_negatives, 0); +} + +// Checks that, after removing existing expectations, observing no detection and +// a prediction produces a true negative. +TEST(ClippingPredictionEvalTest, + TrueNegativeWithNoDetectPredictAfterRemoveExpectations) { + ClippingPredictorEvaluator evaluator(/*history_size=*/3); + + // Set an expectation, then remove it. + evaluator.Observe(kNotDetected, kPredicted); + evaluator.RemoveExpectations(); + const auto pre = evaluator.counters(); + + evaluator.Observe(kNotDetected, kPredicted); + const auto diff = evaluator.counters() - pre; + EXPECT_EQ(diff.true_positives, 0); + EXPECT_EQ(diff.true_negatives, 1); + EXPECT_EQ(diff.false_positives, 0); + EXPECT_EQ(diff.false_negatives, 0); +} + +// Checks that, after removing existing expectations, observing a detection and +// no prediction produces a false negative. +TEST(ClippingPredictionEvalTest, + FalseNegativeWithDetectNoPredictAfterRemoveExpectations) { + ClippingPredictorEvaluator evaluator(/*history_size=*/3); + + // Set an expectation, then remove it. + evaluator.Observe(kNotDetected, kPredicted); + evaluator.RemoveExpectations(); + const auto pre = evaluator.counters(); + + evaluator.Observe(kDetected, kNotPredicted); + const auto diff = evaluator.counters() - pre; + EXPECT_EQ(diff.true_positives, 0); + EXPECT_EQ(diff.true_negatives, 0); + EXPECT_EQ(diff.false_positives, 0); + EXPECT_EQ(diff.false_negatives, 1); +} + +// Checks that, after removing existing expectations, simultaneously observing a +// detection and a prediction produces a false negative. +TEST(ClippingPredictionEvalTest, + FalseNegativeWithDetectPredictAfterRemoveExpectations) { + ClippingPredictorEvaluator evaluator(/*history_size=*/3); + + // Set an expectation, then remove it. + evaluator.Observe(kNotDetected, kPredicted); + evaluator.RemoveExpectations(); + const auto pre = evaluator.counters(); + + evaluator.Observe(kDetected, kPredicted); + const auto diff = evaluator.counters() - pre; + EXPECT_EQ(diff.false_negatives, 1); + EXPECT_EQ(diff.true_positives, 0); + EXPECT_EQ(diff.true_negatives, 0); + EXPECT_EQ(diff.false_positives, 0); +} + // Checks that the evaluator detects true negatives when clipping is neither // predicted nor detected. -TEST(ClippingPredictionEvalTest, NeverDetectedAndNotPredicted) { +TEST(ClippingPredictionEvalTest, TrueNegativesWhenNeverDetectedOrPredicted) { ClippingPredictorEvaluator evaluator(/*history_size=*/3); evaluator.Observe(kNotDetected, kNotPredicted); evaluator.Observe(kNotDetected, kNotPredicted); evaluator.Observe(kNotDetected, kNotPredicted); evaluator.Observe(kNotDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_negatives, 4); - - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); } -// Checks that the evaluator detects a false negative when clipping is detected -// but not predicted. -TEST(ClippingPredictionEvalTest, DetectedButNotPredicted) { - ClippingPredictorEvaluator evaluator(/*history_size=*/3); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kNotDetected, kNotPredicted); - evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_negatives, 1); - - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 3); - EXPECT_EQ(evaluator.counters().false_positives, 0); -} - -// Checks that the evaluator does not detect a false positive when clipping is -// predicted but not detected until the observation period expires. +// Checks that, until the observation period expires, the evaluator does not +// count a false positive when clipping is predicted and not detected. TEST(ClippingPredictionEvalTest, PredictedOnceAndNeverDetectedBeforeDeadline) { ClippingPredictorEvaluator evaluator(/*history_size=*/3); evaluator.Observe(kNotDetected, kPredicted); @@ -272,15 +321,12 @@ TEST(ClippingPredictionEvalTest, PredictedOnceAndNeverDetectedBeforeDeadline) { evaluator.Observe(kNotDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().false_positives, 0); evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 1); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); + EXPECT_EQ(evaluator.counters().false_positives, 1); } -// Checks that the evaluator detects a false positive when clipping is predicted -// but detected after the observation period expires. +// Checks that, after the observation period expires, the evaluator detects a +// false positive when clipping is predicted and detected. TEST(ClippingPredictionEvalTest, PredictedOnceButDetectedAfterDeadline) { ClippingPredictorEvaluator evaluator(/*history_size=*/3); evaluator.Observe(kNotDetected, kPredicted); @@ -288,24 +334,17 @@ TEST(ClippingPredictionEvalTest, PredictedOnceButDetectedAfterDeadline) { evaluator.Observe(kNotDetected, kNotPredicted); evaluator.Observe(kNotDetected, kNotPredicted); evaluator.Observe(kDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 1); - EXPECT_EQ(evaluator.counters().true_positives, 0); - EXPECT_EQ(evaluator.counters().true_negatives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 1); + EXPECT_EQ(evaluator.counters().false_positives, 1); } // Checks that a prediction followed by a detection counts as true positive. TEST(ClippingPredictionEvalTest, PredictedOnceAndThenImmediatelyDetected) { ClippingPredictorEvaluator evaluator(/*history_size=*/3); evaluator.Observe(kNotDetected, kPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); evaluator.Observe(kDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_positives, 1); - - EXPECT_EQ(evaluator.counters().true_negatives, 0); EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); } // Checks that a prediction followed by a delayed detection counts as true @@ -313,15 +352,10 @@ TEST(ClippingPredictionEvalTest, PredictedOnceAndThenImmediatelyDetected) { TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedBeforeDeadline) { ClippingPredictorEvaluator evaluator(/*history_size=*/3); evaluator.Observe(kNotDetected, kPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); evaluator.Observe(kDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_positives, 1); - - EXPECT_EQ(evaluator.counters().true_negatives, 0); EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); } // Checks that a prediction followed by a delayed detection counts as true @@ -329,17 +363,11 @@ TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedBeforeDeadline) { TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedAtDeadline) { ClippingPredictorEvaluator evaluator(/*history_size=*/3); evaluator.Observe(kNotDetected, kPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); evaluator.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(evaluator.counters().false_positives, 0); evaluator.Observe(kDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_positives, 1); - - EXPECT_EQ(evaluator.counters().true_negatives, 0); EXPECT_EQ(evaluator.counters().false_positives, 0); - EXPECT_EQ(evaluator.counters().false_negatives, 0); } // Checks that a prediction followed by a multiple adjacent detections within @@ -352,19 +380,22 @@ TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedMultipleTimes) { // Multiple detections. evaluator.Observe(kDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_positives, 1); + EXPECT_EQ(evaluator.counters().false_negatives, 0); + EXPECT_EQ(evaluator.counters().false_positives, 0); evaluator.Observe(kDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_positives, 1); + EXPECT_EQ(evaluator.counters().false_negatives, 0); + EXPECT_EQ(evaluator.counters().false_positives, 0); // A detection outside of the observation period counts as false negative. evaluator.Observe(kDetected, kNotPredicted); + EXPECT_EQ(evaluator.counters().true_positives, 1); EXPECT_EQ(evaluator.counters().false_negatives, 1); - EXPECT_EQ(SumTrueFalsePositivesNegatives(evaluator), 2); - - EXPECT_EQ(evaluator.counters().true_negatives, 0); EXPECT_EQ(evaluator.counters().false_positives, 0); } -// Checks that a false positive is added when clipping is detected after a too -// early prediction. +// Checks that when clipping is predicted multiple times, a prediction that is +// observed too early counts as a false positive, whereas the other predictions +// that are matched to a detection count as true positives. TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndDetectedOnceAfterDeadline) { ClippingPredictorEvaluator evaluator(/*history_size=*/3); @@ -378,9 +409,8 @@ TEST(ClippingPredictionEvalTest, // The detection above does not match the first prediction because it happened // after the deadline of the 1st prediction. EXPECT_EQ(evaluator.counters().false_positives, 1); - + // However, the detection matches all the other predictions. EXPECT_EQ(evaluator.counters().true_positives, 3); - EXPECT_EQ(evaluator.counters().true_negatives, 0); EXPECT_EQ(evaluator.counters().false_negatives, 0); } @@ -401,7 +431,6 @@ TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndDetectedOnce) { evaluator.Observe(kNotDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_negatives, true_negatives); - EXPECT_EQ(evaluator.counters().true_negatives, 0); EXPECT_EQ(evaluator.counters().false_positives, 0); EXPECT_EQ(evaluator.counters().false_negatives, 0); } @@ -424,7 +453,6 @@ TEST(ClippingPredictionEvalTest, evaluator.Observe(kNotDetected, kNotPredicted); EXPECT_EQ(evaluator.counters().true_negatives, true_negatives); - EXPECT_EQ(evaluator.counters().true_negatives, 0); EXPECT_EQ(evaluator.counters().false_positives, 0); EXPECT_EQ(evaluator.counters().false_negatives, 0); } @@ -440,7 +468,7 @@ TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndAllDetected) { evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ evaluator.Observe(kDetected, kNotPredicted); // <-+ EXPECT_EQ(evaluator.counters().true_positives, 3); - EXPECT_EQ(evaluator.counters().true_negatives, 0); + EXPECT_EQ(evaluator.counters().false_positives, 0); EXPECT_EQ(evaluator.counters().false_negatives, 0); } @@ -456,7 +484,7 @@ TEST(ClippingPredictionEvalTest, PredictedMultipleTimesWithGapAndAllDetected) { evaluator.Observe(kDetected, kNotPredicted); // <-+ evaluator.Observe(kDetected, kNotPredicted); // <-+ EXPECT_EQ(evaluator.counters().true_positives, 2); - EXPECT_EQ(evaluator.counters().true_negatives, 0); + EXPECT_EQ(evaluator.counters().false_positives, 0); EXPECT_EQ(evaluator.counters().false_negatives, 0); } @@ -469,16 +497,16 @@ class ClippingPredictorEvaluatorPredictionIntervalParameterization }; // Checks that the minimum prediction interval is returned if clipping is -// correctly predicted as soon as detected - i.e., no anticipation. +// correctly predicted just before clipping is detected - i.e., smallest +// anticipation. TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, MinimumPredictionInterval) { ClippingPredictorEvaluator evaluator(history_size()); for (int i = 0; i < num_extra_observe_calls(); ++i) { EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt); } - absl::optional prediction_interval = - evaluator.Observe(kDetected, kPredicted); - EXPECT_THAT(prediction_interval, Optional(Eq(0))); + EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); + EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(1))); } // Checks that a prediction interval between the minimum and the maximum is @@ -493,9 +521,7 @@ TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); - absl::optional prediction_interval = - evaluator.Observe(kDetected, kPredicted); - EXPECT_THAT(prediction_interval, Optional(Eq(3))); + EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(3))); } // Checks that the maximum prediction interval is returned if clipping is @@ -509,9 +535,8 @@ TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, for (int i = 0; i < history_size(); ++i) { EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt); } - absl::optional prediction_interval = - evaluator.Observe(kDetected, kPredicted); - EXPECT_THAT(prediction_interval, Optional(Eq(history_size()))); + EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), + Optional(Eq(history_size()))); } // Checks that `Observe()` returns the prediction interval as soon as a true @@ -527,7 +552,7 @@ TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, } // Observe a detection. absl::optional prediction_interval = - evaluator.Observe(kDetected, kPredicted); + evaluator.Observe(kDetected, kNotPredicted); EXPECT_TRUE(prediction_interval.has_value()); // `Observe()` does not return a prediction interval anymore during ongoing // detections observed while a detection is still expected. @@ -539,31 +564,36 @@ TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization, INSTANTIATE_TEST_SUITE_P( ClippingPredictionEvalTest, ClippingPredictorEvaluatorPredictionIntervalParameterization, - ::testing::Combine(::testing::Values(0, 3, 5), ::testing::Values(7, 11))); + ::testing::Combine(::testing::Values(1, 3, 5), ::testing::Values(7, 11))); -// Checks that, when a detection is expected, the expectation is removed if and -// only if `Reset()` is called after a prediction is observed. -TEST(ClippingPredictionEvalTest, NoFalsePositivesAfterReset) { +// Checks that, when a detection is expected, the expectation is not removed +// before the detection deadline expires unless `RemoveExpectations()` is +// called. +TEST(ClippingPredictionEvalTest, NoFalsePositivesAfterRemoveExpectations) { constexpr int kHistorySize = 2; - ClippingPredictorEvaluator with_reset(kHistorySize); - with_reset.Observe(kNotDetected, kPredicted); - with_reset.Reset(); - with_reset.Observe(kNotDetected, kNotPredicted); - with_reset.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(with_reset.counters().true_positives, 0); - EXPECT_EQ(with_reset.counters().true_negatives, 2); - EXPECT_EQ(with_reset.counters().false_positives, 0); - EXPECT_EQ(with_reset.counters().false_negatives, 0); + // Case 1: `RemoveExpectations()` is NOT called. + ClippingPredictorEvaluator e1(kHistorySize); + e1.Observe(kNotDetected, kPredicted); + ASSERT_EQ(e1.counters().true_negatives, 1); + e1.Observe(kNotDetected, kNotPredicted); + e1.Observe(kNotDetected, kNotPredicted); + EXPECT_EQ(e1.counters().true_positives, 0); + EXPECT_EQ(e1.counters().true_negatives, 1); + EXPECT_EQ(e1.counters().false_positives, 1); + EXPECT_EQ(e1.counters().false_negatives, 0); - ClippingPredictorEvaluator no_reset(kHistorySize); - no_reset.Observe(kNotDetected, kPredicted); - no_reset.Observe(kNotDetected, kNotPredicted); - no_reset.Observe(kNotDetected, kNotPredicted); - EXPECT_EQ(no_reset.counters().true_positives, 0); - EXPECT_EQ(no_reset.counters().true_negatives, 0); - EXPECT_EQ(no_reset.counters().false_positives, 1); - EXPECT_EQ(no_reset.counters().false_negatives, 0); + // Case 2: `RemoveExpectations()` is called. + ClippingPredictorEvaluator e2(kHistorySize); + e2.Observe(kNotDetected, kPredicted); + ASSERT_EQ(e2.counters().true_negatives, 1); + e2.RemoveExpectations(); + e2.Observe(kNotDetected, kNotPredicted); + e2.Observe(kNotDetected, kNotPredicted); + EXPECT_EQ(e2.counters().true_positives, 0); + EXPECT_EQ(e2.counters().true_negatives, 3); + EXPECT_EQ(e2.counters().false_positives, 0); + EXPECT_EQ(e2.counters().false_negatives, 0); } class ComputeClippingPredictionMetricsParameterization