ClippingPredictorEvaluator: predictions only match future detections
To focus on the ability to predict clipping, the clipping predictor evaluator doesn't increment the true positive count anymore when a prediction is simultaneously observed with a detection. Note that `WebRTC.Audio.Agc.ClippingPredictor.F1Score` is still used to log the F1 score - i.e., the histogram hasn't been renamed. Bug: webrtc:12774 Change-Id: Ia987e568a6df2a3ddba7fa1b5697d6feda22d20c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/231233 Commit-Queue: Alessio Bazzica <alessiob@webrtc.org> Reviewed-by: Hanna Silen <silen@webrtc.org> Cr-Commit-Position: refs/heads/main@{#34942}
This commit is contained in:
Alessio Bazzica
WebRTC LUCI CQ
parent
bc93575749
commit
f6c6b4a831
@ -629,7 +629,7 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio,
|
||||
frames_since_clipped_ = 0;
|
||||
if (!!clipping_predictor_) {
|
||||
clipping_predictor_->Reset();
|
||||
clipping_predictor_evaluator_.Reset();
|
||||
clipping_predictor_evaluator_.RemoveExpectations();
|
||||
}
|
||||
}
|
||||
AggregateChannelLevels();
|
||||
|
||||
@ -50,10 +50,6 @@ absl::optional<int> ClippingPredictorEvaluator::Observe(
|
||||
RTC_DCHECK_LT(ring_buffer_tail_, ring_buffer_capacity_);
|
||||
|
||||
DecreaseTimesToLive();
|
||||
if (clipping_predicted) {
|
||||
// TODO(bugs.webrtc.org/12874): Use designated initializers one fixed.
|
||||
Push(/*expected_detection=*/{/*ttl=*/history_size_, /*detected=*/false});
|
||||
}
|
||||
// Clipping is expected if there are expected detections regardless of
|
||||
// whether all the expected detections have been previously matched - i.e.,
|
||||
// `ExpectedDetection::detected` is true.
|
||||
@ -80,15 +76,29 @@ absl::optional<int> ClippingPredictorEvaluator::Observe(
|
||||
RTC_DCHECK(!clipping_expected && !clipping_detected);
|
||||
counters_.true_negatives++;
|
||||
}
|
||||
|
||||
if (clipping_predicted) {
|
||||
// TODO(bugs.webrtc.org/12874): Use designated initializers one fixed.
|
||||
Push(/*expected_detection=*/{/*ttl=*/history_size_, /*detected=*/false});
|
||||
}
|
||||
|
||||
return prediction_interval;
|
||||
}
|
||||
|
||||
void ClippingPredictorEvaluator::Reset() {
|
||||
void ClippingPredictorEvaluator::RemoveExpectations() {
|
||||
// Empty the ring buffer of expected detections.
|
||||
ring_buffer_tail_ = 0;
|
||||
ring_buffer_size_ = 0;
|
||||
}
|
||||
|
||||
void ClippingPredictorEvaluator::Reset() {
|
||||
counters_.true_positives = 0;
|
||||
counters_.true_negatives = 0;
|
||||
counters_.false_positives = 0;
|
||||
counters_.false_negatives = 0;
|
||||
RemoveExpectations();
|
||||
}
|
||||
|
||||
// Cost: O(1).
|
||||
void ClippingPredictorEvaluator::Push(ExpectedDetection value) {
|
||||
ring_buffer_[ring_buffer_tail_] = value;
|
||||
|
||||
@ -57,6 +57,9 @@ class ClippingPredictorEvaluator {
|
||||
|
||||
// Removes any expectation recently set after a call to `Observe()` having
|
||||
// `clipping_predicted` set to true. Counters won't be reset.
|
||||
void RemoveExpectations();
|
||||
|
||||
// Resets counters and removes any expectation (see `RemoveExpectations()`).
|
||||
void Reset();
|
||||
|
||||
ClippingPredictionCounters counters() const { return counters_; }
|
||||
|
||||
@ -34,12 +34,14 @@ constexpr bool kNotDetected = false;
|
||||
constexpr bool kPredicted = true;
|
||||
constexpr bool kNotPredicted = false;
|
||||
|
||||
int SumTrueFalsePositivesNegatives(
|
||||
const ClippingPredictorEvaluator& evaluator) {
|
||||
return evaluator.counters().true_positives +
|
||||
evaluator.counters().true_negatives +
|
||||
evaluator.counters().false_positives +
|
||||
evaluator.counters().false_negatives;
|
||||
ClippingPredictionCounters operator-(const ClippingPredictionCounters& lhs,
|
||||
const ClippingPredictionCounters& rhs) {
|
||||
return {
|
||||
lhs.true_positives - rhs.true_positives,
|
||||
lhs.true_negatives - rhs.true_negatives,
|
||||
lhs.false_positives - rhs.false_positives,
|
||||
lhs.false_negatives - rhs.false_negatives,
|
||||
};
|
||||
}
|
||||
|
||||
// Checks the metrics after init - i.e., no call to `Observe()`.
|
||||
@ -69,21 +71,19 @@ TEST_P(ClippingPredictorEvaluatorParameterization, AtMostOneMetricChanges) {
|
||||
for (int i = 0; i < kNumCalls; ++i) {
|
||||
SCOPED_TRACE(i);
|
||||
// Read metrics before `Observe()` is called.
|
||||
const int last_tp = evaluator.counters().true_positives;
|
||||
const int last_tn = evaluator.counters().true_negatives;
|
||||
const int last_fp = evaluator.counters().false_positives;
|
||||
const int last_fn = evaluator.counters().false_negatives;
|
||||
const auto pre = evaluator.counters();
|
||||
// `Observe()` a random observation.
|
||||
bool clipping_detected = random_generator.Rand<bool>();
|
||||
bool clipping_predicted = random_generator.Rand<bool>();
|
||||
evaluator.Observe(clipping_detected, clipping_predicted);
|
||||
|
||||
// Check that at most one metric has changed.
|
||||
const auto post = evaluator.counters();
|
||||
int num_changes = 0;
|
||||
num_changes += last_tp == evaluator.counters().true_positives ? 0 : 1;
|
||||
num_changes += last_tn == evaluator.counters().true_negatives ? 0 : 1;
|
||||
num_changes += last_fp == evaluator.counters().false_positives ? 0 : 1;
|
||||
num_changes += last_fn == evaluator.counters().false_negatives ? 0 : 1;
|
||||
num_changes += pre.true_positives == post.true_positives ? 0 : 1;
|
||||
num_changes += pre.true_negatives == post.true_negatives ? 0 : 1;
|
||||
num_changes += pre.false_positives == post.false_positives ? 0 : 1;
|
||||
num_changes += pre.false_negatives == post.false_negatives ? 0 : 1;
|
||||
EXPECT_GE(num_changes, 0);
|
||||
EXPECT_LE(num_changes, 1);
|
||||
}
|
||||
@ -99,20 +99,18 @@ TEST_P(ClippingPredictorEvaluatorParameterization, MetricsAreWeaklyMonotonic) {
|
||||
for (int i = 0; i < kNumCalls; ++i) {
|
||||
SCOPED_TRACE(i);
|
||||
// Read metrics before `Observe()` is called.
|
||||
const int last_tp = evaluator.counters().true_positives;
|
||||
const int last_tn = evaluator.counters().true_negatives;
|
||||
const int last_fp = evaluator.counters().false_positives;
|
||||
const int last_fn = evaluator.counters().false_negatives;
|
||||
const auto pre = evaluator.counters();
|
||||
// `Observe()` a random observation.
|
||||
bool clipping_detected = random_generator.Rand<bool>();
|
||||
bool clipping_predicted = random_generator.Rand<bool>();
|
||||
evaluator.Observe(clipping_detected, clipping_predicted);
|
||||
|
||||
// Check that metrics are weakly monotonic.
|
||||
EXPECT_GE(evaluator.counters().true_positives, last_tp);
|
||||
EXPECT_GE(evaluator.counters().true_negatives, last_tn);
|
||||
EXPECT_GE(evaluator.counters().false_positives, last_fp);
|
||||
EXPECT_GE(evaluator.counters().false_negatives, last_fn);
|
||||
const auto post = evaluator.counters();
|
||||
EXPECT_GE(post.true_positives, pre.true_positives);
|
||||
EXPECT_GE(post.true_negatives, pre.true_negatives);
|
||||
EXPECT_GE(post.false_positives, pre.false_positives);
|
||||
EXPECT_GE(post.false_negatives, pre.false_negatives);
|
||||
}
|
||||
}
|
||||
|
||||
@ -126,23 +124,20 @@ TEST_P(ClippingPredictorEvaluatorParameterization, BoundedMetricsGrowth) {
|
||||
for (int i = 0; i < kNumCalls; ++i) {
|
||||
SCOPED_TRACE(i);
|
||||
// Read metrics before `Observe()` is called.
|
||||
const int last_tp = evaluator.counters().true_positives;
|
||||
const int last_tn = evaluator.counters().true_negatives;
|
||||
const int last_fp = evaluator.counters().false_positives;
|
||||
const int last_fn = evaluator.counters().false_negatives;
|
||||
const auto pre = evaluator.counters();
|
||||
// `Observe()` a random observation.
|
||||
bool clipping_detected = random_generator.Rand<bool>();
|
||||
bool clipping_predicted = random_generator.Rand<bool>();
|
||||
evaluator.Observe(clipping_detected, clipping_predicted);
|
||||
|
||||
const auto diff = evaluator.counters() - pre;
|
||||
// Check that TPs grow by at most `history_size() + 1`. Such an upper bound
|
||||
// is reached when multiple predictions are matched by a single detection.
|
||||
EXPECT_LE(evaluator.counters().true_positives - last_tp,
|
||||
history_size() + 1);
|
||||
// Check that TNs, FPs and FNs grow by at most one. `max_growth`.
|
||||
EXPECT_LE(evaluator.counters().true_negatives - last_tn, 1);
|
||||
EXPECT_LE(evaluator.counters().false_positives - last_fp, 1);
|
||||
EXPECT_LE(evaluator.counters().false_negatives - last_fn, 1);
|
||||
EXPECT_LE(diff.true_positives, history_size() + 1);
|
||||
// Check that TNs, FPs and FNs grow by at most one.
|
||||
EXPECT_LE(diff.true_negatives, 1);
|
||||
EXPECT_LE(diff.false_positives, 1);
|
||||
EXPECT_LE(diff.false_negatives, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@ -180,90 +175,144 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
::testing::Combine(::testing::Values(4, 8, 15, 16, 23, 42),
|
||||
::testing::Values(1, 10, 21)));
|
||||
|
||||
// Checks that, observing a detection and a prediction after init, produces a
|
||||
// true positive.
|
||||
TEST(ClippingPredictionEvalTest, OneTruePositiveAfterInit) {
|
||||
// Checks that after initialization, when no detection is expected,
|
||||
// observing no detection and no prediction produces a true negative.
|
||||
TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectNoPredictAfterInit) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kDetected, kPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that, observing a detection but no prediction after init, produces a
|
||||
// false negative.
|
||||
TEST(ClippingPredictionEvalTest, OneFalseNegativeAfterInit) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 1);
|
||||
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
}
|
||||
|
||||
// Checks that, observing no detection but a prediction after init, produces a
|
||||
// false positive after expiration.
|
||||
TEST(ClippingPredictionEvalTest, OneFalsePositiveAfterInit) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that, observing no detection and no prediction after init, produces a
|
||||
// true negative.
|
||||
TEST(ClippingPredictionEvalTest, OneTrueNegativeAfterInit) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that after initialization, when no detection is expected,
|
||||
// observing no detection and prediction produces a true negative.
|
||||
TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectPredictAfterInit) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 1);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that after initialization, when no detection is expected,
|
||||
// observing a detection and no prediction produces a false negative.
|
||||
TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectNoPredictAfterInit) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 1);
|
||||
}
|
||||
|
||||
// Checks that after initialization, when no detection is expected,
|
||||
// simultaneously observing a detection and a prediction produces a false
|
||||
// negative.
|
||||
TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectPredictAfterInit) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
|
||||
evaluator.Observe(kDetected, kPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 1);
|
||||
}
|
||||
|
||||
// Checks that, after removing existing expectations, observing no detection and
|
||||
// no prediction produces a true negative.
|
||||
TEST(ClippingPredictionEvalTest,
|
||||
TrueNegativeWithNoDetectNoPredictAfterRemoveExpectations) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
|
||||
// Set an expectation, then remove it.
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
evaluator.RemoveExpectations();
|
||||
const auto pre = evaluator.counters();
|
||||
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
const auto diff = evaluator.counters() - pre;
|
||||
EXPECT_EQ(diff.true_positives, 0);
|
||||
EXPECT_EQ(diff.true_negatives, 1);
|
||||
EXPECT_EQ(diff.false_positives, 0);
|
||||
EXPECT_EQ(diff.false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that, after removing existing expectations, observing no detection and
|
||||
// a prediction produces a true negative.
|
||||
TEST(ClippingPredictionEvalTest,
|
||||
TrueNegativeWithNoDetectPredictAfterRemoveExpectations) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
|
||||
// Set an expectation, then remove it.
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
evaluator.RemoveExpectations();
|
||||
const auto pre = evaluator.counters();
|
||||
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
const auto diff = evaluator.counters() - pre;
|
||||
EXPECT_EQ(diff.true_positives, 0);
|
||||
EXPECT_EQ(diff.true_negatives, 1);
|
||||
EXPECT_EQ(diff.false_positives, 0);
|
||||
EXPECT_EQ(diff.false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that, after removing existing expectations, observing a detection and
|
||||
// no prediction produces a false negative.
|
||||
TEST(ClippingPredictionEvalTest,
|
||||
FalseNegativeWithDetectNoPredictAfterRemoveExpectations) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
|
||||
// Set an expectation, then remove it.
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
evaluator.RemoveExpectations();
|
||||
const auto pre = evaluator.counters();
|
||||
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
const auto diff = evaluator.counters() - pre;
|
||||
EXPECT_EQ(diff.true_positives, 0);
|
||||
EXPECT_EQ(diff.true_negatives, 0);
|
||||
EXPECT_EQ(diff.false_positives, 0);
|
||||
EXPECT_EQ(diff.false_negatives, 1);
|
||||
}
|
||||
|
||||
// Checks that, after removing existing expectations, simultaneously observing a
|
||||
// detection and a prediction produces a false negative.
|
||||
TEST(ClippingPredictionEvalTest,
|
||||
FalseNegativeWithDetectPredictAfterRemoveExpectations) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
|
||||
// Set an expectation, then remove it.
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
evaluator.RemoveExpectations();
|
||||
const auto pre = evaluator.counters();
|
||||
|
||||
evaluator.Observe(kDetected, kPredicted);
|
||||
const auto diff = evaluator.counters() - pre;
|
||||
EXPECT_EQ(diff.false_negatives, 1);
|
||||
EXPECT_EQ(diff.true_positives, 0);
|
||||
EXPECT_EQ(diff.true_negatives, 0);
|
||||
EXPECT_EQ(diff.false_positives, 0);
|
||||
}
|
||||
|
||||
// Checks that the evaluator detects true negatives when clipping is neither
|
||||
// predicted nor detected.
|
||||
TEST(ClippingPredictionEvalTest, NeverDetectedAndNotPredicted) {
|
||||
TEST(ClippingPredictionEvalTest, TrueNegativesWhenNeverDetectedOrPredicted) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 4);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that the evaluator detects a false negative when clipping is detected
|
||||
// but not predicted.
|
||||
TEST(ClippingPredictionEvalTest, DetectedButNotPredicted) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 3);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
}
|
||||
|
||||
// Checks that the evaluator does not detect a false positive when clipping is
|
||||
// predicted but not detected until the observation period expires.
|
||||
// Checks that, until the observation period expires, the evaluator does not
|
||||
// count a false positive when clipping is predicted and not detected.
|
||||
TEST(ClippingPredictionEvalTest, PredictedOnceAndNeverDetectedBeforeDeadline) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
@ -272,15 +321,12 @@ TEST(ClippingPredictionEvalTest, PredictedOnceAndNeverDetectedBeforeDeadline) {
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 1);
|
||||
}
|
||||
|
||||
// Checks that the evaluator detects a false positive when clipping is predicted
|
||||
// but detected after the observation period expires.
|
||||
// Checks that, after the observation period expires, the evaluator detects a
|
||||
// false positive when clipping is predicted and detected.
|
||||
TEST(ClippingPredictionEvalTest, PredictedOnceButDetectedAfterDeadline) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
@ -288,24 +334,17 @@ TEST(ClippingPredictionEvalTest, PredictedOnceButDetectedAfterDeadline) {
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 1);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 1);
|
||||
}
|
||||
|
||||
// Checks that a prediction followed by a detection counts as true positive.
|
||||
TEST(ClippingPredictionEvalTest, PredictedOnceAndThenImmediatelyDetected) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that a prediction followed by a delayed detection counts as true
|
||||
@ -313,15 +352,10 @@ TEST(ClippingPredictionEvalTest, PredictedOnceAndThenImmediatelyDetected) {
|
||||
TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedBeforeDeadline) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that a prediction followed by a delayed detection counts as true
|
||||
@ -329,17 +363,11 @@ TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedBeforeDeadline) {
|
||||
TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedAtDeadline) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
evaluator.Observe(kNotDetected, kPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 1);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
// Checks that a prediction followed by a multiple adjacent detections within
|
||||
@ -352,19 +380,22 @@ TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedMultipleTimes) {
|
||||
// Multiple detections.
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 1);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 1);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
// A detection outside of the observation period counts as false negative.
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 1);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 1);
|
||||
EXPECT_EQ(SumTrueFalsePositivesNegatives(evaluator), 2);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
}
|
||||
|
||||
// Checks that a false positive is added when clipping is detected after a too
|
||||
// early prediction.
|
||||
// Checks that when clipping is predicted multiple times, a prediction that is
|
||||
// observed too early counts as a false positive, whereas the other predictions
|
||||
// that are matched to a detection count as true positives.
|
||||
TEST(ClippingPredictionEvalTest,
|
||||
PredictedMultipleTimesAndDetectedOnceAfterDeadline) {
|
||||
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
|
||||
@ -378,9 +409,8 @@ TEST(ClippingPredictionEvalTest,
|
||||
// The detection above does not match the first prediction because it happened
|
||||
// after the deadline of the 1st prediction.
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 1);
|
||||
|
||||
// However, the detection matches all the other predictions.
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 3);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
@ -401,7 +431,6 @@ TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndDetectedOnce) {
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, true_negatives);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
@ -424,7 +453,6 @@ TEST(ClippingPredictionEvalTest,
|
||||
evaluator.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, true_negatives);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
@ -440,7 +468,7 @@ TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndAllDetected) {
|
||||
evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+
|
||||
evaluator.Observe(kDetected, kNotPredicted); // <-+
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 3);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
@ -456,7 +484,7 @@ TEST(ClippingPredictionEvalTest, PredictedMultipleTimesWithGapAndAllDetected) {
|
||||
evaluator.Observe(kDetected, kNotPredicted); // <-+
|
||||
evaluator.Observe(kDetected, kNotPredicted); // <-+
|
||||
EXPECT_EQ(evaluator.counters().true_positives, 2);
|
||||
EXPECT_EQ(evaluator.counters().true_negatives, 0);
|
||||
|
||||
EXPECT_EQ(evaluator.counters().false_positives, 0);
|
||||
EXPECT_EQ(evaluator.counters().false_negatives, 0);
|
||||
}
|
||||
@ -469,16 +497,16 @@ class ClippingPredictorEvaluatorPredictionIntervalParameterization
|
||||
};
|
||||
|
||||
// Checks that the minimum prediction interval is returned if clipping is
|
||||
// correctly predicted as soon as detected - i.e., no anticipation.
|
||||
// correctly predicted just before clipping is detected - i.e., smallest
|
||||
// anticipation.
|
||||
TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
|
||||
MinimumPredictionInterval) {
|
||||
ClippingPredictorEvaluator evaluator(history_size());
|
||||
for (int i = 0; i < num_extra_observe_calls(); ++i) {
|
||||
EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt);
|
||||
}
|
||||
absl::optional<int> prediction_interval =
|
||||
evaluator.Observe(kDetected, kPredicted);
|
||||
EXPECT_THAT(prediction_interval, Optional(Eq(0)));
|
||||
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
|
||||
EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(1)));
|
||||
}
|
||||
|
||||
// Checks that a prediction interval between the minimum and the maximum is
|
||||
@ -493,9 +521,7 @@ TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
|
||||
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
|
||||
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
|
||||
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
|
||||
absl::optional<int> prediction_interval =
|
||||
evaluator.Observe(kDetected, kPredicted);
|
||||
EXPECT_THAT(prediction_interval, Optional(Eq(3)));
|
||||
EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(3)));
|
||||
}
|
||||
|
||||
// Checks that the maximum prediction interval is returned if clipping is
|
||||
@ -509,9 +535,8 @@ TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
|
||||
for (int i = 0; i < history_size(); ++i) {
|
||||
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
|
||||
}
|
||||
absl::optional<int> prediction_interval =
|
||||
evaluator.Observe(kDetected, kPredicted);
|
||||
EXPECT_THAT(prediction_interval, Optional(Eq(history_size())));
|
||||
EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted),
|
||||
Optional(Eq(history_size())));
|
||||
}
|
||||
|
||||
// Checks that `Observe()` returns the prediction interval as soon as a true
|
||||
@ -527,7 +552,7 @@ TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
|
||||
}
|
||||
// Observe a detection.
|
||||
absl::optional<int> prediction_interval =
|
||||
evaluator.Observe(kDetected, kPredicted);
|
||||
evaluator.Observe(kDetected, kNotPredicted);
|
||||
EXPECT_TRUE(prediction_interval.has_value());
|
||||
// `Observe()` does not return a prediction interval anymore during ongoing
|
||||
// detections observed while a detection is still expected.
|
||||
@ -539,31 +564,36 @@ TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
ClippingPredictionEvalTest,
|
||||
ClippingPredictorEvaluatorPredictionIntervalParameterization,
|
||||
::testing::Combine(::testing::Values(0, 3, 5), ::testing::Values(7, 11)));
|
||||
::testing::Combine(::testing::Values(1, 3, 5), ::testing::Values(7, 11)));
|
||||
|
||||
// Checks that, when a detection is expected, the expectation is removed if and
|
||||
// only if `Reset()` is called after a prediction is observed.
|
||||
TEST(ClippingPredictionEvalTest, NoFalsePositivesAfterReset) {
|
||||
// Checks that, when a detection is expected, the expectation is not removed
|
||||
// before the detection deadline expires unless `RemoveExpectations()` is
|
||||
// called.
|
||||
TEST(ClippingPredictionEvalTest, NoFalsePositivesAfterRemoveExpectations) {
|
||||
constexpr int kHistorySize = 2;
|
||||
|
||||
ClippingPredictorEvaluator with_reset(kHistorySize);
|
||||
with_reset.Observe(kNotDetected, kPredicted);
|
||||
with_reset.Reset();
|
||||
with_reset.Observe(kNotDetected, kNotPredicted);
|
||||
with_reset.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(with_reset.counters().true_positives, 0);
|
||||
EXPECT_EQ(with_reset.counters().true_negatives, 2);
|
||||
EXPECT_EQ(with_reset.counters().false_positives, 0);
|
||||
EXPECT_EQ(with_reset.counters().false_negatives, 0);
|
||||
// Case 1: `RemoveExpectations()` is NOT called.
|
||||
ClippingPredictorEvaluator e1(kHistorySize);
|
||||
e1.Observe(kNotDetected, kPredicted);
|
||||
ASSERT_EQ(e1.counters().true_negatives, 1);
|
||||
e1.Observe(kNotDetected, kNotPredicted);
|
||||
e1.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(e1.counters().true_positives, 0);
|
||||
EXPECT_EQ(e1.counters().true_negatives, 1);
|
||||
EXPECT_EQ(e1.counters().false_positives, 1);
|
||||
EXPECT_EQ(e1.counters().false_negatives, 0);
|
||||
|
||||
ClippingPredictorEvaluator no_reset(kHistorySize);
|
||||
no_reset.Observe(kNotDetected, kPredicted);
|
||||
no_reset.Observe(kNotDetected, kNotPredicted);
|
||||
no_reset.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(no_reset.counters().true_positives, 0);
|
||||
EXPECT_EQ(no_reset.counters().true_negatives, 0);
|
||||
EXPECT_EQ(no_reset.counters().false_positives, 1);
|
||||
EXPECT_EQ(no_reset.counters().false_negatives, 0);
|
||||
// Case 2: `RemoveExpectations()` is called.
|
||||
ClippingPredictorEvaluator e2(kHistorySize);
|
||||
e2.Observe(kNotDetected, kPredicted);
|
||||
ASSERT_EQ(e2.counters().true_negatives, 1);
|
||||
e2.RemoveExpectations();
|
||||
e2.Observe(kNotDetected, kNotPredicted);
|
||||
e2.Observe(kNotDetected, kNotPredicted);
|
||||
EXPECT_EQ(e2.counters().true_positives, 0);
|
||||
EXPECT_EQ(e2.counters().true_negatives, 3);
|
||||
EXPECT_EQ(e2.counters().false_positives, 0);
|
||||
EXPECT_EQ(e2.counters().false_negatives, 0);
|
||||
}
|
||||
|
||||
class ComputeClippingPredictionMetricsParameterization
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user