ClippingPredictorEvaluator: predictions only match future detections
To focus on the ability to predict clipping, the clipping predictor
evaluator doesn't increment the true positive count anymore when a
prediction is simultaneously observed with a detection.
Note that `WebRTC.Audio.Agc.ClippingPredictor.F1Score` is still used
to log the F1 score - i.e., the histogram hasn't been renamed.
Bug: webrtc:12774
Change-Id: Ia987e568a6df2a3ddba7fa1b5697d6feda22d20c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/231233
Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
Reviewed-by: Hanna Silen <silen@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#34942}
diff --git a/modules/audio_processing/agc/agc_manager_direct.cc b/modules/audio_processing/agc/agc_manager_direct.cc
index 22a097e..bda1cae 100644
--- a/modules/audio_processing/agc/agc_manager_direct.cc
+++ b/modules/audio_processing/agc/agc_manager_direct.cc
@@ -629,7 +629,7 @@
frames_since_clipped_ = 0;
if (!!clipping_predictor_) {
clipping_predictor_->Reset();
- clipping_predictor_evaluator_.Reset();
+ clipping_predictor_evaluator_.RemoveExpectations();
}
}
AggregateChannelLevels();
diff --git a/modules/audio_processing/agc/clipping_predictor_evaluator.cc b/modules/audio_processing/agc/clipping_predictor_evaluator.cc
index f211d6e..ed7198d 100644
--- a/modules/audio_processing/agc/clipping_predictor_evaluator.cc
+++ b/modules/audio_processing/agc/clipping_predictor_evaluator.cc
@@ -50,10 +50,6 @@
RTC_DCHECK_LT(ring_buffer_tail_, ring_buffer_capacity_);
DecreaseTimesToLive();
- if (clipping_predicted) {
- // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed.
- Push(/*expected_detection=*/{/*ttl=*/history_size_, /*detected=*/false});
- }
// Clipping is expected if there are expected detections regardless of
// whether all the expected detections have been previously matched - i.e.,
// `ExpectedDetection::detected` is true.
@@ -80,15 +76,29 @@
RTC_DCHECK(!clipping_expected && !clipping_detected);
counters_.true_negatives++;
}
+
+ if (clipping_predicted) {
+ // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed.
+ Push(/*expected_detection=*/{/*ttl=*/history_size_, /*detected=*/false});
+ }
+
return prediction_interval;
}
-void ClippingPredictorEvaluator::Reset() {
+void ClippingPredictorEvaluator::RemoveExpectations() {
// Empty the ring buffer of expected detections.
ring_buffer_tail_ = 0;
ring_buffer_size_ = 0;
}
+void ClippingPredictorEvaluator::Reset() {
+ counters_.true_positives = 0;
+ counters_.true_negatives = 0;
+ counters_.false_positives = 0;
+ counters_.false_negatives = 0;
+ RemoveExpectations();
+}
+
// Cost: O(1).
void ClippingPredictorEvaluator::Push(ExpectedDetection value) {
ring_buffer_[ring_buffer_tail_] = value;
diff --git a/modules/audio_processing/agc/clipping_predictor_evaluator.h b/modules/audio_processing/agc/clipping_predictor_evaluator.h
index 3a4ebd5..348f753 100644
--- a/modules/audio_processing/agc/clipping_predictor_evaluator.h
+++ b/modules/audio_processing/agc/clipping_predictor_evaluator.h
@@ -57,6 +57,9 @@
// Removes any expectation recently set after a call to `Observe()` having
// `clipping_predicted` set to true. Counters won't be reset.
+ void RemoveExpectations();
+
+ // Resets counters and removes any expectation (see `RemoveExpectations()`).
void Reset();
ClippingPredictionCounters counters() const { return counters_; }
diff --git a/modules/audio_processing/agc/clipping_predictor_evaluator_unittest.cc b/modules/audio_processing/agc/clipping_predictor_evaluator_unittest.cc
index caebe8f..b2d2797 100644
--- a/modules/audio_processing/agc/clipping_predictor_evaluator_unittest.cc
+++ b/modules/audio_processing/agc/clipping_predictor_evaluator_unittest.cc
@@ -34,12 +34,14 @@
constexpr bool kPredicted = true;
constexpr bool kNotPredicted = false;
-int SumTrueFalsePositivesNegatives(
- const ClippingPredictorEvaluator& evaluator) {
- return evaluator.counters().true_positives +
- evaluator.counters().true_negatives +
- evaluator.counters().false_positives +
- evaluator.counters().false_negatives;
+ClippingPredictionCounters operator-(const ClippingPredictionCounters& lhs,
+ const ClippingPredictionCounters& rhs) {
+ return {
+ lhs.true_positives - rhs.true_positives,
+ lhs.true_negatives - rhs.true_negatives,
+ lhs.false_positives - rhs.false_positives,
+ lhs.false_negatives - rhs.false_negatives,
+ };
}
// Checks the metrics after init - i.e., no call to `Observe()`.
@@ -69,21 +71,19 @@
for (int i = 0; i < kNumCalls; ++i) {
SCOPED_TRACE(i);
// Read metrics before `Observe()` is called.
- const int last_tp = evaluator.counters().true_positives;
- const int last_tn = evaluator.counters().true_negatives;
- const int last_fp = evaluator.counters().false_positives;
- const int last_fn = evaluator.counters().false_negatives;
+ const auto pre = evaluator.counters();
// `Observe()` a random observation.
bool clipping_detected = random_generator.Rand<bool>();
bool clipping_predicted = random_generator.Rand<bool>();
evaluator.Observe(clipping_detected, clipping_predicted);
// Check that at most one metric has changed.
+ const auto post = evaluator.counters();
int num_changes = 0;
- num_changes += last_tp == evaluator.counters().true_positives ? 0 : 1;
- num_changes += last_tn == evaluator.counters().true_negatives ? 0 : 1;
- num_changes += last_fp == evaluator.counters().false_positives ? 0 : 1;
- num_changes += last_fn == evaluator.counters().false_negatives ? 0 : 1;
+ num_changes += pre.true_positives == post.true_positives ? 0 : 1;
+ num_changes += pre.true_negatives == post.true_negatives ? 0 : 1;
+ num_changes += pre.false_positives == post.false_positives ? 0 : 1;
+ num_changes += pre.false_negatives == post.false_negatives ? 0 : 1;
EXPECT_GE(num_changes, 0);
EXPECT_LE(num_changes, 1);
}
@@ -99,20 +99,18 @@
for (int i = 0; i < kNumCalls; ++i) {
SCOPED_TRACE(i);
// Read metrics before `Observe()` is called.
- const int last_tp = evaluator.counters().true_positives;
- const int last_tn = evaluator.counters().true_negatives;
- const int last_fp = evaluator.counters().false_positives;
- const int last_fn = evaluator.counters().false_negatives;
+ const auto pre = evaluator.counters();
// `Observe()` a random observation.
bool clipping_detected = random_generator.Rand<bool>();
bool clipping_predicted = random_generator.Rand<bool>();
evaluator.Observe(clipping_detected, clipping_predicted);
// Check that metrics are weakly monotonic.
- EXPECT_GE(evaluator.counters().true_positives, last_tp);
- EXPECT_GE(evaluator.counters().true_negatives, last_tn);
- EXPECT_GE(evaluator.counters().false_positives, last_fp);
- EXPECT_GE(evaluator.counters().false_negatives, last_fn);
+ const auto post = evaluator.counters();
+ EXPECT_GE(post.true_positives, pre.true_positives);
+ EXPECT_GE(post.true_negatives, pre.true_negatives);
+ EXPECT_GE(post.false_positives, pre.false_positives);
+ EXPECT_GE(post.false_negatives, pre.false_negatives);
}
}
@@ -126,23 +124,20 @@
for (int i = 0; i < kNumCalls; ++i) {
SCOPED_TRACE(i);
// Read metrics before `Observe()` is called.
- const int last_tp = evaluator.counters().true_positives;
- const int last_tn = evaluator.counters().true_negatives;
- const int last_fp = evaluator.counters().false_positives;
- const int last_fn = evaluator.counters().false_negatives;
+ const auto pre = evaluator.counters();
// `Observe()` a random observation.
bool clipping_detected = random_generator.Rand<bool>();
bool clipping_predicted = random_generator.Rand<bool>();
evaluator.Observe(clipping_detected, clipping_predicted);
+ const auto diff = evaluator.counters() - pre;
// Check that TPs grow by at most `history_size() + 1`. Such an upper bound
// is reached when multiple predictions are matched by a single detection.
- EXPECT_LE(evaluator.counters().true_positives - last_tp,
- history_size() + 1);
- // Check that TNs, FPs and FNs grow by at most one. `max_growth`.
- EXPECT_LE(evaluator.counters().true_negatives - last_tn, 1);
- EXPECT_LE(evaluator.counters().false_positives - last_fp, 1);
- EXPECT_LE(evaluator.counters().false_negatives - last_fn, 1);
+ EXPECT_LE(diff.true_positives, history_size() + 1);
+ // Check that TNs, FPs and FNs grow by at most one.
+ EXPECT_LE(diff.true_negatives, 1);
+ EXPECT_LE(diff.false_positives, 1);
+ EXPECT_LE(diff.false_negatives, 1);
}
}
@@ -180,90 +175,144 @@
::testing::Combine(::testing::Values(4, 8, 15, 16, 23, 42),
::testing::Values(1, 10, 21)));
-// Checks that, observing a detection and a prediction after init, produces a
-// true positive.
-TEST(ClippingPredictionEvalTest, OneTruePositiveAfterInit) {
+// Checks that after initialization, when no detection is expected,
+// observing no detection and no prediction produces a true negative.
+TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectNoPredictAfterInit) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
- evaluator.Observe(kDetected, kPredicted);
- EXPECT_EQ(evaluator.counters().true_positives, 1);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
- EXPECT_EQ(evaluator.counters().false_negatives, 0);
-}
-
-// Checks that, observing a detection but no prediction after init, produces a
-// false negative.
-TEST(ClippingPredictionEvalTest, OneFalseNegativeAfterInit) {
- ClippingPredictorEvaluator evaluator(/*history_size=*/3);
- evaluator.Observe(kDetected, kNotPredicted);
- EXPECT_EQ(evaluator.counters().false_negatives, 1);
-
+ evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 0);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
-}
-
-// Checks that, observing no detection but a prediction after init, produces a
-// false positive after expiration.
-TEST(ClippingPredictionEvalTest, OneFalsePositiveAfterInit) {
- ClippingPredictorEvaluator evaluator(/*history_size=*/3);
- evaluator.Observe(kNotDetected, kPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
- evaluator.Observe(kNotDetected, kNotPredicted);
- evaluator.Observe(kNotDetected, kNotPredicted);
- evaluator.Observe(kNotDetected, kNotPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 1);
-
- EXPECT_EQ(evaluator.counters().true_positives, 0);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
- EXPECT_EQ(evaluator.counters().false_negatives, 0);
-}
-
-// Checks that, observing no detection and no prediction after init, produces a
-// true negative.
-TEST(ClippingPredictionEvalTest, OneTrueNegativeAfterInit) {
- ClippingPredictorEvaluator evaluator(/*history_size=*/3);
- evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_negatives, 1);
-
- EXPECT_EQ(evaluator.counters().true_positives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
+// Checks that after initialization, when no detection is expected,
+// observing no detection and prediction produces a true negative.
+TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectPredictAfterInit) {
+ ClippingPredictorEvaluator evaluator(/*history_size=*/3);
+
+ evaluator.Observe(kNotDetected, kPredicted);
+ EXPECT_EQ(evaluator.counters().true_positives, 0);
+ EXPECT_EQ(evaluator.counters().true_negatives, 1);
+ EXPECT_EQ(evaluator.counters().false_positives, 0);
+ EXPECT_EQ(evaluator.counters().false_negatives, 0);
+}
+
+// Checks that after initialization, when no detection is expected,
+// observing a detection and no prediction produces a false negative.
+TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectNoPredictAfterInit) {
+ ClippingPredictorEvaluator evaluator(/*history_size=*/3);
+
+ evaluator.Observe(kDetected, kNotPredicted);
+ EXPECT_EQ(evaluator.counters().true_positives, 0);
+ EXPECT_EQ(evaluator.counters().true_negatives, 0);
+ EXPECT_EQ(evaluator.counters().false_positives, 0);
+ EXPECT_EQ(evaluator.counters().false_negatives, 1);
+}
+
+// Checks that after initialization, when no detection is expected,
+// simultaneously observing a detection and a prediction produces a false
+// negative.
+TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectPredictAfterInit) {
+ ClippingPredictorEvaluator evaluator(/*history_size=*/3);
+
+ evaluator.Observe(kDetected, kPredicted);
+ EXPECT_EQ(evaluator.counters().true_positives, 0);
+ EXPECT_EQ(evaluator.counters().true_negatives, 0);
+ EXPECT_EQ(evaluator.counters().false_positives, 0);
+ EXPECT_EQ(evaluator.counters().false_negatives, 1);
+}
+
+// Checks that, after removing existing expectations, observing no detection and
+// no prediction produces a true negative.
+TEST(ClippingPredictionEvalTest,
+ TrueNegativeWithNoDetectNoPredictAfterRemoveExpectations) {
+ ClippingPredictorEvaluator evaluator(/*history_size=*/3);
+
+ // Set an expectation, then remove it.
+ evaluator.Observe(kNotDetected, kPredicted);
+ evaluator.RemoveExpectations();
+ const auto pre = evaluator.counters();
+
+ evaluator.Observe(kNotDetected, kNotPredicted);
+ const auto diff = evaluator.counters() - pre;
+ EXPECT_EQ(diff.true_positives, 0);
+ EXPECT_EQ(diff.true_negatives, 1);
+ EXPECT_EQ(diff.false_positives, 0);
+ EXPECT_EQ(diff.false_negatives, 0);
+}
+
+// Checks that, after removing existing expectations, observing no detection and
+// a prediction produces a true negative.
+TEST(ClippingPredictionEvalTest,
+ TrueNegativeWithNoDetectPredictAfterRemoveExpectations) {
+ ClippingPredictorEvaluator evaluator(/*history_size=*/3);
+
+ // Set an expectation, then remove it.
+ evaluator.Observe(kNotDetected, kPredicted);
+ evaluator.RemoveExpectations();
+ const auto pre = evaluator.counters();
+
+ evaluator.Observe(kNotDetected, kPredicted);
+ const auto diff = evaluator.counters() - pre;
+ EXPECT_EQ(diff.true_positives, 0);
+ EXPECT_EQ(diff.true_negatives, 1);
+ EXPECT_EQ(diff.false_positives, 0);
+ EXPECT_EQ(diff.false_negatives, 0);
+}
+
+// Checks that, after removing existing expectations, observing a detection and
+// no prediction produces a false negative.
+TEST(ClippingPredictionEvalTest,
+ FalseNegativeWithDetectNoPredictAfterRemoveExpectations) {
+ ClippingPredictorEvaluator evaluator(/*history_size=*/3);
+
+ // Set an expectation, then remove it.
+ evaluator.Observe(kNotDetected, kPredicted);
+ evaluator.RemoveExpectations();
+ const auto pre = evaluator.counters();
+
+ evaluator.Observe(kDetected, kNotPredicted);
+ const auto diff = evaluator.counters() - pre;
+ EXPECT_EQ(diff.true_positives, 0);
+ EXPECT_EQ(diff.true_negatives, 0);
+ EXPECT_EQ(diff.false_positives, 0);
+ EXPECT_EQ(diff.false_negatives, 1);
+}
+
+// Checks that, after removing existing expectations, simultaneously observing a
+// detection and a prediction produces a false negative.
+TEST(ClippingPredictionEvalTest,
+ FalseNegativeWithDetectPredictAfterRemoveExpectations) {
+ ClippingPredictorEvaluator evaluator(/*history_size=*/3);
+
+ // Set an expectation, then remove it.
+ evaluator.Observe(kNotDetected, kPredicted);
+ evaluator.RemoveExpectations();
+ const auto pre = evaluator.counters();
+
+ evaluator.Observe(kDetected, kPredicted);
+ const auto diff = evaluator.counters() - pre;
+ EXPECT_EQ(diff.false_negatives, 1);
+ EXPECT_EQ(diff.true_positives, 0);
+ EXPECT_EQ(diff.true_negatives, 0);
+ EXPECT_EQ(diff.false_positives, 0);
+}
+
// Checks that the evaluator detects true negatives when clipping is neither
// predicted nor detected.
-TEST(ClippingPredictionEvalTest, NeverDetectedAndNotPredicted) {
+TEST(ClippingPredictionEvalTest, TrueNegativesWhenNeverDetectedOrPredicted) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_negatives, 4);
-
- EXPECT_EQ(evaluator.counters().true_positives, 0);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
- EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
-// Checks that the evaluator detects a false negative when clipping is detected
-// but not predicted.
-TEST(ClippingPredictionEvalTest, DetectedButNotPredicted) {
- ClippingPredictorEvaluator evaluator(/*history_size=*/3);
- evaluator.Observe(kNotDetected, kNotPredicted);
- evaluator.Observe(kNotDetected, kNotPredicted);
- evaluator.Observe(kNotDetected, kNotPredicted);
- evaluator.Observe(kDetected, kNotPredicted);
- EXPECT_EQ(evaluator.counters().false_negatives, 1);
-
- EXPECT_EQ(evaluator.counters().true_positives, 0);
- EXPECT_EQ(evaluator.counters().true_negatives, 3);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
-}
-
-// Checks that the evaluator does not detect a false positive when clipping is
-// predicted but not detected until the observation period expires.
+// Checks that, until the observation period expires, the evaluator does not
+// count a false positive when clipping is predicted and not detected.
TEST(ClippingPredictionEvalTest, PredictedOnceAndNeverDetectedBeforeDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
@@ -272,15 +321,12 @@
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kNotDetected, kNotPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 1);
-
EXPECT_EQ(evaluator.counters().true_positives, 0);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
- EXPECT_EQ(evaluator.counters().false_negatives, 0);
+ EXPECT_EQ(evaluator.counters().false_positives, 1);
}
-// Checks that the evaluator detects a false positive when clipping is predicted
-// but detected after the observation period expires.
+// Checks that, after the observation period expires, the evaluator detects a
+// false positive when clipping is predicted and detected.
TEST(ClippingPredictionEvalTest, PredictedOnceButDetectedAfterDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
@@ -288,24 +334,17 @@
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kDetected, kNotPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 1);
-
EXPECT_EQ(evaluator.counters().true_positives, 0);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
- EXPECT_EQ(evaluator.counters().false_negatives, 1);
+ EXPECT_EQ(evaluator.counters().false_positives, 1);
}
// Checks that a prediction followed by a detection counts as true positive.
TEST(ClippingPredictionEvalTest, PredictedOnceAndThenImmediatelyDetected) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
-
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
- EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that a prediction followed by a delayed detection counts as true
@@ -313,15 +352,10 @@
TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedBeforeDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kNotDetected, kNotPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
-
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
- EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that a prediction followed by a delayed detection counts as true
@@ -329,17 +363,11 @@
TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedAtDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kNotDetected, kNotPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kNotDetected, kNotPredicted);
- EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
-
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
- EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that a prediction followed by a multiple adjacent detections within
@@ -352,19 +380,22 @@
// Multiple detections.
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
+ EXPECT_EQ(evaluator.counters().false_negatives, 0);
+ EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
+ EXPECT_EQ(evaluator.counters().false_negatives, 0);
+ EXPECT_EQ(evaluator.counters().false_positives, 0);
// A detection outside of the observation period counts as false negative.
evaluator.Observe(kDetected, kNotPredicted);
+ EXPECT_EQ(evaluator.counters().true_positives, 1);
EXPECT_EQ(evaluator.counters().false_negatives, 1);
- EXPECT_EQ(SumTrueFalsePositivesNegatives(evaluator), 2);
-
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
}
-// Checks that a false positive is added when clipping is detected after a too
-// early prediction.
+// Checks that when clipping is predicted multiple times, a prediction that is
+// observed too early counts as a false positive, whereas the other predictions
+// that are matched to a detection count as true positives.
TEST(ClippingPredictionEvalTest,
PredictedMultipleTimesAndDetectedOnceAfterDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
@@ -378,9 +409,8 @@
// The detection above does not match the first prediction because it happened
// after the deadline of the 1st prediction.
EXPECT_EQ(evaluator.counters().false_positives, 1);
-
+ // However, the detection matches all the other predictions.
EXPECT_EQ(evaluator.counters().true_positives, 3);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
@@ -401,7 +431,6 @@
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_negatives, true_negatives);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
@@ -424,7 +453,6 @@
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_negatives, true_negatives);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
@@ -440,7 +468,7 @@
evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+
evaluator.Observe(kDetected, kNotPredicted); // <-+
EXPECT_EQ(evaluator.counters().true_positives, 3);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
+
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
@@ -456,7 +484,7 @@
evaluator.Observe(kDetected, kNotPredicted); // <-+
evaluator.Observe(kDetected, kNotPredicted); // <-+
EXPECT_EQ(evaluator.counters().true_positives, 2);
- EXPECT_EQ(evaluator.counters().true_negatives, 0);
+
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
@@ -469,16 +497,16 @@
};
// Checks that the minimum prediction interval is returned if clipping is
-// correctly predicted as soon as detected - i.e., no anticipation.
+// correctly predicted just before clipping is detected - i.e., smallest
+// anticipation.
TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
MinimumPredictionInterval) {
ClippingPredictorEvaluator evaluator(history_size());
for (int i = 0; i < num_extra_observe_calls(); ++i) {
EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt);
}
- absl::optional<int> prediction_interval =
- evaluator.Observe(kDetected, kPredicted);
- EXPECT_THAT(prediction_interval, Optional(Eq(0)));
+ EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
+ EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(1)));
}
// Checks that a prediction interval between the minimum and the maximum is
@@ -493,9 +521,7 @@
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
- absl::optional<int> prediction_interval =
- evaluator.Observe(kDetected, kPredicted);
- EXPECT_THAT(prediction_interval, Optional(Eq(3)));
+ EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(3)));
}
// Checks that the maximum prediction interval is returned if clipping is
@@ -509,9 +535,8 @@
for (int i = 0; i < history_size(); ++i) {
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
}
- absl::optional<int> prediction_interval =
- evaluator.Observe(kDetected, kPredicted);
- EXPECT_THAT(prediction_interval, Optional(Eq(history_size())));
+ EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted),
+ Optional(Eq(history_size())));
}
// Checks that `Observe()` returns the prediction interval as soon as a true
@@ -527,7 +552,7 @@
}
// Observe a detection.
absl::optional<int> prediction_interval =
- evaluator.Observe(kDetected, kPredicted);
+ evaluator.Observe(kDetected, kNotPredicted);
EXPECT_TRUE(prediction_interval.has_value());
// `Observe()` does not return a prediction interval anymore during ongoing
// detections observed while a detection is still expected.
@@ -539,31 +564,36 @@
INSTANTIATE_TEST_SUITE_P(
ClippingPredictionEvalTest,
ClippingPredictorEvaluatorPredictionIntervalParameterization,
- ::testing::Combine(::testing::Values(0, 3, 5), ::testing::Values(7, 11)));
+ ::testing::Combine(::testing::Values(1, 3, 5), ::testing::Values(7, 11)));
-// Checks that, when a detection is expected, the expectation is removed if and
-// only if `Reset()` is called after a prediction is observed.
-TEST(ClippingPredictionEvalTest, NoFalsePositivesAfterReset) {
+// Checks that, when a detection is expected, the expectation is not removed
+// before the detection deadline expires unless `RemoveExpectations()` is
+// called.
+TEST(ClippingPredictionEvalTest, NoFalsePositivesAfterRemoveExpectations) {
constexpr int kHistorySize = 2;
- ClippingPredictorEvaluator with_reset(kHistorySize);
- with_reset.Observe(kNotDetected, kPredicted);
- with_reset.Reset();
- with_reset.Observe(kNotDetected, kNotPredicted);
- with_reset.Observe(kNotDetected, kNotPredicted);
- EXPECT_EQ(with_reset.counters().true_positives, 0);
- EXPECT_EQ(with_reset.counters().true_negatives, 2);
- EXPECT_EQ(with_reset.counters().false_positives, 0);
- EXPECT_EQ(with_reset.counters().false_negatives, 0);
+ // Case 1: `RemoveExpectations()` is NOT called.
+ ClippingPredictorEvaluator e1(kHistorySize);
+ e1.Observe(kNotDetected, kPredicted);
+ ASSERT_EQ(e1.counters().true_negatives, 1);
+ e1.Observe(kNotDetected, kNotPredicted);
+ e1.Observe(kNotDetected, kNotPredicted);
+ EXPECT_EQ(e1.counters().true_positives, 0);
+ EXPECT_EQ(e1.counters().true_negatives, 1);
+ EXPECT_EQ(e1.counters().false_positives, 1);
+ EXPECT_EQ(e1.counters().false_negatives, 0);
- ClippingPredictorEvaluator no_reset(kHistorySize);
- no_reset.Observe(kNotDetected, kPredicted);
- no_reset.Observe(kNotDetected, kNotPredicted);
- no_reset.Observe(kNotDetected, kNotPredicted);
- EXPECT_EQ(no_reset.counters().true_positives, 0);
- EXPECT_EQ(no_reset.counters().true_negatives, 0);
- EXPECT_EQ(no_reset.counters().false_positives, 1);
- EXPECT_EQ(no_reset.counters().false_negatives, 0);
+ // Case 2: `RemoveExpectations()` is called.
+ ClippingPredictorEvaluator e2(kHistorySize);
+ e2.Observe(kNotDetected, kPredicted);
+ ASSERT_EQ(e2.counters().true_negatives, 1);
+ e2.RemoveExpectations();
+ e2.Observe(kNotDetected, kNotPredicted);
+ e2.Observe(kNotDetected, kNotPredicted);
+ EXPECT_EQ(e2.counters().true_positives, 0);
+ EXPECT_EQ(e2.counters().true_negatives, 3);
+ EXPECT_EQ(e2.counters().false_positives, 0);
+ EXPECT_EQ(e2.counters().false_negatives, 0);
}
class ComputeClippingPredictionMetricsParameterization
