AEC3: Clockdrift detection

This change introduces a clockdrift detector operating on the estimated
delay of the echo path delay estimator. Each time the delay estimate
changes it is compared to previous estimates. If the estimates are
slowly increasing or decreasing, clockdrift is detected.

Four different patterns are considered clockdrift:
- k, k+1, k+2, k+3
- k, k+2, k+1, k+3
- k, k-1, k-2, k-3
- k, k-2, k-1, k-3

A delay estimate history matching the three last elements in one of the
patterns is considered probable clockdrift. Matching all four elements
is considered verified clockdrift.

If the delay is constant for some time after clockdrift is detected the
clockdrift detector will revert to no detected clockdrift.

The level of clockdrift is reported via an UMA histogram.

Bug: webrtc:10014
Change-Id: I1cce4d593e101a8b3fa99df6935e59b4243cb97a
Reviewed-on: https://webrtc-review.googlesource.com/c/111381
Commit-Queue: Gustaf Ullberg <gustaf@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25758}

modules/audio_processing/aec3/BUILD.gn

@@ -31,6 +31,8 @@ rtc_static_library("aec3") {
     "block_processor_metrics.h",
     "cascaded_biquad_filter.cc",
     "cascaded_biquad_filter.h",
+    "clockdrift_detector.cc",
+    "clockdrift_detector.h",
     "comfort_noise_generator.cc",
     "comfort_noise_generator.h",
     "decimator.cc",
@@ -195,6 +197,7 @@ if (rtc_include_tests) {
         "block_processor_metrics_unittest.cc",
         "block_processor_unittest.cc",
         "cascaded_biquad_filter_unittest.cc",
+        "clockdrift_detector_unittest.cc",
         "comfort_noise_generator_unittest.cc",
         "decimator_unittest.cc",
         "echo_canceller3_unittest.cc",

modules/audio_processing/aec3/block_processor.cc

@@ -194,6 +194,8 @@ void BlockProcessorImpl::ProcessCapture(
     }
   }
 
+  echo_path_variability.clock_drift = delay_controller_->HasClockdrift();
+
   // Remove the echo from the capture signal.
   echo_remover_->ProcessCapture(
       echo_path_variability, capture_signal_saturation, estimated_delay_,

modules/audio_processing/aec3/block_processor2.cc

@@ -166,6 +166,8 @@ void BlockProcessorImpl2::ProcessCapture(
     }
   }
 
+  echo_path_variability.clock_drift = delay_controller_->HasClockdrift();
+
   // Remove the echo from the capture signal.
   echo_remover_->ProcessCapture(
       echo_path_variability, capture_signal_saturation, estimated_delay_,

modules/audio_processing/aec3/clockdrift_detector.cc

@@ -0,0 +1,61 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/audio_processing/aec3/clockdrift_detector.h"
+
+namespace webrtc {
+
+ClockdriftDetector::ClockdriftDetector()
+    : level_(Level::kNone), stability_counter_(0) {
+  delay_history_.fill(0);
+}
+
+ClockdriftDetector::~ClockdriftDetector() = default;
+
+void ClockdriftDetector::Update(int delay_estimate) {
+  if (delay_estimate == delay_history_[0]) {
+    // Reset clockdrift level if delay estimate is stable for 7500 blocks (30
+    // seconds).
+    if (++stability_counter_ > 7500)
+      level_ = Level::kNone;
+    return;
+  }
+
+  stability_counter_ = 0;
+  const int d1 = delay_history_[0] - delay_estimate;
+  const int d2 = delay_history_[1] - delay_estimate;
+  const int d3 = delay_history_[2] - delay_estimate;
+
+  // Patterns recognized as positive clockdrift:
+  // [x-3], x-2, x-1, x.
+  // [x-3], x-1, x-2, x.
+  const bool probable_drift_up =
+      (d1 == -1 && d2 == -2) || (d1 == -2 && d2 == -1);
+  const bool drift_up = probable_drift_up && d3 == -3;
+
+  // Patterns recognized as negative clockdrift:
+  // [x+3], x+2, x+1, x.
+  // [x+3], x+1, x+2, x.
+  const bool probable_drift_down = (d1 == 1 && d2 == 2) || (d1 == 2 && d2 == 1);
+  const bool drift_down = probable_drift_down && d3 == 3;
+
+  // Set clockdrift level.
+  if (drift_up || drift_down) {
+    level_ = Level::kVerified;
+  } else if ((probable_drift_up || probable_drift_down) &&
+             level_ == Level::kNone) {
+    level_ = Level::kProbable;
+  }
+
+  // Shift delay history one step.
+  delay_history_[2] = delay_history_[1];
+  delay_history_[1] = delay_history_[0];
+  delay_history_[0] = delay_estimate;
+}
+}  // namespace webrtc

modules/audio_processing/aec3/clockdrift_detector.h

@@ -0,0 +1,38 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_
+#define MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_
+
+#include <array>
+
+namespace webrtc {
+
+class ApmDataDumper;
+struct DownsampledRenderBuffer;
+struct EchoCanceller3Config;
+
+// Detects clockdrift by analyzing the estimated delay.
+class ClockdriftDetector {
+ public:
+  enum class Level { kNone, kProbable, kVerified, kNumCategories };
+  ClockdriftDetector();
+  ~ClockdriftDetector();
+  void Update(int delay_estimate);
+  Level ClockdriftLevel() const { return level_; }
+
+ private:
+  std::array<int, 3> delay_history_;
+  Level level_;
+  size_t stability_counter_;
+};
+}  // namespace webrtc
+
+#endif  // MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_

modules/audio_processing/aec3/clockdrift_detector_unittest.cc

@@ -0,0 +1,57 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/aec3/clockdrift_detector.h"
+
+#include "test/gtest.h"
+
+namespace webrtc {
+TEST(ClockdriftDetector, ClockdriftDetector) {
+  ClockdriftDetector c;
+  // No clockdrift at start.
+  EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone);
+
+  // Monotonically increasing delay.
+  for (int i = 0; i < 100; i++)
+    c.Update(1000);
+  EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone);
+  for (int i = 0; i < 100; i++)
+    c.Update(1001);
+  EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone);
+  for (int i = 0; i < 100; i++)
+    c.Update(1002);
+  // Probable clockdrift.
+  EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kProbable);
+  for (int i = 0; i < 100; i++)
+    c.Update(1003);
+  // Verified clockdrift.
+  EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kVerified);
+
+  // Stable delay.
+  for (int i = 0; i < 10000; i++)
+    c.Update(1003);
+  // No clockdrift.
+  EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone);
+
+  // Decreasing delay.
+  for (int i = 0; i < 100; i++)
+    c.Update(1001);
+  for (int i = 0; i < 100; i++)
+    c.Update(999);
+  // Probable clockdrift.
+  EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kProbable);
+  for (int i = 0; i < 100; i++)
+    c.Update(1000);
+  for (int i = 0; i < 100; i++)
+    c.Update(998);
+  // Verified clockdrift.
+  EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kVerified);
+}
+}  // namespace webrtc

modules/audio_processing/aec3/echo_path_delay_estimator.cc

@@ -73,6 +73,12 @@ absl::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
       matched_filter_lag_aggregator_.Aggregate(
           matched_filter_.GetLagEstimates());
 
+  // Run clockdrift detection.
+  if (aggregated_matched_filter_lag &&
+      (*aggregated_matched_filter_lag).quality ==
+          DelayEstimate::Quality::kRefined)
+    clockdrift_detector_.Update((*aggregated_matched_filter_lag).delay);
+
   // TODO(peah): Move this logging outside of this class once EchoCanceller3
   // development is done.
   data_dumper_->DumpRaw(
@@ -112,5 +118,4 @@ void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator,
   old_aggregated_lag_ = absl::nullopt;
   consistent_estimate_counter_ = 0;
 }
-
 }  // namespace webrtc

modules/audio_processing/aec3/echo_path_delay_estimator.h

@@ -15,6 +15,7 @@
 
 #include "absl/types/optional.h"
 #include "api/array_view.h"
+#include "modules/audio_processing/aec3/clockdrift_detector.h"
 #include "modules/audio_processing/aec3/decimator.h"
 #include "modules/audio_processing/aec3/delay_estimate.h"
 #include "modules/audio_processing/aec3/matched_filter.h"
@@ -49,6 +50,11 @@ class EchoPathDelayEstimator {
                                         down_sampling_factor_);
   }
 
+  // Returns the level of detected clockdrift.
+  ClockdriftDetector::Level Clockdrift() const {
+    return clockdrift_detector_.ClockdriftLevel();
+  }
+
  private:
   ApmDataDumper* const data_dumper_;
   const size_t down_sampling_factor_;
@@ -58,6 +64,7 @@ class EchoPathDelayEstimator {
   MatchedFilterLagAggregator matched_filter_lag_aggregator_;
   absl::optional<DelayEstimate> old_aggregated_lag_;
   size_t consistent_estimate_counter_ = 0;
+  ClockdriftDetector clockdrift_detector_;
 
   // Internal reset method with more granularity.
   void Reset(bool reset_lag_aggregator, bool reset_delay_confidence);

modules/audio_processing/aec3/mock/mock_render_delay_controller.h

@@ -33,6 +33,7 @@ class MockRenderDelayController : public RenderDelayController {
                    size_t render_delay_buffer_delay,
                    const absl::optional<int>& echo_remover_delay,
                    rtc::ArrayView<const float> capture));
+  MOCK_CONST_METHOD0(HasClockdrift, bool());
 };
 
 }  // namespace test

modules/audio_processing/aec3/render_delay_controller.cc

@@ -64,6 +64,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
       size_t render_delay_buffer_delay,
       const absl::optional<int>& echo_remover_delay,
       rtc::ArrayView<const float> capture) override;
+  bool HasClockdrift() const override;
 
  private:
   static int instance_count_;
@@ -285,7 +286,8 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
 
   metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)
                                  : absl::nullopt,
-                  delay_ ? delay_->delay : 0, skew_shift);
+                  delay_ ? delay_->delay : 0, skew_shift,
+                  delay_estimator_.Clockdrift());
 
   data_dumper_->DumpRaw("aec3_render_delay_controller_delay",
                         delay_samples ? delay_samples->delay : 0);
@@ -301,6 +303,10 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
   return delay_;
 }
 
+bool RenderDelayControllerImpl::HasClockdrift() const {
+  return delay_estimator_.Clockdrift() != ClockdriftDetector::Level::kNone;
+}
+
 }  // namespace
 
 RenderDelayController* RenderDelayController::Create(

modules/audio_processing/aec3/render_delay_controller.h

@@ -44,6 +44,9 @@ class RenderDelayController {
       size_t render_delay_buffer_delay,
       const absl::optional<int>& echo_remover_delay,
       rtc::ArrayView<const float> capture) = 0;
+
+  // Returns true if clockdrift has been detected.
+  virtual bool HasClockdrift() const = 0;
 };
 }  // namespace webrtc
 

modules/audio_processing/aec3/render_delay_controller2.cc

@@ -46,6 +46,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController {
       size_t render_delay_buffer_delay,
       const absl::optional<int>& echo_remover_delay,
       rtc::ArrayView<const float> capture) override;
+  bool HasClockdrift() const override;
 
  private:
   static int instance_count_;
@@ -192,7 +193,7 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl2::GetDelay(
 
   metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)
                                  : absl::nullopt,
-                  delay_ ? delay_->delay : 0, 0);
+                  delay_ ? delay_->delay : 0, 0, delay_estimator_.Clockdrift());
 
   data_dumper_->DumpRaw("aec3_render_delay_controller_delay",
                         delay_samples ? delay_samples->delay : 0);
@@ -202,6 +203,10 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl2::GetDelay(
   return delay_;
 }
 
+bool RenderDelayControllerImpl2::HasClockdrift() const {
+  return delay_estimator_.Clockdrift() != ClockdriftDetector::Level::kNone;
+}
+
 }  // namespace
 
 RenderDelayController* RenderDelayController::Create2(

modules/audio_processing/aec3/render_delay_controller_metrics.cc

@@ -46,7 +46,8 @@ RenderDelayControllerMetrics::RenderDelayControllerMetrics() = default;
 void RenderDelayControllerMetrics::Update(
     absl::optional<size_t> delay_samples,
     size_t buffer_delay_blocks,
-    absl::optional<int> skew_shift_blocks) {
+    absl::optional<int> skew_shift_blocks,
+    ClockdriftDetector::Level clockdrift) {
   ++call_counter_;
 
   if (!initial_update) {
@@ -115,6 +116,10 @@ void RenderDelayControllerMetrics::Update(
         static_cast<int>(delay_changes),
         static_cast<int>(DelayChangesCategory::kNumCategories));
 
+    RTC_HISTOGRAM_ENUMERATION(
+        "WebRTC.Audio.EchoCanceller.Clockdrift", static_cast<int>(clockdrift),
+        static_cast<int>(ClockdriftDetector::Level::kNumCategories));
+
     metrics_reported_ = true;
     call_counter_ = 0;
     ResetMetrics();

modules/audio_processing/aec3/render_delay_controller_metrics.h

@@ -14,6 +14,7 @@
 #include <stddef.h>
 
 #include "absl/types/optional.h"
+#include "modules/audio_processing/aec3/clockdrift_detector.h"
 #include "rtc_base/constructormagic.h"
 
 namespace webrtc {
@@ -26,7 +27,8 @@ class RenderDelayControllerMetrics {
   // Updates the metric with new data.
   void Update(absl::optional<size_t> delay_samples,
               size_t buffer_delay_blocks,
-              absl::optional<int> skew_shift_blocks);
+              absl::optional<int> skew_shift_blocks,
+              ClockdriftDetector::Level clockdrift);
 
   // Returns true if the metrics have just been reported, otherwise false.
   bool MetricsReported() { return metrics_reported_; }

modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc

@@ -22,10 +22,12 @@ TEST(RenderDelayControllerMetrics, NormalUsage) {
 
   for (int j = 0; j < 3; ++j) {
     for (int k = 0; k < kMetricsReportingIntervalBlocks - 1; ++k) {
-      metrics.Update(absl::nullopt, 0, absl::nullopt);
+      metrics.Update(absl::nullopt, 0, absl::nullopt,
+                     ClockdriftDetector::Level::kNone);
       EXPECT_FALSE(metrics.MetricsReported());
     }
-    metrics.Update(absl::nullopt, 0, absl::nullopt);
+    metrics.Update(absl::nullopt, 0, absl::nullopt,
+                   ClockdriftDetector::Level::kNone);
     EXPECT_TRUE(metrics.MetricsReported());
   }
 }