From 777cf263285bd8d38ab78390d19a0d9f18398f7a Mon Sep 17 00:00:00 2001 From: Gustaf Ullberg Date: Thu, 22 Nov 2018 16:02:34 +0100 Subject: [PATCH] AEC3: Clockdrift detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change introduces a clockdrift detector operating on the estimated delay of the echo path delay estimator. Each time the delay estimate changes it is compared to previous estimates. If the estimates are slowly increasing or decreasing, clockdrift is detected. Four different patterns are considered clockdrift: - k, k+1, k+2, k+3 - k, k+2, k+1, k+3 - k, k-1, k-2, k-3 - k, k-2, k-1, k-3 A delay estimate history matching the three last elements in one of the patterns is considered probable clockdrift. Matching all four elements is considered verified clockdrift. If the delay is constant for some time after clockdrift is detected the clockdrift detector will revert to no detected clockdrift. The level of clockdrift is reported via an UMA histogram. Bug: webrtc:10014 Change-Id: I1cce4d593e101a8b3fa99df6935e59b4243cb97a Reviewed-on: https://webrtc-review.googlesource.com/c/111381 Commit-Queue: Gustaf Ullberg Reviewed-by: Per Åhgren Cr-Commit-Position: refs/heads/master@{#25758} --- modules/audio_processing/aec3/BUILD.gn | 3 + .../audio_processing/aec3/block_processor.cc | 2 + .../audio_processing/aec3/block_processor2.cc | 2 + .../aec3/clockdrift_detector.cc | 61 +++++++++++++++++++ .../aec3/clockdrift_detector.h | 38 ++++++++++++ .../aec3/clockdrift_detector_unittest.cc | 57 +++++++++++++++++ .../aec3/echo_path_delay_estimator.cc | 7 ++- .../aec3/echo_path_delay_estimator.h | 7 +++ .../aec3/mock/mock_render_delay_controller.h | 1 + .../aec3/render_delay_controller.cc | 8 ++- .../aec3/render_delay_controller.h | 3 + .../aec3/render_delay_controller2.cc | 7 ++- .../aec3/render_delay_controller_metrics.cc | 7 ++- .../aec3/render_delay_controller_metrics.h | 4 +- ...ender_delay_controller_metrics_unittest.cc | 6 +- 15 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 modules/audio_processing/aec3/clockdrift_detector.cc create mode 100644 modules/audio_processing/aec3/clockdrift_detector.h create mode 100644 modules/audio_processing/aec3/clockdrift_detector_unittest.cc diff --git a/modules/audio_processing/aec3/BUILD.gn b/modules/audio_processing/aec3/BUILD.gn index b5ebff74a0..684f192cc5 100644 --- a/modules/audio_processing/aec3/BUILD.gn +++ b/modules/audio_processing/aec3/BUILD.gn @@ -31,6 +31,8 @@ rtc_static_library("aec3") { "block_processor_metrics.h", "cascaded_biquad_filter.cc", "cascaded_biquad_filter.h", + "clockdrift_detector.cc", + "clockdrift_detector.h", "comfort_noise_generator.cc", "comfort_noise_generator.h", "decimator.cc", @@ -195,6 +197,7 @@ if (rtc_include_tests) { "block_processor_metrics_unittest.cc", "block_processor_unittest.cc", "cascaded_biquad_filter_unittest.cc", + "clockdrift_detector_unittest.cc", "comfort_noise_generator_unittest.cc", "decimator_unittest.cc", "echo_canceller3_unittest.cc", diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index 590380f897..ef25e7c23b 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -194,6 +194,8 @@ void BlockProcessorImpl::ProcessCapture( } } + echo_path_variability.clock_drift = delay_controller_->HasClockdrift(); + // Remove the echo from the capture signal. echo_remover_->ProcessCapture( echo_path_variability, capture_signal_saturation, estimated_delay_, diff --git a/modules/audio_processing/aec3/block_processor2.cc b/modules/audio_processing/aec3/block_processor2.cc index 3616427ce2..30bd3ee5ca 100644 --- a/modules/audio_processing/aec3/block_processor2.cc +++ b/modules/audio_processing/aec3/block_processor2.cc @@ -166,6 +166,8 @@ void BlockProcessorImpl2::ProcessCapture( } } + echo_path_variability.clock_drift = delay_controller_->HasClockdrift(); + // Remove the echo from the capture signal. echo_remover_->ProcessCapture( echo_path_variability, capture_signal_saturation, estimated_delay_, diff --git a/modules/audio_processing/aec3/clockdrift_detector.cc b/modules/audio_processing/aec3/clockdrift_detector.cc new file mode 100644 index 0000000000..2c49b795c4 --- /dev/null +++ b/modules/audio_processing/aec3/clockdrift_detector.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/clockdrift_detector.h" + +namespace webrtc { + +ClockdriftDetector::ClockdriftDetector() + : level_(Level::kNone), stability_counter_(0) { + delay_history_.fill(0); +} + +ClockdriftDetector::~ClockdriftDetector() = default; + +void ClockdriftDetector::Update(int delay_estimate) { + if (delay_estimate == delay_history_[0]) { + // Reset clockdrift level if delay estimate is stable for 7500 blocks (30 + // seconds). + if (++stability_counter_ > 7500) + level_ = Level::kNone; + return; + } + + stability_counter_ = 0; + const int d1 = delay_history_[0] - delay_estimate; + const int d2 = delay_history_[1] - delay_estimate; + const int d3 = delay_history_[2] - delay_estimate; + + // Patterns recognized as positive clockdrift: + // [x-3], x-2, x-1, x. + // [x-3], x-1, x-2, x. + const bool probable_drift_up = + (d1 == -1 && d2 == -2) || (d1 == -2 && d2 == -1); + const bool drift_up = probable_drift_up && d3 == -3; + + // Patterns recognized as negative clockdrift: + // [x+3], x+2, x+1, x. + // [x+3], x+1, x+2, x. + const bool probable_drift_down = (d1 == 1 && d2 == 2) || (d1 == 2 && d2 == 1); + const bool drift_down = probable_drift_down && d3 == 3; + + // Set clockdrift level. + if (drift_up || drift_down) { + level_ = Level::kVerified; + } else if ((probable_drift_up || probable_drift_down) && + level_ == Level::kNone) { + level_ = Level::kProbable; + } + + // Shift delay history one step. + delay_history_[2] = delay_history_[1]; + delay_history_[1] = delay_history_[0]; + delay_history_[0] = delay_estimate; +} +} // namespace webrtc diff --git a/modules/audio_processing/aec3/clockdrift_detector.h b/modules/audio_processing/aec3/clockdrift_detector.h new file mode 100644 index 0000000000..22528c9489 --- /dev/null +++ b/modules/audio_processing/aec3/clockdrift_detector.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_ + +#include + +namespace webrtc { + +class ApmDataDumper; +struct DownsampledRenderBuffer; +struct EchoCanceller3Config; + +// Detects clockdrift by analyzing the estimated delay. +class ClockdriftDetector { + public: + enum class Level { kNone, kProbable, kVerified, kNumCategories }; + ClockdriftDetector(); + ~ClockdriftDetector(); + void Update(int delay_estimate); + Level ClockdriftLevel() const { return level_; } + + private: + std::array delay_history_; + Level level_; + size_t stability_counter_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_ diff --git a/modules/audio_processing/aec3/clockdrift_detector_unittest.cc b/modules/audio_processing/aec3/clockdrift_detector_unittest.cc new file mode 100644 index 0000000000..0f98b01d3a --- /dev/null +++ b/modules/audio_processing/aec3/clockdrift_detector_unittest.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/clockdrift_detector.h" + +#include "test/gtest.h" + +namespace webrtc { +TEST(ClockdriftDetector, ClockdriftDetector) { + ClockdriftDetector c; + // No clockdrift at start. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone); + + // Monotonically increasing delay. + for (int i = 0; i < 100; i++) + c.Update(1000); + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone); + for (int i = 0; i < 100; i++) + c.Update(1001); + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone); + for (int i = 0; i < 100; i++) + c.Update(1002); + // Probable clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kProbable); + for (int i = 0; i < 100; i++) + c.Update(1003); + // Verified clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kVerified); + + // Stable delay. + for (int i = 0; i < 10000; i++) + c.Update(1003); + // No clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone); + + // Decreasing delay. + for (int i = 0; i < 100; i++) + c.Update(1001); + for (int i = 0; i < 100; i++) + c.Update(999); + // Probable clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kProbable); + for (int i = 0; i < 100; i++) + c.Update(1000); + for (int i = 0; i < 100; i++) + c.Update(998); + // Verified clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kVerified); +} +} // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/modules/audio_processing/aec3/echo_path_delay_estimator.cc index 5c838aed8a..6069ed6be6 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.cc +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.cc @@ -73,6 +73,12 @@ absl::optional EchoPathDelayEstimator::EstimateDelay( matched_filter_lag_aggregator_.Aggregate( matched_filter_.GetLagEstimates()); + // Run clockdrift detection. + if (aggregated_matched_filter_lag && + (*aggregated_matched_filter_lag).quality == + DelayEstimate::Quality::kRefined) + clockdrift_detector_.Update((*aggregated_matched_filter_lag).delay); + // TODO(peah): Move this logging outside of this class once EchoCanceller3 // development is done. data_dumper_->DumpRaw( @@ -112,5 +118,4 @@ void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator, old_aggregated_lag_ = absl::nullopt; consistent_estimate_counter_ = 0; } - } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.h b/modules/audio_processing/aec3/echo_path_delay_estimator.h index 060c8753bd..1f14735d1a 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.h +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.h @@ -15,6 +15,7 @@ #include "absl/types/optional.h" #include "api/array_view.h" +#include "modules/audio_processing/aec3/clockdrift_detector.h" #include "modules/audio_processing/aec3/decimator.h" #include "modules/audio_processing/aec3/delay_estimate.h" #include "modules/audio_processing/aec3/matched_filter.h" @@ -49,6 +50,11 @@ class EchoPathDelayEstimator { down_sampling_factor_); } + // Returns the level of detected clockdrift. + ClockdriftDetector::Level Clockdrift() const { + return clockdrift_detector_.ClockdriftLevel(); + } + private: ApmDataDumper* const data_dumper_; const size_t down_sampling_factor_; @@ -58,6 +64,7 @@ class EchoPathDelayEstimator { MatchedFilterLagAggregator matched_filter_lag_aggregator_; absl::optional old_aggregated_lag_; size_t consistent_estimate_counter_ = 0; + ClockdriftDetector clockdrift_detector_; // Internal reset method with more granularity. void Reset(bool reset_lag_aggregator, bool reset_delay_confidence); diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h index 5520f764fb..5f652e192f 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h @@ -33,6 +33,7 @@ class MockRenderDelayController : public RenderDelayController { size_t render_delay_buffer_delay, const absl::optional& echo_remover_delay, rtc::ArrayView capture)); + MOCK_CONST_METHOD0(HasClockdrift, bool()); }; } // namespace test diff --git a/modules/audio_processing/aec3/render_delay_controller.cc b/modules/audio_processing/aec3/render_delay_controller.cc index 36e75d9fa9..c4665eaa22 100644 --- a/modules/audio_processing/aec3/render_delay_controller.cc +++ b/modules/audio_processing/aec3/render_delay_controller.cc @@ -64,6 +64,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { size_t render_delay_buffer_delay, const absl::optional& echo_remover_delay, rtc::ArrayView capture) override; + bool HasClockdrift() const override; private: static int instance_count_; @@ -285,7 +286,8 @@ absl::optional RenderDelayControllerImpl::GetDelay( metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) : absl::nullopt, - delay_ ? delay_->delay : 0, skew_shift); + delay_ ? delay_->delay : 0, skew_shift, + delay_estimator_.Clockdrift()); data_dumper_->DumpRaw("aec3_render_delay_controller_delay", delay_samples ? delay_samples->delay : 0); @@ -301,6 +303,10 @@ absl::optional RenderDelayControllerImpl::GetDelay( return delay_; } +bool RenderDelayControllerImpl::HasClockdrift() const { + return delay_estimator_.Clockdrift() != ClockdriftDetector::Level::kNone; +} + } // namespace RenderDelayController* RenderDelayController::Create( diff --git a/modules/audio_processing/aec3/render_delay_controller.h b/modules/audio_processing/aec3/render_delay_controller.h index 41ba4224c9..b46ed89252 100644 --- a/modules/audio_processing/aec3/render_delay_controller.h +++ b/modules/audio_processing/aec3/render_delay_controller.h @@ -44,6 +44,9 @@ class RenderDelayController { size_t render_delay_buffer_delay, const absl::optional& echo_remover_delay, rtc::ArrayView capture) = 0; + + // Returns true if clockdrift has been detected. + virtual bool HasClockdrift() const = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/render_delay_controller2.cc b/modules/audio_processing/aec3/render_delay_controller2.cc index e27d5f3d60..00daf8f2af 100644 --- a/modules/audio_processing/aec3/render_delay_controller2.cc +++ b/modules/audio_processing/aec3/render_delay_controller2.cc @@ -46,6 +46,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController { size_t render_delay_buffer_delay, const absl::optional& echo_remover_delay, rtc::ArrayView capture) override; + bool HasClockdrift() const override; private: static int instance_count_; @@ -192,7 +193,7 @@ absl::optional RenderDelayControllerImpl2::GetDelay( metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) : absl::nullopt, - delay_ ? delay_->delay : 0, 0); + delay_ ? delay_->delay : 0, 0, delay_estimator_.Clockdrift()); data_dumper_->DumpRaw("aec3_render_delay_controller_delay", delay_samples ? delay_samples->delay : 0); @@ -202,6 +203,10 @@ absl::optional RenderDelayControllerImpl2::GetDelay( return delay_; } +bool RenderDelayControllerImpl2::HasClockdrift() const { + return delay_estimator_.Clockdrift() != ClockdriftDetector::Level::kNone; +} + } // namespace RenderDelayController* RenderDelayController::Create2( diff --git a/modules/audio_processing/aec3/render_delay_controller_metrics.cc b/modules/audio_processing/aec3/render_delay_controller_metrics.cc index c51d4688a7..582e033482 100644 --- a/modules/audio_processing/aec3/render_delay_controller_metrics.cc +++ b/modules/audio_processing/aec3/render_delay_controller_metrics.cc @@ -46,7 +46,8 @@ RenderDelayControllerMetrics::RenderDelayControllerMetrics() = default; void RenderDelayControllerMetrics::Update( absl::optional delay_samples, size_t buffer_delay_blocks, - absl::optional skew_shift_blocks) { + absl::optional skew_shift_blocks, + ClockdriftDetector::Level clockdrift) { ++call_counter_; if (!initial_update) { @@ -115,6 +116,10 @@ void RenderDelayControllerMetrics::Update( static_cast(delay_changes), static_cast(DelayChangesCategory::kNumCategories)); + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.EchoCanceller.Clockdrift", static_cast(clockdrift), + static_cast(ClockdriftDetector::Level::kNumCategories)); + metrics_reported_ = true; call_counter_ = 0; ResetMetrics(); diff --git a/modules/audio_processing/aec3/render_delay_controller_metrics.h b/modules/audio_processing/aec3/render_delay_controller_metrics.h index 50e60bbd9c..22cc202e73 100644 --- a/modules/audio_processing/aec3/render_delay_controller_metrics.h +++ b/modules/audio_processing/aec3/render_delay_controller_metrics.h @@ -14,6 +14,7 @@ #include #include "absl/types/optional.h" +#include "modules/audio_processing/aec3/clockdrift_detector.h" #include "rtc_base/constructormagic.h" namespace webrtc { @@ -26,7 +27,8 @@ class RenderDelayControllerMetrics { // Updates the metric with new data. void Update(absl::optional delay_samples, size_t buffer_delay_blocks, - absl::optional skew_shift_blocks); + absl::optional skew_shift_blocks, + ClockdriftDetector::Level clockdrift); // Returns true if the metrics have just been reported, otherwise false. bool MetricsReported() { return metrics_reported_; } diff --git a/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc b/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc index e867de4df4..216b0e220d 100644 --- a/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc +++ b/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc @@ -22,10 +22,12 @@ TEST(RenderDelayControllerMetrics, NormalUsage) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < kMetricsReportingIntervalBlocks - 1; ++k) { - metrics.Update(absl::nullopt, 0, absl::nullopt); + metrics.Update(absl::nullopt, 0, absl::nullopt, + ClockdriftDetector::Level::kNone); EXPECT_FALSE(metrics.MetricsReported()); } - metrics.Update(absl::nullopt, 0, absl::nullopt); + metrics.Update(absl::nullopt, 0, absl::nullopt, + ClockdriftDetector::Level::kNone); EXPECT_TRUE(metrics.MetricsReported()); } }