diff --git a/modules/audio_processing/aec3/BUILD.gn b/modules/audio_processing/aec3/BUILD.gn index b5ebff74a0..684f192cc5 100644 --- a/modules/audio_processing/aec3/BUILD.gn +++ b/modules/audio_processing/aec3/BUILD.gn @@ -31,6 +31,8 @@ rtc_static_library("aec3") { "block_processor_metrics.h", "cascaded_biquad_filter.cc", "cascaded_biquad_filter.h", + "clockdrift_detector.cc", + "clockdrift_detector.h", "comfort_noise_generator.cc", "comfort_noise_generator.h", "decimator.cc", @@ -195,6 +197,7 @@ if (rtc_include_tests) { "block_processor_metrics_unittest.cc", "block_processor_unittest.cc", "cascaded_biquad_filter_unittest.cc", + "clockdrift_detector_unittest.cc", "comfort_noise_generator_unittest.cc", "decimator_unittest.cc", "echo_canceller3_unittest.cc", diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index 590380f897..ef25e7c23b 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -194,6 +194,8 @@ void BlockProcessorImpl::ProcessCapture( } } + echo_path_variability.clock_drift = delay_controller_->HasClockdrift(); + // Remove the echo from the capture signal. echo_remover_->ProcessCapture( echo_path_variability, capture_signal_saturation, estimated_delay_, diff --git a/modules/audio_processing/aec3/block_processor2.cc b/modules/audio_processing/aec3/block_processor2.cc index 3616427ce2..30bd3ee5ca 100644 --- a/modules/audio_processing/aec3/block_processor2.cc +++ b/modules/audio_processing/aec3/block_processor2.cc @@ -166,6 +166,8 @@ void BlockProcessorImpl2::ProcessCapture( } } + echo_path_variability.clock_drift = delay_controller_->HasClockdrift(); + // Remove the echo from the capture signal. echo_remover_->ProcessCapture( echo_path_variability, capture_signal_saturation, estimated_delay_, diff --git a/modules/audio_processing/aec3/clockdrift_detector.cc b/modules/audio_processing/aec3/clockdrift_detector.cc new file mode 100644 index 0000000000..2c49b795c4 --- /dev/null +++ b/modules/audio_processing/aec3/clockdrift_detector.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/clockdrift_detector.h" + +namespace webrtc { + +ClockdriftDetector::ClockdriftDetector() + : level_(Level::kNone), stability_counter_(0) { + delay_history_.fill(0); +} + +ClockdriftDetector::~ClockdriftDetector() = default; + +void ClockdriftDetector::Update(int delay_estimate) { + if (delay_estimate == delay_history_[0]) { + // Reset clockdrift level if delay estimate is stable for 7500 blocks (30 + // seconds). + if (++stability_counter_ > 7500) + level_ = Level::kNone; + return; + } + + stability_counter_ = 0; + const int d1 = delay_history_[0] - delay_estimate; + const int d2 = delay_history_[1] - delay_estimate; + const int d3 = delay_history_[2] - delay_estimate; + + // Patterns recognized as positive clockdrift: + // [x-3], x-2, x-1, x. + // [x-3], x-1, x-2, x. + const bool probable_drift_up = + (d1 == -1 && d2 == -2) || (d1 == -2 && d2 == -1); + const bool drift_up = probable_drift_up && d3 == -3; + + // Patterns recognized as negative clockdrift: + // [x+3], x+2, x+1, x. + // [x+3], x+1, x+2, x. + const bool probable_drift_down = (d1 == 1 && d2 == 2) || (d1 == 2 && d2 == 1); + const bool drift_down = probable_drift_down && d3 == 3; + + // Set clockdrift level. + if (drift_up || drift_down) { + level_ = Level::kVerified; + } else if ((probable_drift_up || probable_drift_down) && + level_ == Level::kNone) { + level_ = Level::kProbable; + } + + // Shift delay history one step. + delay_history_[2] = delay_history_[1]; + delay_history_[1] = delay_history_[0]; + delay_history_[0] = delay_estimate; +} +} // namespace webrtc diff --git a/modules/audio_processing/aec3/clockdrift_detector.h b/modules/audio_processing/aec3/clockdrift_detector.h new file mode 100644 index 0000000000..22528c9489 --- /dev/null +++ b/modules/audio_processing/aec3/clockdrift_detector.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_ + +#include + +namespace webrtc { + +class ApmDataDumper; +struct DownsampledRenderBuffer; +struct EchoCanceller3Config; + +// Detects clockdrift by analyzing the estimated delay. +class ClockdriftDetector { + public: + enum class Level { kNone, kProbable, kVerified, kNumCategories }; + ClockdriftDetector(); + ~ClockdriftDetector(); + void Update(int delay_estimate); + Level ClockdriftLevel() const { return level_; } + + private: + std::array delay_history_; + Level level_; + size_t stability_counter_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_ diff --git a/modules/audio_processing/aec3/clockdrift_detector_unittest.cc b/modules/audio_processing/aec3/clockdrift_detector_unittest.cc new file mode 100644 index 0000000000..0f98b01d3a --- /dev/null +++ b/modules/audio_processing/aec3/clockdrift_detector_unittest.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/clockdrift_detector.h" + +#include "test/gtest.h" + +namespace webrtc { +TEST(ClockdriftDetector, ClockdriftDetector) { + ClockdriftDetector c; + // No clockdrift at start. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone); + + // Monotonically increasing delay. + for (int i = 0; i < 100; i++) + c.Update(1000); + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone); + for (int i = 0; i < 100; i++) + c.Update(1001); + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone); + for (int i = 0; i < 100; i++) + c.Update(1002); + // Probable clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kProbable); + for (int i = 0; i < 100; i++) + c.Update(1003); + // Verified clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kVerified); + + // Stable delay. + for (int i = 0; i < 10000; i++) + c.Update(1003); + // No clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kNone); + + // Decreasing delay. + for (int i = 0; i < 100; i++) + c.Update(1001); + for (int i = 0; i < 100; i++) + c.Update(999); + // Probable clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kProbable); + for (int i = 0; i < 100; i++) + c.Update(1000); + for (int i = 0; i < 100; i++) + c.Update(998); + // Verified clockdrift. + EXPECT_TRUE(c.ClockdriftLevel() == ClockdriftDetector::Level::kVerified); +} +} // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/modules/audio_processing/aec3/echo_path_delay_estimator.cc index 5c838aed8a..6069ed6be6 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.cc +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.cc @@ -73,6 +73,12 @@ absl::optional EchoPathDelayEstimator::EstimateDelay( matched_filter_lag_aggregator_.Aggregate( matched_filter_.GetLagEstimates()); + // Run clockdrift detection. + if (aggregated_matched_filter_lag && + (*aggregated_matched_filter_lag).quality == + DelayEstimate::Quality::kRefined) + clockdrift_detector_.Update((*aggregated_matched_filter_lag).delay); + // TODO(peah): Move this logging outside of this class once EchoCanceller3 // development is done. data_dumper_->DumpRaw( @@ -112,5 +118,4 @@ void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator, old_aggregated_lag_ = absl::nullopt; consistent_estimate_counter_ = 0; } - } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.h b/modules/audio_processing/aec3/echo_path_delay_estimator.h index 060c8753bd..1f14735d1a 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.h +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.h @@ -15,6 +15,7 @@ #include "absl/types/optional.h" #include "api/array_view.h" +#include "modules/audio_processing/aec3/clockdrift_detector.h" #include "modules/audio_processing/aec3/decimator.h" #include "modules/audio_processing/aec3/delay_estimate.h" #include "modules/audio_processing/aec3/matched_filter.h" @@ -49,6 +50,11 @@ class EchoPathDelayEstimator { down_sampling_factor_); } + // Returns the level of detected clockdrift. + ClockdriftDetector::Level Clockdrift() const { + return clockdrift_detector_.ClockdriftLevel(); + } + private: ApmDataDumper* const data_dumper_; const size_t down_sampling_factor_; @@ -58,6 +64,7 @@ class EchoPathDelayEstimator { MatchedFilterLagAggregator matched_filter_lag_aggregator_; absl::optional old_aggregated_lag_; size_t consistent_estimate_counter_ = 0; + ClockdriftDetector clockdrift_detector_; // Internal reset method with more granularity. void Reset(bool reset_lag_aggregator, bool reset_delay_confidence); diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h index 5520f764fb..5f652e192f 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h @@ -33,6 +33,7 @@ class MockRenderDelayController : public RenderDelayController { size_t render_delay_buffer_delay, const absl::optional& echo_remover_delay, rtc::ArrayView capture)); + MOCK_CONST_METHOD0(HasClockdrift, bool()); }; } // namespace test diff --git a/modules/audio_processing/aec3/render_delay_controller.cc b/modules/audio_processing/aec3/render_delay_controller.cc index 36e75d9fa9..c4665eaa22 100644 --- a/modules/audio_processing/aec3/render_delay_controller.cc +++ b/modules/audio_processing/aec3/render_delay_controller.cc @@ -64,6 +64,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { size_t render_delay_buffer_delay, const absl::optional& echo_remover_delay, rtc::ArrayView capture) override; + bool HasClockdrift() const override; private: static int instance_count_; @@ -285,7 +286,8 @@ absl::optional RenderDelayControllerImpl::GetDelay( metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) : absl::nullopt, - delay_ ? delay_->delay : 0, skew_shift); + delay_ ? delay_->delay : 0, skew_shift, + delay_estimator_.Clockdrift()); data_dumper_->DumpRaw("aec3_render_delay_controller_delay", delay_samples ? delay_samples->delay : 0); @@ -301,6 +303,10 @@ absl::optional RenderDelayControllerImpl::GetDelay( return delay_; } +bool RenderDelayControllerImpl::HasClockdrift() const { + return delay_estimator_.Clockdrift() != ClockdriftDetector::Level::kNone; +} + } // namespace RenderDelayController* RenderDelayController::Create( diff --git a/modules/audio_processing/aec3/render_delay_controller.h b/modules/audio_processing/aec3/render_delay_controller.h index 41ba4224c9..b46ed89252 100644 --- a/modules/audio_processing/aec3/render_delay_controller.h +++ b/modules/audio_processing/aec3/render_delay_controller.h @@ -44,6 +44,9 @@ class RenderDelayController { size_t render_delay_buffer_delay, const absl::optional& echo_remover_delay, rtc::ArrayView capture) = 0; + + // Returns true if clockdrift has been detected. + virtual bool HasClockdrift() const = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/render_delay_controller2.cc b/modules/audio_processing/aec3/render_delay_controller2.cc index e27d5f3d60..00daf8f2af 100644 --- a/modules/audio_processing/aec3/render_delay_controller2.cc +++ b/modules/audio_processing/aec3/render_delay_controller2.cc @@ -46,6 +46,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController { size_t render_delay_buffer_delay, const absl::optional& echo_remover_delay, rtc::ArrayView capture) override; + bool HasClockdrift() const override; private: static int instance_count_; @@ -192,7 +193,7 @@ absl::optional RenderDelayControllerImpl2::GetDelay( metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) : absl::nullopt, - delay_ ? delay_->delay : 0, 0); + delay_ ? delay_->delay : 0, 0, delay_estimator_.Clockdrift()); data_dumper_->DumpRaw("aec3_render_delay_controller_delay", delay_samples ? delay_samples->delay : 0); @@ -202,6 +203,10 @@ absl::optional RenderDelayControllerImpl2::GetDelay( return delay_; } +bool RenderDelayControllerImpl2::HasClockdrift() const { + return delay_estimator_.Clockdrift() != ClockdriftDetector::Level::kNone; +} + } // namespace RenderDelayController* RenderDelayController::Create2( diff --git a/modules/audio_processing/aec3/render_delay_controller_metrics.cc b/modules/audio_processing/aec3/render_delay_controller_metrics.cc index c51d4688a7..582e033482 100644 --- a/modules/audio_processing/aec3/render_delay_controller_metrics.cc +++ b/modules/audio_processing/aec3/render_delay_controller_metrics.cc @@ -46,7 +46,8 @@ RenderDelayControllerMetrics::RenderDelayControllerMetrics() = default; void RenderDelayControllerMetrics::Update( absl::optional delay_samples, size_t buffer_delay_blocks, - absl::optional skew_shift_blocks) { + absl::optional skew_shift_blocks, + ClockdriftDetector::Level clockdrift) { ++call_counter_; if (!initial_update) { @@ -115,6 +116,10 @@ void RenderDelayControllerMetrics::Update( static_cast(delay_changes), static_cast(DelayChangesCategory::kNumCategories)); + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.EchoCanceller.Clockdrift", static_cast(clockdrift), + static_cast(ClockdriftDetector::Level::kNumCategories)); + metrics_reported_ = true; call_counter_ = 0; ResetMetrics(); diff --git a/modules/audio_processing/aec3/render_delay_controller_metrics.h b/modules/audio_processing/aec3/render_delay_controller_metrics.h index 50e60bbd9c..22cc202e73 100644 --- a/modules/audio_processing/aec3/render_delay_controller_metrics.h +++ b/modules/audio_processing/aec3/render_delay_controller_metrics.h @@ -14,6 +14,7 @@ #include #include "absl/types/optional.h" +#include "modules/audio_processing/aec3/clockdrift_detector.h" #include "rtc_base/constructormagic.h" namespace webrtc { @@ -26,7 +27,8 @@ class RenderDelayControllerMetrics { // Updates the metric with new data. void Update(absl::optional delay_samples, size_t buffer_delay_blocks, - absl::optional skew_shift_blocks); + absl::optional skew_shift_blocks, + ClockdriftDetector::Level clockdrift); // Returns true if the metrics have just been reported, otherwise false. bool MetricsReported() { return metrics_reported_; } diff --git a/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc b/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc index e867de4df4..216b0e220d 100644 --- a/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc +++ b/modules/audio_processing/aec3/render_delay_controller_metrics_unittest.cc @@ -22,10 +22,12 @@ TEST(RenderDelayControllerMetrics, NormalUsage) { for (int j = 0; j < 3; ++j) { for (int k = 0; k < kMetricsReportingIntervalBlocks - 1; ++k) { - metrics.Update(absl::nullopt, 0, absl::nullopt); + metrics.Update(absl::nullopt, 0, absl::nullopt, + ClockdriftDetector::Level::kNone); EXPECT_FALSE(metrics.MetricsReported()); } - metrics.Update(absl::nullopt, 0, absl::nullopt); + metrics.Update(absl::nullopt, 0, absl::nullopt, + ClockdriftDetector::Level::kNone); EXPECT_TRUE(metrics.MetricsReported()); } }