diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index a7e388ea8f..b444e80c3a 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -589,11 +589,15 @@ if (rtc_include_tests) { "audio_processing/audio_processing_unittest.cc", "audio_processing/echo_cancellation_bit_exact_unittest.cc", "audio_processing/echo_control_mobile_unittest.cc", + "audio_processing/echo_detector/circular_buffer_unittest.cc", + "audio_processing/echo_detector/mean_variance_estimator_unittest.cc", + "audio_processing/echo_detector/normalized_covariance_estimator_unittest.cc", "audio_processing/gain_control_unittest.cc", "audio_processing/high_pass_filter_unittest.cc", "audio_processing/level_controller/level_controller_unittest.cc", "audio_processing/level_estimator_unittest.cc", "audio_processing/noise_suppression_unittest.cc", + "audio_processing/residual_echo_detector_unittest.cc", "audio_processing/test/bitexactness_tools.cc", "audio_processing/test/bitexactness_tools.h", "audio_processing/test/debug_dump_replayer.cc", diff --git a/webrtc/modules/audio_processing/BUILD.gn b/webrtc/modules/audio_processing/BUILD.gn index 8aded107be..b187db2ecd 100644 --- a/webrtc/modules/audio_processing/BUILD.gn +++ b/webrtc/modules/audio_processing/BUILD.gn @@ -61,6 +61,12 @@ rtc_static_library("audio_processing") { "echo_cancellation_impl.h", "echo_control_mobile_impl.cc", "echo_control_mobile_impl.h", + "echo_detector/circular_buffer.cc", + "echo_detector/circular_buffer.h", + "echo_detector/mean_variance_estimator.cc", + "echo_detector/mean_variance_estimator.h", + "echo_detector/normalized_covariance_estimator.cc", + "echo_detector/normalized_covariance_estimator.h", "gain_control_for_experimental_agc.cc", "gain_control_for_experimental_agc.h", "gain_control_impl.cc", diff --git a/webrtc/modules/audio_processing/audio_processing.gypi b/webrtc/modules/audio_processing/audio_processing.gypi index d25f9e578e..15dad4f979 100644 --- a/webrtc/modules/audio_processing/audio_processing.gypi +++ b/webrtc/modules/audio_processing/audio_processing.gypi @@ -73,6 +73,12 @@ 'echo_cancellation_impl.h', 'echo_control_mobile_impl.cc', 'echo_control_mobile_impl.h', + 'echo_detector/circular_buffer.cc', + 'echo_detector/circular_buffer.h', + 'echo_detector/mean_variance_estimator.cc', + 'echo_detector/mean_variance_estimator.h', + 'echo_detector/normalized_covariance_estimator.cc', + 'echo_detector/normalized_covariance_estimator.h', 'gain_control_for_experimental_agc.cc', 'gain_control_for_experimental_agc.h', 'gain_control_impl.cc', diff --git a/webrtc/modules/audio_processing/echo_detector/circular_buffer.cc b/webrtc/modules/audio_processing/echo_detector/circular_buffer.cc new file mode 100644 index 0000000000..c67be05416 --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/circular_buffer.cc @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/echo_detector/circular_buffer.h" + +#include + +#include "webrtc/base/checks.h" + +namespace webrtc { + +CircularBuffer::CircularBuffer(size_t size) : buffer_(size) {} +CircularBuffer::~CircularBuffer() = default; + +void CircularBuffer::Push(float value) { + buffer_[next_insertion_index_] = value; + ++next_insertion_index_; + next_insertion_index_ %= buffer_.size(); + RTC_DCHECK_LT(next_insertion_index_, buffer_.size()); + nr_elements_in_buffer_ = std::min(nr_elements_in_buffer_ + 1, buffer_.size()); + RTC_DCHECK_LE(nr_elements_in_buffer_, buffer_.size()); +} + +rtc::Optional CircularBuffer::Pop() { + if (nr_elements_in_buffer_ == 0) { + return rtc::Optional(); + } + const size_t index = + (buffer_.size() + next_insertion_index_ - nr_elements_in_buffer_) % + buffer_.size(); + RTC_DCHECK_LT(index, buffer_.size()); + --nr_elements_in_buffer_; + return rtc::Optional(buffer_[index]); +} + +void CircularBuffer::Clear() { + std::fill(buffer_.begin(), buffer_.end(), 0.f); + next_insertion_index_ = 0; + nr_elements_in_buffer_ = 0; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_detector/circular_buffer.h b/webrtc/modules/audio_processing/echo_detector/circular_buffer.h new file mode 100644 index 0000000000..667b1f8824 --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/circular_buffer.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_CIRCULAR_BUFFER_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_CIRCULAR_BUFFER_H_ + +#include + +#include "webrtc/base/optional.h" + +namespace webrtc { + +// Ring buffer containing floating point values. +struct CircularBuffer { + public: + explicit CircularBuffer(size_t size); + ~CircularBuffer(); + + void Push(float value); + rtc::Optional Pop(); + size_t Size() const { return nr_elements_in_buffer_; } + // This function fills the buffer with zeros, but does not change its size. + void Clear(); + + private: + std::vector buffer_; + size_t next_insertion_index_ = 0; + // This is the number of elements that have been pushed into the circular + // buffer, not the allocated buffer size. + size_t nr_elements_in_buffer_ = 0; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_CIRCULAR_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/echo_detector/circular_buffer_unittest.cc b/webrtc/modules/audio_processing/echo_detector/circular_buffer_unittest.cc new file mode 100644 index 0000000000..88f55a8ee7 --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/circular_buffer_unittest.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/echo_detector/circular_buffer.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { + +TEST(CircularBufferTests, LessThanMaxTest) { + CircularBuffer test_buffer(3); + test_buffer.Push(1.f); + test_buffer.Push(2.f); + EXPECT_EQ(rtc::Optional(1.f), test_buffer.Pop()); + EXPECT_EQ(rtc::Optional(2.f), test_buffer.Pop()); +} + +TEST(CircularBufferTests, FillTest) { + CircularBuffer test_buffer(3); + test_buffer.Push(1.f); + test_buffer.Push(2.f); + test_buffer.Push(3.f); + EXPECT_EQ(rtc::Optional(1.f), test_buffer.Pop()); + EXPECT_EQ(rtc::Optional(2.f), test_buffer.Pop()); + EXPECT_EQ(rtc::Optional(3.f), test_buffer.Pop()); +} + +TEST(CircularBufferTests, OverflowTest) { + CircularBuffer test_buffer(3); + test_buffer.Push(1.f); + test_buffer.Push(2.f); + test_buffer.Push(3.f); + test_buffer.Push(4.f); + // Because the circular buffer has a size of 3, the first insert should have + // been forgotten. + EXPECT_EQ(rtc::Optional(2.f), test_buffer.Pop()); + EXPECT_EQ(rtc::Optional(3.f), test_buffer.Pop()); + EXPECT_EQ(rtc::Optional(4.f), test_buffer.Pop()); +} + +TEST(CircularBufferTests, ReadFromEmpty) { + CircularBuffer test_buffer(3); + EXPECT_EQ(rtc::Optional(), test_buffer.Pop()); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.cc b/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.cc new file mode 100644 index 0000000000..e1236159e1 --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h" + +#include + +#include "webrtc/base/checks.h" + +namespace webrtc { +namespace { + +// Parameter controlling the adaptation speed. +constexpr float kAlpha = 0.01f; + +} // namespace + +void MeanVarianceEstimator::Update(float value) { + mean_ = (1.f - kAlpha) * mean_ + kAlpha * value; + variance_ = + (1.f - kAlpha) * variance_ + kAlpha * (value - mean_) * (value - mean_); +} + +float MeanVarianceEstimator::std_deviation() const { + RTC_DCHECK_GE(variance_, 0.f); + return sqrtf(variance_); +} + +float MeanVarianceEstimator::mean() const { + return mean_; +} + +void MeanVarianceEstimator::Clear() { + mean_ = 0.f; + variance_ = 0.f; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h b/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h new file mode 100644 index 0000000000..3c0700fad0 --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MEAN_VARIANCE_ESTIMATOR_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MEAN_VARIANCE_ESTIMATOR_H_ + +namespace webrtc { + +// This class iteratively estimates the mean and variance of a signal. +class MeanVarianceEstimator { + public: + void Update(float value); + float std_deviation() const; + float mean() const; + void Clear(); + + private: + // Estimate of the expected value of the input values. + float mean_ = 0.f; + // Estimate of the variance of the input values. + float variance_ = 0.f; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_MEAN_VARIANCE_ESTIMATOR_H_ diff --git a/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator_unittest.cc b/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator_unittest.cc new file mode 100644 index 0000000000..ec4350f45e --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/mean_variance_estimator_unittest.cc @@ -0,0 +1,64 @@ + +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { + +TEST(MeanVarianceEstimatorTests, InsertTwoValues) { + MeanVarianceEstimator test_estimator; + // Insert two values. + test_estimator.Update(3.f); + test_estimator.Update(5.f); + + EXPECT_GT(test_estimator.mean(), 0.f); + EXPECT_GT(test_estimator.std_deviation(), 0.f); + // Test Clear method + test_estimator.Clear(); + EXPECT_EQ(test_estimator.mean(), 0.f); + EXPECT_EQ(test_estimator.std_deviation(), 0.f); +} + +TEST(MeanVarianceEstimatorTests, InsertZeroes) { + MeanVarianceEstimator test_estimator; + // Insert the same value many times. + for (size_t i = 0; i < 10000; i++) { + test_estimator.Update(0.f); + } + EXPECT_EQ(test_estimator.mean(), 0.f); + EXPECT_EQ(test_estimator.std_deviation(), 0.f); +} + +TEST(MeanVarianceEstimatorTests, ConstantValueTest) { + MeanVarianceEstimator test_estimator; + for (size_t i = 0; i < 10000; i++) { + test_estimator.Update(3.f); + } + // The mean should be close to three, and the standard deviation should be + // close to zero. + EXPECT_NEAR(3.0f, test_estimator.mean(), 0.01f); + EXPECT_NEAR(0.0f, test_estimator.std_deviation(), 0.01f); +} + +TEST(MeanVarianceEstimatorTests, AlternatingValueTest) { + MeanVarianceEstimator test_estimator; + for (size_t i = 0; i < 10000; i++) { + test_estimator.Update(1.f); + test_estimator.Update(-1.f); + } + // The mean should be close to zero, and the standard deviation should be + // close to one. + EXPECT_NEAR(0.0f, test_estimator.mean(), 0.01f); + EXPECT_NEAR(1.0f, test_estimator.std_deviation(), 0.01f); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc b/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc new file mode 100644 index 0000000000..aad806d0e4 --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h" + +namespace webrtc { +namespace { + +// Parameter controlling the adaptation speed. +constexpr float kAlpha = 0.01f; + +} // namespace + +void NormalizedCovarianceEstimator::Update(float x, + float x_mean, + float x_sigma, + float y, + float y_mean, + float y_sigma) { + covariance_ = + (1.f - kAlpha) * covariance_ + kAlpha * (x - x_mean) * (y - y_mean); + normalized_cross_correlation_ = covariance_ / (x_sigma * y_sigma + .0001f); +} + +void NormalizedCovarianceEstimator::Clear() { + covariance_ = 0.f; + normalized_cross_correlation_ = 0.f; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h b/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h new file mode 100644 index 0000000000..f77a30ec73 --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_NORMALIZED_COVARIANCE_ESTIMATOR_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_NORMALIZED_COVARIANCE_ESTIMATOR_H_ + +namespace webrtc { + +// This class iteratively estimates the normalized covariance between two +// signals. +class NormalizedCovarianceEstimator { + public: + void Update(float x, + float x_mean, + float x_var, + float y, + float y_mean, + float y_var); + // This function returns an estimate of the Pearson product-moment correlation + // coefficient of the two signals. + float normalized_cross_correlation() const { + return normalized_cross_correlation_; + } + // This function resets the estimated values to zero. + void Clear(); + + private: + float normalized_cross_correlation_ = 0.f; + // Estimate of the covariance value. + float covariance_ = 0.f; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_DETECTOR_NORMALIZED_COVARIANCE_ESTIMATOR_H_ diff --git a/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator_unittest.cc b/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator_unittest.cc new file mode 100644 index 0000000000..04b3a92fb5 --- /dev/null +++ b/webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator_unittest.cc @@ -0,0 +1,40 @@ + +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { + +TEST(NormalizedCovarianceEstimatorTests, IdenticalSignalTest) { + NormalizedCovarianceEstimator test_estimator; + for (size_t i = 0; i < 10000; i++) { + test_estimator.Update(1.f, 0.f, 1.f, 1.f, 0.f, 1.f); + test_estimator.Update(-1.f, 0.f, 1.f, -1.f, 0.f, 1.f); + } + // A normalized covariance value close to 1 is expected. + EXPECT_NEAR(1.f, test_estimator.normalized_cross_correlation(), 0.01f); + test_estimator.Clear(); + EXPECT_EQ(0.f, test_estimator.normalized_cross_correlation()); +} + +TEST(NormalizedCovarianceEstimatorTests, OppositeSignalTest) { + NormalizedCovarianceEstimator test_estimator; + // Insert the same value many times. + for (size_t i = 0; i < 10000; i++) { + test_estimator.Update(1.f, 0.f, 1.f, -1.f, 0.f, 1.f); + test_estimator.Update(-1.f, 0.f, 1.f, 1.f, 0.f, 1.f); + } + // A normalized covariance value close to -1 is expected. + EXPECT_NEAR(-1.f, test_estimator.normalized_cross_correlation(), 0.01f); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/residual_echo_detector.cc b/webrtc/modules/audio_processing/residual_echo_detector.cc index df99de45bf..ff59dc2f5b 100644 --- a/webrtc/modules/audio_processing/residual_echo_detector.cc +++ b/webrtc/modules/audio_processing/residual_echo_detector.cc @@ -10,26 +10,113 @@ #include "webrtc/modules/audio_processing/residual_echo_detector.h" +#include +#include + #include "webrtc/modules/audio_processing/audio_buffer.h" +namespace { + +float Power(rtc::ArrayView input) { + return std::inner_product(input.begin(), input.end(), input.begin(), 0.f); +} + +constexpr size_t kLookbackFrames = 650; +// TODO(ivoc): Verify the size of this buffer. +constexpr size_t kRenderBufferSize = 30; + +} // namespace + namespace webrtc { -ResidualEchoDetector::ResidualEchoDetector() = default; +ResidualEchoDetector::ResidualEchoDetector() + : render_buffer_(kRenderBufferSize), + render_power_(kLookbackFrames), + render_power_mean_(kLookbackFrames), + render_power_std_dev_(kLookbackFrames), + covariances_(kLookbackFrames){}; ResidualEchoDetector::~ResidualEchoDetector() = default; void ResidualEchoDetector::AnalyzeRenderAudio( - rtc::ArrayView render_audio) const { - // TODO(ivoc): Add implementation. + rtc::ArrayView render_audio) { + if (render_buffer_.Size() == 0) { + frames_since_zero_buffer_size_ = 0; + } else if (frames_since_zero_buffer_size_ >= kRenderBufferSize) { + // This can happen in a few cases: at the start of a call, due to a glitch + // or due to clock drift. The excess capture value will be ignored. + // TODO(ivoc): Include how often this happens in APM stats. + render_buffer_.Pop(); + frames_since_zero_buffer_size_ = 0; + } + ++frames_since_zero_buffer_size_; + float power = Power(render_audio); + render_buffer_.Push(power); } void ResidualEchoDetector::AnalyzeCaptureAudio( - rtc::ArrayView render_audio) { - // TODO(ivoc): Add implementation. + rtc::ArrayView capture_audio) { + if (first_process_call_) { + // On the first process call (so the start of a call), we must flush the + // render buffer, otherwise the render data will be delayed. + render_buffer_.Clear(); + first_process_call_ = false; + } + + // Get the next render value. + const rtc::Optional buffered_render_power = render_buffer_.Pop(); + if (!buffered_render_power) { + // This can happen in a few cases: at the start of a call, due to a glitch + // or due to clock drift. The excess capture value will be ignored. + // TODO(ivoc): Include how often this happens in APM stats. + return; + } + // Update the render statistics, and store the statistics in circular buffers. + render_statistics_.Update(*buffered_render_power); + RTC_DCHECK_LT(next_insertion_index_, kLookbackFrames); + render_power_[next_insertion_index_] = *buffered_render_power; + render_power_mean_[next_insertion_index_] = render_statistics_.mean(); + render_power_std_dev_[next_insertion_index_] = + render_statistics_.std_deviation(); + + // Get the next capture value, update capture statistics and add the relevant + // values to the buffers. + const float capture_power = Power(capture_audio); + capture_statistics_.Update(capture_power); + const float capture_mean = capture_statistics_.mean(); + const float capture_std_deviation = capture_statistics_.std_deviation(); + + // Update the covariance values and determine the new echo likelihood. + echo_likelihood_ = 0.f; + for (size_t delay = 0; delay < covariances_.size(); ++delay) { + const size_t read_index = + (kLookbackFrames + next_insertion_index_ - delay) % kLookbackFrames; + RTC_DCHECK_LT(read_index, render_power_.size()); + covariances_[delay].Update(capture_power, capture_mean, + capture_std_deviation, render_power_[read_index], + render_power_mean_[read_index], + render_power_std_dev_[read_index]); + echo_likelihood_ = std::max( + echo_likelihood_, covariances_[delay].normalized_cross_correlation()); + } + + // Update the next insertion index. + ++next_insertion_index_; + next_insertion_index_ %= kLookbackFrames; } void ResidualEchoDetector::Initialize() { - // TODO(ivoc): Add implementation. + render_buffer_.Clear(); + std::fill(render_power_.begin(), render_power_.end(), 0.f); + std::fill(render_power_mean_.begin(), render_power_mean_.end(), 0.f); + std::fill(render_power_std_dev_.begin(), render_power_std_dev_.end(), 0.f); + render_statistics_.Clear(); + capture_statistics_.Clear(); + for (auto& cov : covariances_) { + cov.Clear(); + } + echo_likelihood_ = 0.f; + next_insertion_index_ = 0; } void ResidualEchoDetector::PackRenderAudioBuffer( @@ -44,9 +131,4 @@ void ResidualEchoDetector::PackRenderAudioBuffer( audio->num_frames_per_band())); } -float ResidualEchoDetector::get_echo_likelihood() const { - // TODO(ivoc): Add implementation. - return 0.0f; -} - } // namespace webrtc diff --git a/webrtc/modules/audio_processing/residual_echo_detector.h b/webrtc/modules/audio_processing/residual_echo_detector.h index 723371c7ce..37aa283b08 100644 --- a/webrtc/modules/audio_processing/residual_echo_detector.h +++ b/webrtc/modules/audio_processing/residual_echo_detector.h @@ -11,10 +11,12 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ -#include #include #include "webrtc/base/array_view.h" +#include "webrtc/modules/audio_processing/echo_detector/circular_buffer.h" +#include "webrtc/modules/audio_processing/echo_detector/mean_variance_estimator.h" +#include "webrtc/modules/audio_processing/echo_detector/normalized_covariance_estimator.h" namespace webrtc { @@ -27,7 +29,7 @@ class ResidualEchoDetector { ~ResidualEchoDetector(); // This function should be called while holding the render lock. - void AnalyzeRenderAudio(rtc::ArrayView render_audio) const; + void AnalyzeRenderAudio(rtc::ArrayView render_audio); // This function should be called while holding the capture lock. void AnalyzeCaptureAudio(rtc::ArrayView capture_audio); @@ -39,7 +41,36 @@ class ResidualEchoDetector { std::vector* packed_buffer); // This function should be called while holding the capture lock. - float get_echo_likelihood() const; + float echo_likelihood() const { return echo_likelihood_; } + + private: + // Keep track if the |Process| function has been previously called. + bool first_process_call_ = true; + // Buffer for storing the power of incoming farend buffers. This is needed for + // cases where calls to BufferFarend and Process are jittery. + CircularBuffer render_buffer_; + // Count how long ago it was that the size of |render_buffer_| was zero. This + // value is also reset to zero when clock drift is detected and a value from + // the renderbuffer is discarded, even though the buffer is not actually zero + // at that point. This is done to avoid repeatedly removing elements in this + // situation. + size_t frames_since_zero_buffer_size_ = 0; + + // Circular buffers containing delayed versions of the power, mean and + // standard deviation, for calculating the delayed covariance values. + std::vector render_power_; + std::vector render_power_mean_; + std::vector render_power_std_dev_; + // Covariance estimates for different delay values. + std::vector covariances_; + // Index where next element should be inserted in all of the above circular + // buffers. + size_t next_insertion_index_ = 0; + + MeanVarianceEstimator render_statistics_; + MeanVarianceEstimator capture_statistics_; + // Current echo likelihood. + float echo_likelihood_ = 0.f; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/residual_echo_detector_unittest.cc b/webrtc/modules/audio_processing/residual_echo_detector_unittest.cc new file mode 100644 index 0000000000..7f6cf16abd --- /dev/null +++ b/webrtc/modules/audio_processing/residual_echo_detector_unittest.cc @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "webrtc/modules/audio_processing/residual_echo_detector.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { + +TEST(ResidualEchoDetectorTests, Echo) { + ResidualEchoDetector echo_detector; + std::vector ones(160, 1.f); + std::vector zeros(160, 0.f); + + // In this test the capture signal has a delay of 10 frames w.r.t. the render + // signal, but is otherwise identical. Both signals are periodic with a 20 + // frame interval. + for (int i = 0; i < 1000; i++) { + if (i % 20 == 0) { + echo_detector.AnalyzeRenderAudio(ones); + echo_detector.AnalyzeCaptureAudio(zeros); + } else if (i % 20 == 10) { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(ones); + } else { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(zeros); + } + } + // We expect to detect echo with near certain likelihood. + EXPECT_NEAR(1.f, echo_detector.echo_likelihood(), 0.01f); +} + +TEST(ResidualEchoDetectorTests, NoEcho) { + ResidualEchoDetector echo_detector; + std::vector ones(160, 1.f); + std::vector zeros(160, 0.f); + + // In this test the capture signal is always zero, so no echo should be + // detected. + for (int i = 0; i < 1000; i++) { + if (i % 20 == 0) { + echo_detector.AnalyzeRenderAudio(ones); + } else { + echo_detector.AnalyzeRenderAudio(zeros); + } + echo_detector.AnalyzeCaptureAudio(zeros); + } + // We expect to not detect any echo. + EXPECT_NEAR(0.f, echo_detector.echo_likelihood(), 0.01f); +} + +TEST(ResidualEchoDetectorTests, EchoWithRenderClockDrift) { + ResidualEchoDetector echo_detector; + std::vector ones(160, 1.f); + std::vector zeros(160, 0.f); + + // In this test the capture signal has a delay of 10 frames w.r.t. the render + // signal, but is otherwise identical. Both signals are periodic with a 20 + // frame interval. There is a simulated clock drift of 1% in this test, with + // the render side producing data slightly faster. + for (int i = 0; i < 1000; i++) { + if (i % 20 == 0) { + echo_detector.AnalyzeRenderAudio(ones); + echo_detector.AnalyzeCaptureAudio(zeros); + } else if (i % 20 == 10) { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(ones); + } else { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(zeros); + } + if (i % 100 == 0) { + // This is causing the simulated clock drift. + echo_detector.AnalyzeRenderAudio(zeros); + } + } + // We expect to detect echo with high likelihood. Clock drift is harder to + // correct on the render side than on the capture side. This is due to the + // render buffer, clock drift can only be discovered after a certain delay. + // A growing buffer can be caused by jitter or clock drift and it's not + // possible to make this decision right away. For this reason we only expect + // an echo likelihood of 80% in this test. + EXPECT_GT(echo_detector.echo_likelihood(), 0.8f); +} + +TEST(ResidualEchoDetectorTests, EchoWithCaptureClockDrift) { + ResidualEchoDetector echo_detector; + std::vector ones(160, 1.f); + std::vector zeros(160, 0.f); + + // In this test the capture signal has a delay of 10 frames w.r.t. the render + // signal, but is otherwise identical. Both signals are periodic with a 20 + // frame interval. There is a simulated clock drift of 1% in this test, with + // the capture side producing data slightly faster. + for (int i = 0; i < 1000; i++) { + if (i % 20 == 0) { + echo_detector.AnalyzeRenderAudio(ones); + echo_detector.AnalyzeCaptureAudio(zeros); + } else if (i % 20 == 10) { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(ones); + } else { + echo_detector.AnalyzeRenderAudio(zeros); + echo_detector.AnalyzeCaptureAudio(zeros); + } + if (i % 100 == 0) { + // This is causing the simulated clock drift. + echo_detector.AnalyzeCaptureAudio(zeros); + } + } + // We expect to detect echo with near certain likelihood. + EXPECT_NEAR(1.f, echo_detector.echo_likelihood(), 0.01f); +} + +} // namespace webrtc