From 78026754a73496542b3e7bba7153385d7acc3613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=85hgren?= Date: Wed, 1 Aug 2018 16:24:08 +0200 Subject: [PATCH] AEC3: Utilize shadow filter output to respond to audio path changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL adds functionality to use the shadow filter output instead of the main filter output for cases when the former is better than the latter. One case when that happens is when there have been an echo path change, either in the acoustic path, in the audio buffers or due to some active audio processing effects being applied on the device. The CL causes less echo leaks, in particular on devices with active render processing. Bug: webrtc:9581,chromium:869821 Change-Id: Icb8df1b94141598da82dc188051ac59e43338938 Reviewed-on: https://webrtc-review.googlesource.com/91820 Reviewed-by: Sam Zackrisson Commit-Queue: Per Åhgren Cr-Commit-Position: refs/heads/master@{#24166} --- modules/audio_processing/aec3/echo_remover.cc | 38 ++++++++++++++++++- modules/audio_processing/aec3/subtractor.cc | 4 +- .../aec3/subtractor_output.cc | 6 +++ .../audio_processing/aec3/subtractor_output.h | 3 ++ 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index 67929b147a..8777887431 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -32,11 +32,17 @@ #include "rtc_base/atomicops.h" #include "rtc_base/constructormagic.h" #include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { +bool UseShadowFilterOutput() { + return !field_trial::IsEnabled( + "WebRTC-Aec3UtilizeShadowFilterOutputKillSwitch"); +} + void LinearEchoPower(const FftData& E, const FftData& Y, std::array* S2) { @@ -86,12 +92,41 @@ class EchoRemoverImpl final : public EchoRemover { } private: + // Selects which of the shadow and main linear filter outputs that is most + // appropriate to pass to the suppressor. + const std::array& ChooseLinearFilterOutput( + const SubtractorOutput& subtractor_output) { + if (!use_shadow_filter_output_) { + return subtractor_output.e_main; + } + + // As the output of the main adaptive filter generally should be better than + // the shadow filter output, add a margin and threshold for when choosing + // the shadow filter output. + if (subtractor_output.e2_shadow < 0.9f * subtractor_output.e2_main && + subtractor_output.y2 > 30.f * 30.f * kBlockSize && + (subtractor_output.s2_main > 60.f * 60.f * kBlockSize || + subtractor_output.s2_shadow > 60.f * 60.f * kBlockSize)) { + return subtractor_output.e_shadow; + } + + // If the main filter is diverged, choose the filter output that has the + // lowest power. + if (subtractor_output.e2_shadow < subtractor_output.e2_main && + subtractor_output.y2 < subtractor_output.e2_main) { + return subtractor_output.e_shadow; + } + + return subtractor_output.e_main; + } + static int instance_count_; const EchoCanceller3Config config_; const Aec3Fft fft_; std::unique_ptr data_dumper_; const Aec3Optimization optimization_; const int sample_rate_hz_; + const bool use_shadow_filter_output_; Subtractor subtractor_; SuppressionGain suppression_gain_; ComfortNoiseGenerator cng_; @@ -121,6 +156,7 @@ EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), optimization_(DetectOptimization()), sample_rate_hz_(sample_rate_hz), + use_shadow_filter_output_(UseShadowFilterOutput()), subtractor_(config, data_dumper_.get(), optimization_), suppression_gain_(config_, optimization_, sample_rate_hz), cng_(optimization_), @@ -222,7 +258,7 @@ void EchoRemoverImpl::ProcessCapture( // If the delay is known, use the echo subtractor. subtractor_.Process(*render_buffer, y0, render_signal_analyzer_, aec_state_, &subtractor_output); - const auto& e = subtractor_output.e_main; + const auto& e = ChooseLinearFilterOutput(subtractor_output); // Compute spectra. WindowedPaddedFft(fft_, y0, y_old_, &Y); diff --git a/modules/audio_processing/aec3/subtractor.cc b/modules/audio_processing/aec3/subtractor.cc index 41711547c3..785c208025 100644 --- a/modules/audio_processing/aec3/subtractor.cc +++ b/modules/audio_processing/aec3/subtractor.cc @@ -182,8 +182,8 @@ void Subtractor::Process(const RenderBuffer& render_buffer, shadow_filter_.Filter(render_buffer, &S); bool shadow_saturation = false; - PredictionError(fft_, S, y, &e_shadow, nullptr, adaptation_during_saturation_, - &shadow_saturation); + PredictionError(fft_, S, y, &e_shadow, &output->s_shadow, + adaptation_during_saturation_, &shadow_saturation); // Compute the signal powers in the subtractor output. output->UpdatePowers(y); diff --git a/modules/audio_processing/aec3/subtractor_output.cc b/modules/audio_processing/aec3/subtractor_output.cc index a405c3e71a..affa4a3a06 100644 --- a/modules/audio_processing/aec3/subtractor_output.cc +++ b/modules/audio_processing/aec3/subtractor_output.cc @@ -19,6 +19,7 @@ SubtractorOutput::~SubtractorOutput() = default; void SubtractorOutput::Reset() { s_main.fill(0.f); + s_shadow.fill(0.f); e_main.fill(0.f); e_shadow.fill(0.f); E_main.re.fill(0.f); @@ -27,6 +28,8 @@ void SubtractorOutput::Reset() { E2_shadow.fill(0.f); e2_main = 0.f; e2_shadow = 0.f; + s2_main = 0.f; + s2_shadow = 0.f; y2 = 0.f; } @@ -36,6 +39,9 @@ void SubtractorOutput::UpdatePowers(rtc::ArrayView y) { e2_main = std::accumulate(e_main.begin(), e_main.end(), 0.f, sum_of_squares); e2_shadow = std::accumulate(e_shadow.begin(), e_shadow.end(), 0.f, sum_of_squares); + s2_main = std::accumulate(s_main.begin(), s_main.end(), 0.f, sum_of_squares); + s2_shadow = + std::accumulate(s_shadow.begin(), s_shadow.end(), 0.f, sum_of_squares); } } // namespace webrtc diff --git a/modules/audio_processing/aec3/subtractor_output.h b/modules/audio_processing/aec3/subtractor_output.h index 229fd92a3c..89727e9396 100644 --- a/modules/audio_processing/aec3/subtractor_output.h +++ b/modules/audio_processing/aec3/subtractor_output.h @@ -25,11 +25,14 @@ struct SubtractorOutput { ~SubtractorOutput(); std::array s_main; + std::array s_shadow; std::array e_main; std::array e_shadow; FftData E_main; std::array E2_main; std::array E2_shadow; + float s2_main = 0.f; + float s2_shadow = 0.f; float e2_main = 0.f; float e2_shadow = 0.f; float y2 = 0.f;