From 8718afb283b7bf6b071d4a65ad15ab51d438123f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=85hgren?= Date: Tue, 15 Oct 2019 10:31:35 +0200 Subject: [PATCH] AEC3: Made EchoAudibility multichannel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL corrects the EchoAudibility and StationarityEstimator code to work properly with multiple channels. It also changes the naming of the FilterDelayBlocks() method to better reflect what it does. The changes have been verified to be bitexact over a large number of recordings. Bug: webrtc:10913 Change-Id: I070b531efcdff4c33f70fd5b37fbb556dcebe5b4 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/156565 Reviewed-by: Sam Zackrisson Commit-Queue: Per Åhgren Cr-Commit-Position: refs/heads/master@{#29482} --- .../aec3/adaptive_fir_filter_unittest.cc | 2 +- modules/audio_processing/aec3/aec_state.cc | 29 +++++------ modules/audio_processing/aec3/aec_state.h | 11 +++-- .../audio_processing/aec3/echo_audibility.cc | 46 +++++++++-------- .../audio_processing/aec3/echo_audibility.h | 15 +++--- modules/audio_processing/aec3/echo_remover.cc | 9 ++-- .../aec3/echo_remover_metrics.cc | 3 +- .../aec3/main_filter_update_gain_unittest.cc | 2 +- .../aec3/residual_echo_estimator.cc | 9 ++-- .../aec3/stationarity_estimator.cc | 49 +++++++++++++++---- .../aec3/stationarity_estimator.h | 7 +-- .../aec3/subtractor_unittest.cc | 2 +- 12 files changed, 115 insertions(+), 69 deletions(-) diff --git a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc index f1a6489a6d..022e860d45 100644 --- a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc +++ b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc @@ -441,7 +441,7 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) { auto* const render_buffer = render_delay_buffer->GetRenderBuffer(); render_signal_analyzer.Update(*render_buffer, - aec_state.FilterDelayBlocks()); + aec_state.MinDirectPathFilterDelay()); filter.Filter(*render_buffer, &S); fft.Ifft(S, &s_scratch); diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index 7518e3a3ea..13b9bccf03 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -29,7 +29,7 @@ namespace { constexpr size_t kBlocksSinceConvergencedFilterInit = 10000; constexpr size_t kBlocksSinceConsistentEstimateInit = 10000; -void UpdateAndComputeReverb( +void ComputeAvgRenderReverb( const SpectrumBuffer& spectrum_buffer, int delay_blocks, float reverb_decay, @@ -211,16 +211,16 @@ void AecState::Update( strong_not_saturated_render_blocks_ += active_render && !SaturatedCapture() ? 1 : 0; - std::array X2_reverb; + std::array avg_render_spectrum_with_reverb; - UpdateAndComputeReverb(render_buffer.GetSpectrumBuffer(), - delay_state_.DirectPathFilterDelays()[0], - ReverbDecay(), &reverb_model_, X2_reverb); + ComputeAvgRenderReverb(render_buffer.GetSpectrumBuffer(), + delay_state_.MinDirectPathFilterDelay(), ReverbDecay(), + &avg_render_reverb_, avg_render_spectrum_with_reverb); if (config_.echo_audibility.use_stationarity_properties) { // Update the echo audibility evaluator. - echo_audibility_.Update(render_buffer, reverb_model_.reverb(), - delay_state_.DirectPathFilterDelays()[0], + echo_audibility_.Update(render_buffer, avg_render_reverb_.reverb(), + delay_state_.MinDirectPathFilterDelay(), delay_state_.ExternalDelayReported()); } @@ -229,17 +229,15 @@ void AecState::Update( erle_estimator_.Reset(false); } - // TODO(bugs.webrtc.org/10913): Take all channels into account. - const auto& X2 = - render_buffer.Spectrum(delay_state_.DirectPathFilterDelays()[0], - /*channel=*/0); - const auto& X2_input_erle = X2_reverb; - erle_estimator_.Update(render_buffer, adaptive_filter_frequency_responses[0], - X2_input_erle, Y2[0], E2_main[0], + avg_render_spectrum_with_reverb, Y2[0], E2_main[0], subtractor_output_analyzers_[0].ConvergedFilter(), config_.erle.onset_detection); + // TODO(bugs.webrtc.org/10913): Take all channels into account. + const auto& X2 = + render_buffer.Spectrum(delay_state_.MinDirectPathFilterDelay(), + /*channel=*/0); erl_estimator_.Update(subtractor_output_analyzers_[0].ConvergedFilter(), X2, Y2[0]); @@ -357,6 +355,9 @@ void AecState::FilterDelay::Update( analyzer_filter_delay_estimates_blocks.end(), filter_delays_blocks_.begin()); } + + min_filter_delay_ = *std::min_element(filter_delays_blocks_.begin(), + filter_delays_blocks_.end()); } AecState::TransparentMode::TransparentMode(const EchoCanceller3Config& config) diff --git a/modules/audio_processing/aec3/aec_state.h b/modules/audio_processing/aec3/aec_state.h index 79fe13e431..71000b4328 100644 --- a/modules/audio_processing/aec3/aec_state.h +++ b/modules/audio_processing/aec3/aec_state.h @@ -91,8 +91,8 @@ class AecState { float ErlTimeDomain() const { return erl_estimator_.ErlTimeDomain(); } // Returns the delay estimate based on the linear filter. - int FilterDelayBlocks() const { - return delay_state_.DirectPathFilterDelays()[0]; + int MinDirectPathFilterDelay() const { + return delay_state_.MinDirectPathFilterDelay(); } // Returns whether the capture signal is saturated. @@ -194,6 +194,10 @@ class AecState { return filter_delays_blocks_; } + // Returns the minimum delay among the direct path delays relative to the + // beginning of the filter + int MinDirectPathFilterDelay() const { return min_filter_delay_; } + // Updates the delay estimates based on new data. void Update( rtc::ArrayView analyzer_filter_delay_estimates_blocks, @@ -204,6 +208,7 @@ class AecState { const int delay_headroom_samples_; bool external_delay_reported_ = false; std::vector filter_delays_blocks_; + int min_filter_delay_ = 0; absl::optional external_delay_; } delay_state_; @@ -308,7 +313,7 @@ class AecState { absl::optional external_delay_; EchoAudibility echo_audibility_; ReverbModelEstimator reverb_model_estimator_; - ReverbModel reverb_model_; + ReverbModel avg_render_reverb_; std::vector subtractor_output_analyzers_; }; diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc index c5341080b4..db62236c11 100644 --- a/modules/audio_processing/aec3/echo_audibility.cc +++ b/modules/audio_processing/aec3/echo_audibility.cc @@ -29,18 +29,16 @@ EchoAudibility::EchoAudibility(bool use_render_stationarity_at_init) EchoAudibility::~EchoAudibility() = default; -void EchoAudibility::Update( - const RenderBuffer& render_buffer, - rtc::ArrayView render_reverb_contribution_spectrum, - int delay_blocks, - bool external_delay_seen) { +void EchoAudibility::Update(const RenderBuffer& render_buffer, + rtc::ArrayView average_reverb, + int delay_blocks, + bool external_delay_seen) { UpdateRenderNoiseEstimator(render_buffer.GetSpectrumBuffer(), render_buffer.GetBlockBuffer(), external_delay_seen); if (external_delay_seen || use_render_stationarity_at_init_) { - UpdateRenderStationarityFlags( - render_buffer, render_reverb_contribution_spectrum, delay_blocks); + UpdateRenderStationarityFlags(render_buffer, average_reverb, delay_blocks); } } @@ -52,18 +50,17 @@ void EchoAudibility::Reset() { void EchoAudibility::UpdateRenderStationarityFlags( const RenderBuffer& render_buffer, - rtc::ArrayView render_reverb_contribution_spectrum, - int delay_blocks) { + rtc::ArrayView average_reverb, + int min_channel_delay_blocks) { const SpectrumBuffer& spectrum_buffer = render_buffer.GetSpectrumBuffer(); - int idx_at_delay = - spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks); + int idx_at_delay = spectrum_buffer.OffsetIndex(spectrum_buffer.read, + min_channel_delay_blocks); - int num_lookahead = render_buffer.Headroom() - delay_blocks + 1; + int num_lookahead = render_buffer.Headroom() - min_channel_delay_blocks + 1; num_lookahead = std::max(0, num_lookahead); - render_stationarity_.UpdateStationarityFlags( - spectrum_buffer, render_reverb_contribution_spectrum, idx_at_delay, - num_lookahead); + render_stationarity_.UpdateStationarityFlags(spectrum_buffer, average_reverb, + idx_at_delay, num_lookahead); } void EchoAudibility::UpdateRenderNoiseEstimator( @@ -83,14 +80,15 @@ void EchoAudibility::UpdateRenderNoiseEstimator( for (int idx = render_spectrum_write_prev_.value(); idx != render_spectrum_write_current; idx = spectrum_buffer.DecIndex(idx)) { - render_stationarity_.UpdateNoiseEstimator( - spectrum_buffer.buffer[idx][/*channel=*/0]); + render_stationarity_.UpdateNoiseEstimator(spectrum_buffer.buffer[idx]); } } render_spectrum_write_prev_ = render_spectrum_write_current; } bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) { + const int num_render_channels = + static_cast(block_buffer.buffer[0].size()); bool too_low = false; const int render_block_write_current = block_buffer.write; if (render_block_write_current == render_block_write_prev_) { @@ -98,10 +96,16 @@ bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) { } else { for (int idx = render_block_write_prev_; idx != render_block_write_current; idx = block_buffer.IncIndex(idx)) { - auto block = block_buffer.buffer[idx][0][0]; - auto r = std::minmax_element(block.cbegin(), block.cend()); - float max_abs = std::max(std::fabs(*r.first), std::fabs(*r.second)); - if (max_abs < 10) { + float max_abs_over_channels = 0.f; + for (int ch = 0; ch < num_render_channels; ++ch) { + auto block = block_buffer.buffer[idx][0][ch]; + auto r = std::minmax_element(block.cbegin(), block.cend()); + float max_abs_channel = + std::max(std::fabs(*r.first), std::fabs(*r.second)); + max_abs_over_channels = + std::max(max_abs_over_channels, max_abs_channel); + } + if (max_abs_over_channels < 10.f) { too_low = true; // Discards all blocks if one of them is too low. break; } diff --git a/modules/audio_processing/aec3/echo_audibility.h b/modules/audio_processing/aec3/echo_audibility.h index 0152ea474f..1ffc017b7d 100644 --- a/modules/audio_processing/aec3/echo_audibility.h +++ b/modules/audio_processing/aec3/echo_audibility.h @@ -28,10 +28,13 @@ class EchoAudibility { explicit EchoAudibility(bool use_render_stationarity_at_init); ~EchoAudibility(); + EchoAudibility(const EchoAudibility&) = delete; + EchoAudibility& operator=(const EchoAudibility&) = delete; + // Feed new render data to the echo audibility estimator. void Update(const RenderBuffer& render_buffer, - rtc::ArrayView render_reverb_contribution_spectrum, - int delay_blocks, + rtc::ArrayView average_reverb, + int min_channel_delay_blocks, bool external_delay_seen); // Get the residual echo scaling. void GetResidualEchoScaling(bool filter_has_had_time_to_converge, @@ -57,10 +60,9 @@ class EchoAudibility { void Reset(); // Updates the render stationarity flags for the current frame. - void UpdateRenderStationarityFlags( - const RenderBuffer& render_buffer, - rtc::ArrayView render_reverb_contribution_spectrum, - int delay_blocks); + void UpdateRenderStationarityFlags(const RenderBuffer& render_buffer, + rtc::ArrayView average_reverb, + int delay_blocks); // Updates the noise estimator with the new render data since the previous // call to this method. @@ -77,7 +79,6 @@ class EchoAudibility { bool non_zero_render_seen_; const bool use_render_stationarity_at_init_; StationarityEstimator render_stationarity_; - RTC_DISALLOW_COPY_AND_ASSIGN(EchoAudibility); }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index e6f17c7e32..7cec47cca0 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -356,7 +356,7 @@ void EchoRemoverImpl::ProcessCapture( // Analyze the render signal. render_signal_analyzer_.Update(*render_buffer, - aec_state_.FilterDelayBlocks()); + aec_state_.MinDirectPathFilterDelay()); // State transition. if (aec_state_.TransitionTriggered()) { @@ -457,10 +457,11 @@ void EchoRemoverImpl::ProcessCapture( data_dumper_->DumpRaw("aec3_S2_linear", S2_linear[0]); data_dumper_->DumpRaw("aec3_Y2", Y2[0]); data_dumper_->DumpRaw( - "aec3_X2", - render_buffer->Spectrum(aec_state_.FilterDelayBlocks(), /*channel=*/0)); + "aec3_X2", render_buffer->Spectrum(aec_state_.MinDirectPathFilterDelay(), + /*channel=*/0)); data_dumper_->DumpRaw("aec3_R2", R2[0]); - data_dumper_->DumpRaw("aec3_filter_delay", aec_state_.FilterDelayBlocks()); + data_dumper_->DumpRaw("aec3_filter_delay", + aec_state_.MinDirectPathFilterDelay()); data_dumper_->DumpRaw("aec3_capture_saturation", aec_state_.SaturatedCapture() ? 1 : 0); } diff --git a/modules/audio_processing/aec3/echo_remover_metrics.cc b/modules/audio_processing/aec3/echo_remover_metrics.cc index 4ab05f804b..69d2252c2b 100644 --- a/modules/audio_processing/aec3/echo_remover_metrics.cc +++ b/modules/audio_processing/aec3/echo_remover_metrics.cc @@ -241,7 +241,8 @@ void EchoRemoverMetrics::Update( static_cast( active_render_count_ > kMetricsCollectionBlocksBy2 ? 1 : 0)); RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.FilterDelay", - aec_state.FilterDelayBlocks(), 0, 30, 31); + aec_state.MinDirectPathFilterDelay(), 0, 30, + 31); RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.EchoCanceller.CaptureSaturation", static_cast(saturated_capture_ ? 1 : 0)); break; diff --git a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc index 92334c21b7..fa3b263502 100644 --- a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc +++ b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc @@ -150,7 +150,7 @@ void RunFilterUpdateTest(int num_blocks_to_process, render_delay_buffer->PrepareCaptureProcessing(); render_signal_analyzer.Update(*render_delay_buffer->GetRenderBuffer(), - aec_state.FilterDelayBlocks()); + aec_state.MinDirectPathFilterDelay()); // Apply the main filter. main_filter.Filter(*render_delay_buffer->GetRenderBuffer(), &S); diff --git a/modules/audio_processing/aec3/residual_echo_estimator.cc b/modules/audio_processing/aec3/residual_echo_estimator.cc index 07197e3d3a..6a8a3f290c 100644 --- a/modules/audio_processing/aec3/residual_echo_estimator.cc +++ b/modules/audio_processing/aec3/residual_echo_estimator.cc @@ -203,7 +203,7 @@ void ResidualEchoEstimator::Estimate( std::array X2; EchoGeneratingPower(num_render_channels_, render_buffer.GetSpectrumBuffer(), config_.echo_model, - aec_state.FilterDelayBlocks(), X2); + aec_state.MinDirectPathFilterDelay(), X2); if (!aec_state.UseStationarityProperties()) { ApplyNoiseGate(config_.echo_model, X2); } @@ -288,9 +288,10 @@ void ResidualEchoEstimator::AddReverb( const size_t num_capture_channels = R2.size(); // Choose reverb partition based on what type of echo power model is used. - const size_t first_reverb_partition = reverb_type == ReverbType::kLinear - ? aec_state.FilterLengthBlocks() + 1 - : aec_state.FilterDelayBlocks() + 1; + const size_t first_reverb_partition = + reverb_type == ReverbType::kLinear + ? aec_state.FilterLengthBlocks() + 1 + : aec_state.MinDirectPathFilterDelay() + 1; // Compute render power for the reverb. std::array render_power_data; diff --git a/modules/audio_processing/aec3/stationarity_estimator.cc b/modules/audio_processing/aec3/stationarity_estimator.cc index 080d13d9bf..d0c3c9ca34 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.cc +++ b/modules/audio_processing/aec3/stationarity_estimator.cc @@ -45,7 +45,7 @@ void StationarityEstimator::Reset() { // Update just the noise estimator. Usefull until the delay is known void StationarityEstimator::UpdateNoiseEstimator( - rtc::ArrayView spectrum) { + rtc::ArrayView> spectrum) { noise_.Update(spectrum); data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum()); data_dumper_->DumpRaw("aec3_stationarity_is_block_stationary", @@ -99,15 +99,20 @@ bool StationarityEstimator::IsBlockStationary() const { bool StationarityEstimator::EstimateBandStationarity( const SpectrumBuffer& spectrum_buffer, - rtc::ArrayView reverb, + rtc::ArrayView average_reverb, const std::array& indexes, size_t band) const { constexpr float kThrStationarity = 10.f; float acum_power = 0.f; + const int num_render_channels = + static_cast(spectrum_buffer.buffer[0].size()); + const float one_by_num_channels = 1.f / num_render_channels; for (auto idx : indexes) { - acum_power += spectrum_buffer.buffer[idx][/*channel=*/0][band]; + for (int ch = 0; ch < num_render_channels; ++ch) { + acum_power += spectrum_buffer.buffer[idx][ch][band] * one_by_num_channels; + } } - acum_power += reverb[band]; + acum_power += average_reverb[band]; float noise = kWindowLength * GetStationarityPowerBand(band); RTC_CHECK_LT(0.f, noise); bool stationary = acum_power < kThrStationarity * noise; @@ -163,16 +168,42 @@ void StationarityEstimator::NoiseSpectrum::Reset() { } void StationarityEstimator::NoiseSpectrum::Update( - rtc::ArrayView spectrum) { - RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum.size()); + rtc::ArrayView> spectrum) { + RTC_DCHECK_LE(1, spectrum[0].size()); + const int num_render_channels = static_cast(spectrum.size()); + + std::array avg_spectrum_data; + rtc::ArrayView avg_spectrum; + RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum[0].size()); + if (num_render_channels == 1) { + avg_spectrum = spectrum[0]; + } else { + // For multiple channels, average the channel spectra before passing to the + // noise spectrum estimator. + avg_spectrum = avg_spectrum_data; + std::copy(spectrum[0].begin(), spectrum[0].end(), + avg_spectrum_data.begin()); + for (int ch = 1; ch < num_render_channels; ++ch) { + RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum[ch].size()); + for (size_t k = 1; k < kFftLengthBy2Plus1; ++k) { + avg_spectrum_data[k] += spectrum[ch][k]; + } + } + + const float one_by_num_channels = 1.f / num_render_channels; + for (size_t k = 1; k < kFftLengthBy2Plus1; ++k) { + avg_spectrum_data[k] *= one_by_num_channels; + } + } + ++block_counter_; float alpha = GetAlpha(); - for (size_t k = 0; k < spectrum.size(); ++k) { + for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { if (block_counter_ <= kNBlocksAverageInitPhase) { - noise_spectrum_[k] += (1.f / kNBlocksAverageInitPhase) * spectrum[k]; + noise_spectrum_[k] += (1.f / kNBlocksAverageInitPhase) * avg_spectrum[k]; } else { noise_spectrum_[k] = - UpdateBandBySmoothing(spectrum[k], noise_spectrum_[k], alpha); + UpdateBandBySmoothing(avg_spectrum[k], noise_spectrum_[k], alpha); } } } diff --git a/modules/audio_processing/aec3/stationarity_estimator.h b/modules/audio_processing/aec3/stationarity_estimator.h index 504fea7138..5860ef1625 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.h +++ b/modules/audio_processing/aec3/stationarity_estimator.h @@ -15,6 +15,7 @@ #include #include +#include #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" // kFftLengthBy2Plus1... @@ -35,7 +36,7 @@ class StationarityEstimator { void Reset(); // Update just the noise estimator. Usefull until the delay is known - void UpdateNoiseEstimator(rtc::ArrayView spectrum); + void UpdateNoiseEstimator(rtc::ArrayView> spectrum); // Update the flag indicating whether this current frame is stationary. For // getting a more robust estimation, it looks at future and/or past frames. @@ -61,7 +62,7 @@ class StationarityEstimator { // Get an estimation of the stationarity for the current band by looking // at the past/present/future available data. bool EstimateBandStationarity(const SpectrumBuffer& spectrum_buffer, - rtc::ArrayView reverb, + rtc::ArrayView average_reverb, const std::array& indexes, size_t band) const; @@ -85,7 +86,7 @@ class StationarityEstimator { void Reset(); // Update the noise power spectrum with a new frame. - void Update(rtc::ArrayView spectrum); + void Update(rtc::ArrayView> spectrum); // Get the noise estimation power spectrum. rtc::ArrayView Spectrum() const { return noise_spectrum_; } diff --git a/modules/audio_processing/aec3/subtractor_unittest.cc b/modules/audio_processing/aec3/subtractor_unittest.cc index 05faa4fd70..b59fa7b5e8 100644 --- a/modules/audio_processing/aec3/subtractor_unittest.cc +++ b/modules/audio_processing/aec3/subtractor_unittest.cc @@ -135,7 +135,7 @@ std::vector RunSubtractorTest( } render_delay_buffer->PrepareCaptureProcessing(); render_signal_analyzer.Update(*render_delay_buffer->GetRenderBuffer(), - aec_state.FilterDelayBlocks()); + aec_state.MinDirectPathFilterDelay()); // Handle echo path changes. if (std::find(blocks_with_echo_path_changes.begin(),