diff --git a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc index f1a6489a6d..022e860d45 100644 --- a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc +++ b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc @@ -441,7 +441,7 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) { auto* const render_buffer = render_delay_buffer->GetRenderBuffer(); render_signal_analyzer.Update(*render_buffer, - aec_state.FilterDelayBlocks()); + aec_state.MinDirectPathFilterDelay()); filter.Filter(*render_buffer, &S); fft.Ifft(S, &s_scratch); diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index 7518e3a3ea..13b9bccf03 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -29,7 +29,7 @@ namespace { constexpr size_t kBlocksSinceConvergencedFilterInit = 10000; constexpr size_t kBlocksSinceConsistentEstimateInit = 10000; -void UpdateAndComputeReverb( +void ComputeAvgRenderReverb( const SpectrumBuffer& spectrum_buffer, int delay_blocks, float reverb_decay, @@ -211,16 +211,16 @@ void AecState::Update( strong_not_saturated_render_blocks_ += active_render && !SaturatedCapture() ? 1 : 0; - std::array X2_reverb; + std::array avg_render_spectrum_with_reverb; - UpdateAndComputeReverb(render_buffer.GetSpectrumBuffer(), - delay_state_.DirectPathFilterDelays()[0], - ReverbDecay(), &reverb_model_, X2_reverb); + ComputeAvgRenderReverb(render_buffer.GetSpectrumBuffer(), + delay_state_.MinDirectPathFilterDelay(), ReverbDecay(), + &avg_render_reverb_, avg_render_spectrum_with_reverb); if (config_.echo_audibility.use_stationarity_properties) { // Update the echo audibility evaluator. - echo_audibility_.Update(render_buffer, reverb_model_.reverb(), - delay_state_.DirectPathFilterDelays()[0], + echo_audibility_.Update(render_buffer, avg_render_reverb_.reverb(), + delay_state_.MinDirectPathFilterDelay(), delay_state_.ExternalDelayReported()); } @@ -229,17 +229,15 @@ void AecState::Update( erle_estimator_.Reset(false); } - // TODO(bugs.webrtc.org/10913): Take all channels into account. - const auto& X2 = - render_buffer.Spectrum(delay_state_.DirectPathFilterDelays()[0], - /*channel=*/0); - const auto& X2_input_erle = X2_reverb; - erle_estimator_.Update(render_buffer, adaptive_filter_frequency_responses[0], - X2_input_erle, Y2[0], E2_main[0], + avg_render_spectrum_with_reverb, Y2[0], E2_main[0], subtractor_output_analyzers_[0].ConvergedFilter(), config_.erle.onset_detection); + // TODO(bugs.webrtc.org/10913): Take all channels into account. + const auto& X2 = + render_buffer.Spectrum(delay_state_.MinDirectPathFilterDelay(), + /*channel=*/0); erl_estimator_.Update(subtractor_output_analyzers_[0].ConvergedFilter(), X2, Y2[0]); @@ -357,6 +355,9 @@ void AecState::FilterDelay::Update( analyzer_filter_delay_estimates_blocks.end(), filter_delays_blocks_.begin()); } + + min_filter_delay_ = *std::min_element(filter_delays_blocks_.begin(), + filter_delays_blocks_.end()); } AecState::TransparentMode::TransparentMode(const EchoCanceller3Config& config) diff --git a/modules/audio_processing/aec3/aec_state.h b/modules/audio_processing/aec3/aec_state.h index 79fe13e431..71000b4328 100644 --- a/modules/audio_processing/aec3/aec_state.h +++ b/modules/audio_processing/aec3/aec_state.h @@ -91,8 +91,8 @@ class AecState { float ErlTimeDomain() const { return erl_estimator_.ErlTimeDomain(); } // Returns the delay estimate based on the linear filter. - int FilterDelayBlocks() const { - return delay_state_.DirectPathFilterDelays()[0]; + int MinDirectPathFilterDelay() const { + return delay_state_.MinDirectPathFilterDelay(); } // Returns whether the capture signal is saturated. @@ -194,6 +194,10 @@ class AecState { return filter_delays_blocks_; } + // Returns the minimum delay among the direct path delays relative to the + // beginning of the filter + int MinDirectPathFilterDelay() const { return min_filter_delay_; } + // Updates the delay estimates based on new data. void Update( rtc::ArrayView analyzer_filter_delay_estimates_blocks, @@ -204,6 +208,7 @@ class AecState { const int delay_headroom_samples_; bool external_delay_reported_ = false; std::vector filter_delays_blocks_; + int min_filter_delay_ = 0; absl::optional external_delay_; } delay_state_; @@ -308,7 +313,7 @@ class AecState { absl::optional external_delay_; EchoAudibility echo_audibility_; ReverbModelEstimator reverb_model_estimator_; - ReverbModel reverb_model_; + ReverbModel avg_render_reverb_; std::vector subtractor_output_analyzers_; }; diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc index c5341080b4..db62236c11 100644 --- a/modules/audio_processing/aec3/echo_audibility.cc +++ b/modules/audio_processing/aec3/echo_audibility.cc @@ -29,18 +29,16 @@ EchoAudibility::EchoAudibility(bool use_render_stationarity_at_init) EchoAudibility::~EchoAudibility() = default; -void EchoAudibility::Update( - const RenderBuffer& render_buffer, - rtc::ArrayView render_reverb_contribution_spectrum, - int delay_blocks, - bool external_delay_seen) { +void EchoAudibility::Update(const RenderBuffer& render_buffer, + rtc::ArrayView average_reverb, + int delay_blocks, + bool external_delay_seen) { UpdateRenderNoiseEstimator(render_buffer.GetSpectrumBuffer(), render_buffer.GetBlockBuffer(), external_delay_seen); if (external_delay_seen || use_render_stationarity_at_init_) { - UpdateRenderStationarityFlags( - render_buffer, render_reverb_contribution_spectrum, delay_blocks); + UpdateRenderStationarityFlags(render_buffer, average_reverb, delay_blocks); } } @@ -52,18 +50,17 @@ void EchoAudibility::Reset() { void EchoAudibility::UpdateRenderStationarityFlags( const RenderBuffer& render_buffer, - rtc::ArrayView render_reverb_contribution_spectrum, - int delay_blocks) { + rtc::ArrayView average_reverb, + int min_channel_delay_blocks) { const SpectrumBuffer& spectrum_buffer = render_buffer.GetSpectrumBuffer(); - int idx_at_delay = - spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks); + int idx_at_delay = spectrum_buffer.OffsetIndex(spectrum_buffer.read, + min_channel_delay_blocks); - int num_lookahead = render_buffer.Headroom() - delay_blocks + 1; + int num_lookahead = render_buffer.Headroom() - min_channel_delay_blocks + 1; num_lookahead = std::max(0, num_lookahead); - render_stationarity_.UpdateStationarityFlags( - spectrum_buffer, render_reverb_contribution_spectrum, idx_at_delay, - num_lookahead); + render_stationarity_.UpdateStationarityFlags(spectrum_buffer, average_reverb, + idx_at_delay, num_lookahead); } void EchoAudibility::UpdateRenderNoiseEstimator( @@ -83,14 +80,15 @@ void EchoAudibility::UpdateRenderNoiseEstimator( for (int idx = render_spectrum_write_prev_.value(); idx != render_spectrum_write_current; idx = spectrum_buffer.DecIndex(idx)) { - render_stationarity_.UpdateNoiseEstimator( - spectrum_buffer.buffer[idx][/*channel=*/0]); + render_stationarity_.UpdateNoiseEstimator(spectrum_buffer.buffer[idx]); } } render_spectrum_write_prev_ = render_spectrum_write_current; } bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) { + const int num_render_channels = + static_cast(block_buffer.buffer[0].size()); bool too_low = false; const int render_block_write_current = block_buffer.write; if (render_block_write_current == render_block_write_prev_) { @@ -98,10 +96,16 @@ bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) { } else { for (int idx = render_block_write_prev_; idx != render_block_write_current; idx = block_buffer.IncIndex(idx)) { - auto block = block_buffer.buffer[idx][0][0]; - auto r = std::minmax_element(block.cbegin(), block.cend()); - float max_abs = std::max(std::fabs(*r.first), std::fabs(*r.second)); - if (max_abs < 10) { + float max_abs_over_channels = 0.f; + for (int ch = 0; ch < num_render_channels; ++ch) { + auto block = block_buffer.buffer[idx][0][ch]; + auto r = std::minmax_element(block.cbegin(), block.cend()); + float max_abs_channel = + std::max(std::fabs(*r.first), std::fabs(*r.second)); + max_abs_over_channels = + std::max(max_abs_over_channels, max_abs_channel); + } + if (max_abs_over_channels < 10.f) { too_low = true; // Discards all blocks if one of them is too low. break; } diff --git a/modules/audio_processing/aec3/echo_audibility.h b/modules/audio_processing/aec3/echo_audibility.h index 0152ea474f..1ffc017b7d 100644 --- a/modules/audio_processing/aec3/echo_audibility.h +++ b/modules/audio_processing/aec3/echo_audibility.h @@ -28,10 +28,13 @@ class EchoAudibility { explicit EchoAudibility(bool use_render_stationarity_at_init); ~EchoAudibility(); + EchoAudibility(const EchoAudibility&) = delete; + EchoAudibility& operator=(const EchoAudibility&) = delete; + // Feed new render data to the echo audibility estimator. void Update(const RenderBuffer& render_buffer, - rtc::ArrayView render_reverb_contribution_spectrum, - int delay_blocks, + rtc::ArrayView average_reverb, + int min_channel_delay_blocks, bool external_delay_seen); // Get the residual echo scaling. void GetResidualEchoScaling(bool filter_has_had_time_to_converge, @@ -57,10 +60,9 @@ class EchoAudibility { void Reset(); // Updates the render stationarity flags for the current frame. - void UpdateRenderStationarityFlags( - const RenderBuffer& render_buffer, - rtc::ArrayView render_reverb_contribution_spectrum, - int delay_blocks); + void UpdateRenderStationarityFlags(const RenderBuffer& render_buffer, + rtc::ArrayView average_reverb, + int delay_blocks); // Updates the noise estimator with the new render data since the previous // call to this method. @@ -77,7 +79,6 @@ class EchoAudibility { bool non_zero_render_seen_; const bool use_render_stationarity_at_init_; StationarityEstimator render_stationarity_; - RTC_DISALLOW_COPY_AND_ASSIGN(EchoAudibility); }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index e6f17c7e32..7cec47cca0 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -356,7 +356,7 @@ void EchoRemoverImpl::ProcessCapture( // Analyze the render signal. render_signal_analyzer_.Update(*render_buffer, - aec_state_.FilterDelayBlocks()); + aec_state_.MinDirectPathFilterDelay()); // State transition. if (aec_state_.TransitionTriggered()) { @@ -457,10 +457,11 @@ void EchoRemoverImpl::ProcessCapture( data_dumper_->DumpRaw("aec3_S2_linear", S2_linear[0]); data_dumper_->DumpRaw("aec3_Y2", Y2[0]); data_dumper_->DumpRaw( - "aec3_X2", - render_buffer->Spectrum(aec_state_.FilterDelayBlocks(), /*channel=*/0)); + "aec3_X2", render_buffer->Spectrum(aec_state_.MinDirectPathFilterDelay(), + /*channel=*/0)); data_dumper_->DumpRaw("aec3_R2", R2[0]); - data_dumper_->DumpRaw("aec3_filter_delay", aec_state_.FilterDelayBlocks()); + data_dumper_->DumpRaw("aec3_filter_delay", + aec_state_.MinDirectPathFilterDelay()); data_dumper_->DumpRaw("aec3_capture_saturation", aec_state_.SaturatedCapture() ? 1 : 0); } diff --git a/modules/audio_processing/aec3/echo_remover_metrics.cc b/modules/audio_processing/aec3/echo_remover_metrics.cc index 4ab05f804b..69d2252c2b 100644 --- a/modules/audio_processing/aec3/echo_remover_metrics.cc +++ b/modules/audio_processing/aec3/echo_remover_metrics.cc @@ -241,7 +241,8 @@ void EchoRemoverMetrics::Update( static_cast( active_render_count_ > kMetricsCollectionBlocksBy2 ? 1 : 0)); RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.FilterDelay", - aec_state.FilterDelayBlocks(), 0, 30, 31); + aec_state.MinDirectPathFilterDelay(), 0, 30, + 31); RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.EchoCanceller.CaptureSaturation", static_cast(saturated_capture_ ? 1 : 0)); break; diff --git a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc index 92334c21b7..fa3b263502 100644 --- a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc +++ b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc @@ -150,7 +150,7 @@ void RunFilterUpdateTest(int num_blocks_to_process, render_delay_buffer->PrepareCaptureProcessing(); render_signal_analyzer.Update(*render_delay_buffer->GetRenderBuffer(), - aec_state.FilterDelayBlocks()); + aec_state.MinDirectPathFilterDelay()); // Apply the main filter. main_filter.Filter(*render_delay_buffer->GetRenderBuffer(), &S); diff --git a/modules/audio_processing/aec3/residual_echo_estimator.cc b/modules/audio_processing/aec3/residual_echo_estimator.cc index 07197e3d3a..6a8a3f290c 100644 --- a/modules/audio_processing/aec3/residual_echo_estimator.cc +++ b/modules/audio_processing/aec3/residual_echo_estimator.cc @@ -203,7 +203,7 @@ void ResidualEchoEstimator::Estimate( std::array X2; EchoGeneratingPower(num_render_channels_, render_buffer.GetSpectrumBuffer(), config_.echo_model, - aec_state.FilterDelayBlocks(), X2); + aec_state.MinDirectPathFilterDelay(), X2); if (!aec_state.UseStationarityProperties()) { ApplyNoiseGate(config_.echo_model, X2); } @@ -288,9 +288,10 @@ void ResidualEchoEstimator::AddReverb( const size_t num_capture_channels = R2.size(); // Choose reverb partition based on what type of echo power model is used. - const size_t first_reverb_partition = reverb_type == ReverbType::kLinear - ? aec_state.FilterLengthBlocks() + 1 - : aec_state.FilterDelayBlocks() + 1; + const size_t first_reverb_partition = + reverb_type == ReverbType::kLinear + ? aec_state.FilterLengthBlocks() + 1 + : aec_state.MinDirectPathFilterDelay() + 1; // Compute render power for the reverb. std::array render_power_data; diff --git a/modules/audio_processing/aec3/stationarity_estimator.cc b/modules/audio_processing/aec3/stationarity_estimator.cc index 080d13d9bf..d0c3c9ca34 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.cc +++ b/modules/audio_processing/aec3/stationarity_estimator.cc @@ -45,7 +45,7 @@ void StationarityEstimator::Reset() { // Update just the noise estimator. Usefull until the delay is known void StationarityEstimator::UpdateNoiseEstimator( - rtc::ArrayView spectrum) { + rtc::ArrayView> spectrum) { noise_.Update(spectrum); data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum()); data_dumper_->DumpRaw("aec3_stationarity_is_block_stationary", @@ -99,15 +99,20 @@ bool StationarityEstimator::IsBlockStationary() const { bool StationarityEstimator::EstimateBandStationarity( const SpectrumBuffer& spectrum_buffer, - rtc::ArrayView reverb, + rtc::ArrayView average_reverb, const std::array& indexes, size_t band) const { constexpr float kThrStationarity = 10.f; float acum_power = 0.f; + const int num_render_channels = + static_cast(spectrum_buffer.buffer[0].size()); + const float one_by_num_channels = 1.f / num_render_channels; for (auto idx : indexes) { - acum_power += spectrum_buffer.buffer[idx][/*channel=*/0][band]; + for (int ch = 0; ch < num_render_channels; ++ch) { + acum_power += spectrum_buffer.buffer[idx][ch][band] * one_by_num_channels; + } } - acum_power += reverb[band]; + acum_power += average_reverb[band]; float noise = kWindowLength * GetStationarityPowerBand(band); RTC_CHECK_LT(0.f, noise); bool stationary = acum_power < kThrStationarity * noise; @@ -163,16 +168,42 @@ void StationarityEstimator::NoiseSpectrum::Reset() { } void StationarityEstimator::NoiseSpectrum::Update( - rtc::ArrayView spectrum) { - RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum.size()); + rtc::ArrayView> spectrum) { + RTC_DCHECK_LE(1, spectrum[0].size()); + const int num_render_channels = static_cast(spectrum.size()); + + std::array avg_spectrum_data; + rtc::ArrayView avg_spectrum; + RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum[0].size()); + if (num_render_channels == 1) { + avg_spectrum = spectrum[0]; + } else { + // For multiple channels, average the channel spectra before passing to the + // noise spectrum estimator. + avg_spectrum = avg_spectrum_data; + std::copy(spectrum[0].begin(), spectrum[0].end(), + avg_spectrum_data.begin()); + for (int ch = 1; ch < num_render_channels; ++ch) { + RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum[ch].size()); + for (size_t k = 1; k < kFftLengthBy2Plus1; ++k) { + avg_spectrum_data[k] += spectrum[ch][k]; + } + } + + const float one_by_num_channels = 1.f / num_render_channels; + for (size_t k = 1; k < kFftLengthBy2Plus1; ++k) { + avg_spectrum_data[k] *= one_by_num_channels; + } + } + ++block_counter_; float alpha = GetAlpha(); - for (size_t k = 0; k < spectrum.size(); ++k) { + for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { if (block_counter_ <= kNBlocksAverageInitPhase) { - noise_spectrum_[k] += (1.f / kNBlocksAverageInitPhase) * spectrum[k]; + noise_spectrum_[k] += (1.f / kNBlocksAverageInitPhase) * avg_spectrum[k]; } else { noise_spectrum_[k] = - UpdateBandBySmoothing(spectrum[k], noise_spectrum_[k], alpha); + UpdateBandBySmoothing(avg_spectrum[k], noise_spectrum_[k], alpha); } } } diff --git a/modules/audio_processing/aec3/stationarity_estimator.h b/modules/audio_processing/aec3/stationarity_estimator.h index 504fea7138..5860ef1625 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.h +++ b/modules/audio_processing/aec3/stationarity_estimator.h @@ -15,6 +15,7 @@ #include #include +#include #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" // kFftLengthBy2Plus1... @@ -35,7 +36,7 @@ class StationarityEstimator { void Reset(); // Update just the noise estimator. Usefull until the delay is known - void UpdateNoiseEstimator(rtc::ArrayView spectrum); + void UpdateNoiseEstimator(rtc::ArrayView> spectrum); // Update the flag indicating whether this current frame is stationary. For // getting a more robust estimation, it looks at future and/or past frames. @@ -61,7 +62,7 @@ class StationarityEstimator { // Get an estimation of the stationarity for the current band by looking // at the past/present/future available data. bool EstimateBandStationarity(const SpectrumBuffer& spectrum_buffer, - rtc::ArrayView reverb, + rtc::ArrayView average_reverb, const std::array& indexes, size_t band) const; @@ -85,7 +86,7 @@ class StationarityEstimator { void Reset(); // Update the noise power spectrum with a new frame. - void Update(rtc::ArrayView spectrum); + void Update(rtc::ArrayView> spectrum); // Get the noise estimation power spectrum. rtc::ArrayView Spectrum() const { return noise_spectrum_; } diff --git a/modules/audio_processing/aec3/subtractor_unittest.cc b/modules/audio_processing/aec3/subtractor_unittest.cc index 05faa4fd70..b59fa7b5e8 100644 --- a/modules/audio_processing/aec3/subtractor_unittest.cc +++ b/modules/audio_processing/aec3/subtractor_unittest.cc @@ -135,7 +135,7 @@ std::vector RunSubtractorTest( } render_delay_buffer->PrepareCaptureProcessing(); render_signal_analyzer.Update(*render_delay_buffer->GetRenderBuffer(), - aec_state.FilterDelayBlocks()); + aec_state.MinDirectPathFilterDelay()); // Handle echo path changes. if (std::find(blocks_with_echo_path_changes.begin(),