diff --git a/api/audio/echo_canceller3_config.h b/api/audio/echo_canceller3_config.h index a58362ebec..ed39b66420 100644 --- a/api/audio/echo_canceller3_config.h +++ b/api/audio/echo_canceller3_config.h @@ -65,7 +65,8 @@ struct EchoCanceller3Config { float lf = 1.f; float mf = 1.f; float hf = 1.f; - float default_len = 0.f; + float default_len = 0.7f; + bool reverb_based_on_render = true; bool echo_can_saturate = true; bool bounded_erl = false; } ep_strength; diff --git a/modules/audio_processing/aec3/BUILD.gn b/modules/audio_processing/aec3/BUILD.gn index 21cb63c9ac..15ca70a688 100644 --- a/modules/audio_processing/aec3/BUILD.gn +++ b/modules/audio_processing/aec3/BUILD.gn @@ -81,6 +81,10 @@ rtc_static_library("aec3") { "render_signal_analyzer.h", "residual_echo_estimator.cc", "residual_echo_estimator.h", + "reverb_model.cc", + "reverb_model.h", + "reverb_model_fallback.cc", + "reverb_model_fallback.h", "shadow_filter_update_gain.cc", "shadow_filter_update_gain.h", "skew_estimator.cc", diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index 516f6abc3f..5fe8276b3d 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -165,8 +165,9 @@ void AecState::Update( if (UseStationaryProperties()) { // Update the echo audibility evaluator. - echo_audibility_.Update(render_buffer, FilterDelayBlocks(), - external_delay_seen_); + echo_audibility_.Update( + render_buffer, FilterDelayBlocks(), external_delay_seen_, + config_.ep_strength.reverb_based_on_render ? ReverbDecay() : 0.f); } // Update the ERL and ERLE measures. @@ -272,6 +273,8 @@ void AecState::Update( use_linear_filter_output_ = usable_linear_estimate_ && !TransparentMode(); + UpdateReverb(adaptive_filter_impulse_response); + data_dumper_->DumpRaw("aec3_erle", Erle()); data_dumper_->DumpRaw("aec3_erle_onset", erle_estimator_.ErleOnsets()); data_dumper_->DumpRaw("aec3_erl", Erl()); @@ -304,11 +307,12 @@ void AecState::Update( recently_converged_filter); data_dumper_->DumpRaw("aec3_suppresion_gain_limiter_running", IsSuppressionGainLimitActive()); + data_dumper_->DumpRaw("aec3_filter_tail_energy", GetFilterTailGain()); } void AecState::UpdateReverb(const std::vector& impulse_response) { // Echo tail estimation enabled if the below variable is set as negative. - if (config_.ep_strength.default_len > 0.f) { + if (config_.ep_strength.default_len >= 0.f) { return; } @@ -438,7 +442,7 @@ void AecState::UpdateReverb(const std::vector& impulse_response) { const float N = num_reverb_decay_sections_ * kFftLengthBy2; accumulated_nz_ = 0.f; const float k1By12 = 1.f / 12.f; - // Arithmetic sum $2 \sum_{i=0}^{(N-1)/2}i^2$ calculated directly. + // Arithmetic sum $2 \sum_{i=0.5}^{(N-1)/2}i^2$ calculated directly. accumulated_nn_ = N * (N * N - 1.0f) * k1By12; accumulated_count_ = -N * 0.5f; // Linear regression approach assumes symmetric index around 0. diff --git a/modules/audio_processing/aec3/aec_state.h b/modules/audio_processing/aec3/aec_state.h index d513a4084d..710aedbe58 100644 --- a/modules/audio_processing/aec3/aec_state.h +++ b/modules/audio_processing/aec3/aec_state.h @@ -136,6 +136,14 @@ class AecState { const std::array& Y2, const std::array& s); + // Returns the gain at the tail of the linear filter. + float GetFilterTailGain() const { return filter_analyzer_.GetTailGain(); } + + // Returns filter length in blocks. + int FilterLengthBlocks() const { + return filter_analyzer_.FilterLengthBlocks(); + } + private: void UpdateReverb(const std::vector& impulse_response); bool DetectActiveRender(rtc::ArrayView x) const; @@ -194,7 +202,6 @@ class AecState { bool finite_erl_ = false; size_t active_blocks_since_converged_filter_ = 0; EchoAudibility echo_audibility_; - RTC_DISALLOW_COPY_AND_ASSIGN(AecState); }; diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc index 68d2dd11e6..a504cca4be 100644 --- a/modules/audio_processing/aec3/echo_audibility.cc +++ b/modules/audio_processing/aec3/echo_audibility.cc @@ -28,13 +28,14 @@ EchoAudibility::~EchoAudibility() = default; void EchoAudibility::Update(const RenderBuffer& render_buffer, int delay_blocks, - bool external_delay_seen) { + bool external_delay_seen, + float reverb_decay) { UpdateRenderNoiseEstimator(render_buffer.GetSpectrumBuffer(), render_buffer.GetBlockBuffer(), external_delay_seen); if (external_delay_seen) { - UpdateRenderStationarityFlags(render_buffer, delay_blocks); + UpdateRenderStationarityFlags(render_buffer, delay_blocks, reverb_decay); } } @@ -46,7 +47,8 @@ void EchoAudibility::Reset() { void EchoAudibility::UpdateRenderStationarityFlags( const RenderBuffer& render_buffer, - int delay_blocks) { + int delay_blocks, + float reverb_decay) { const VectorBuffer& spectrum_buffer = render_buffer.GetSpectrumBuffer(); int idx_at_delay = spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks); @@ -55,7 +57,7 @@ void EchoAudibility::UpdateRenderStationarityFlags( num_lookahead = std::max(0, num_lookahead); render_stationarity_.UpdateStationarityFlags(spectrum_buffer, idx_at_delay, - num_lookahead); + num_lookahead, reverb_decay); } void EchoAudibility::UpdateRenderNoiseEstimator( diff --git a/modules/audio_processing/aec3/echo_audibility.h b/modules/audio_processing/aec3/echo_audibility.h index 038951e987..58a6599a37 100644 --- a/modules/audio_processing/aec3/echo_audibility.h +++ b/modules/audio_processing/aec3/echo_audibility.h @@ -37,7 +37,8 @@ class EchoAudibility { // Feed new render data to the echo audibility estimator. void Update(const RenderBuffer& render_buffer, int delay_blocks, - bool external_delay_seen); + bool external_delay_seen, + float reverb_decay); // Get the residual echo scaling. void GetResidualEchoScaling(rtc::ArrayView residual_scaling) const { @@ -56,7 +57,8 @@ class EchoAudibility { // Updates the render stationarity flags for the current frame. void UpdateRenderStationarityFlags(const RenderBuffer& render_buffer, - int delay_blocks); + int delay_blocks, + float reverb_decay); // Updates the noise estimator with the new render data since the previous // call to this method. diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index 07ee54f1b5..3acb31b259 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -33,10 +33,22 @@ bool UseShortDelayEstimatorWindow() { return field_trial::IsEnabled("WebRTC-Aec3UseShortDelayEstimatorWindow"); } +bool EnableReverbBasedOnRender() { + return !field_trial::IsEnabled("WebRTC-Aec3ReverbBasedOnRenderKillSwitch"); +} + +bool EnableReverbModelling() { + return !field_trial::IsEnabled("WebRTC-Aec3ReverbModellingKillSwitch"); +} + // Method for adjusting config parameter dependencies.. EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { EchoCanceller3Config adjusted_cfg = config; + if (!EnableReverbModelling()) { + adjusted_cfg.ep_strength.default_len = 0.f; + } + // Use customized parameters when the system has clock-drift. if (config.echo_removal_control.has_clock_drift) { RTC_LOG(LS_WARNING) @@ -73,6 +85,10 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { std::min(adjusted_cfg.delay.num_filters, static_cast(5)); } + if (EnableReverbBasedOnRender() == false) { + adjusted_cfg.ep_strength.reverb_based_on_render = false; + } + return adjusted_cfg; } diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index df34e861b9..a2d5e362e7 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -276,6 +276,8 @@ void EchoRemoverImpl::ProcessCapture( data_dumper_->DumpRaw( "aec3_X2", render_buffer->Spectrum(aec_state_.FilterDelayBlocks())); data_dumper_->DumpRaw("aec3_R2", R2); + data_dumper_->DumpRaw("aec3_R2_reverb", + residual_echo_estimator_.GetReverbPowerSpectrum()); data_dumper_->DumpRaw("aec3_filter_delay", aec_state_.FilterDelayBlocks()); data_dumper_->DumpRaw("aec3_capture_saturation", aec_state_.SaturatedCapture() ? 1 : 0); diff --git a/modules/audio_processing/aec3/filter_analyzer.cc b/modules/audio_processing/aec3/filter_analyzer.cc index fd78f2a8ee..ce0ddbbd69 100644 --- a/modules/audio_processing/aec3/filter_analyzer.cc +++ b/modules/audio_processing/aec3/filter_analyzer.cc @@ -15,6 +15,7 @@ #include #include +#include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/atomicops.h" #include "rtc_base/checks.h" @@ -55,7 +56,8 @@ FilterAnalyzer::FilterAnalyzer(const EchoCanceller3Config& config) active_render_threshold_(config.render_levels.active_render_limit * config.render_levels.active_render_limit * kFftLengthBy2), - h_highpass_(GetTimeDomainLength(config.filter.main.length_blocks), 0.f) { + h_highpass_(GetTimeDomainLength(config.filter.main.length_blocks), 0.f), + filter_length_blocks_(config.filter.main_initial.length_blocks) { Reset(); } @@ -141,6 +143,8 @@ void FilterAnalyzer::Update(rtc::ArrayView filter_time_domain, consistent_estimate_ = consistent_estimate_counter_ > 1.5f * kNumBlocksPerSecond; + UpdateFilterTailGain(filter_time_domain); + filter_length_blocks_ = filter_time_domain.size() * (1.f / kBlockSize); } void FilterAnalyzer::UpdateFilterGain( @@ -162,4 +166,24 @@ void FilterAnalyzer::UpdateFilterGain( } } +/* Estimates a bound of the contributions of the filter tail to the + * energy of the echo signal. The estimation is done as the maximum + * energy of the impulse response at the tail times the number of + * coefficients used for describing the tail (kFftLengthBy2 in this case). */ + +void FilterAnalyzer::UpdateFilterTailGain( + rtc::ArrayView filter_time_domain) { + float tail_max_energy = 0.f; + + const auto& h = filter_time_domain; + RTC_DCHECK_GE(h.size(), kFftLengthBy2); + for (size_t k = h.size() - kFftLengthBy2; k < h.size(); ++k) { + tail_max_energy = std::max(tail_max_energy, h[k] * h[k]); + } + + tail_max_energy *= kFftLengthBy2; + + tail_gain_ += 0.1f * (tail_max_energy - tail_gain_); +} + } // namespace webrtc diff --git a/modules/audio_processing/aec3/filter_analyzer.h b/modules/audio_processing/aec3/filter_analyzer.h index d7d568da08..e47606cc93 100644 --- a/modules/audio_processing/aec3/filter_analyzer.h +++ b/modules/audio_processing/aec3/filter_analyzer.h @@ -48,11 +48,21 @@ class FilterAnalyzer { // Returns the estimated filter gain. float Gain() const { return gain_; } + // Returns the estimated energy gain at the tail of the filter. + float GetTailGain() const { return tail_gain_; } + + // Returns the number of blocks for the current used filter. + float FilterLengthBlocks() const { return filter_length_blocks_; } + private: void UpdateFilterGain(rtc::ArrayView filter_time_domain, size_t max_index); void PreProcessFilter(rtc::ArrayView filter_time_domain); + // Updates the estimation of the energy gain that the linear filter + // is applying at its tail. + void UpdateFilterTailGain(rtc::ArrayView filter_time_domain); + static int instance_count_; std::unique_ptr data_dumper_; const bool use_preprocessed_filter_; @@ -66,7 +76,8 @@ class FilterAnalyzer { size_t consistent_estimate_counter_ = 0; int consistent_delay_reference_ = -10; float gain_; - + float tail_gain_ = 0; + int filter_length_blocks_; RTC_DISALLOW_COPY_AND_ASSIGN(FilterAnalyzer); }; diff --git a/modules/audio_processing/aec3/residual_echo_estimator.cc b/modules/audio_processing/aec3/residual_echo_estimator.cc index 94d12d491a..85a15e1a50 100644 --- a/modules/audio_processing/aec3/residual_echo_estimator.cc +++ b/modules/audio_processing/aec3/residual_echo_estimator.cc @@ -14,6 +14,8 @@ #include #include +#include "modules/audio_processing/aec3/reverb_model.h" +#include "modules/audio_processing/aec3/reverb_model_fallback.h" #include "rtc_base/checks.h" #include "system_wrappers/include/field_trial.h" @@ -71,9 +73,14 @@ void GetRenderIndexesToAnalyze( ResidualEchoEstimator::ResidualEchoEstimator(const EchoCanceller3Config& config) : config_(config), - S2_old_(config_.filter.main.length_blocks), soft_transparent_mode_(EnableSoftTransparentMode()), override_estimated_echo_path_gain_(OverrideEstimatedEchoPathGain()) { + if (config_.ep_strength.reverb_based_on_render) { + echo_reverb_.reset(new ReverbModel()); + } else { + echo_reverb_fallback.reset( + new ReverbModelFallback(config_.filter.main.length_blocks)); + } Reset(); } @@ -94,8 +101,19 @@ void ResidualEchoEstimator::Estimate( if (aec_state.UsableLinearEstimate()) { RTC_DCHECK(!aec_state.SaturatedEcho()); LinearEstimate(S2_linear, aec_state.Erle(), R2); - AddEchoReverb(S2_linear, aec_state.FilterDelayBlocks(), - aec_state.ReverbDecay(), R2); + // Adds the estimated unmodelled echo power to the residual echo power + // estimate. + if (echo_reverb_) { + echo_reverb_->AddReverb( + render_buffer.Spectrum(aec_state.FilterLengthBlocks() + 1), + aec_state.GetFilterTailGain(), aec_state.ReverbDecay(), *R2); + } else { + RTC_DCHECK(echo_reverb_fallback); + echo_reverb_fallback->AddEchoReverb(S2_linear, + aec_state.FilterDelayBlocks(), + aec_state.ReverbDecay(), R2); + } + } else { // Estimate the echo generating signal power. std::array X2; @@ -131,8 +149,18 @@ void ResidualEchoEstimator::Estimate( R2->fill((*std::max_element(R2->begin(), R2->end())) * 100.f); } - AddEchoReverb(*R2, config_.filter.main.length_blocks, - aec_state.ReverbDecay(), R2); + if (!(aec_state.TransparentMode() && soft_transparent_mode_)) { + if (echo_reverb_) { + echo_reverb_->AddReverb( + render_buffer.Spectrum(aec_state.FilterDelayBlocks() + 1), + echo_path_gain * echo_path_gain, aec_state.ReverbDecay(), *R2); + } else { + RTC_DCHECK(echo_reverb_fallback); + echo_reverb_fallback->AddEchoReverb(*R2, + config_.filter.main.length_blocks, + aec_state.ReverbDecay(), R2); + } + } } if (aec_state.UseStationaryProperties()) { @@ -159,14 +187,16 @@ void ResidualEchoEstimator::Estimate( } void ResidualEchoEstimator::Reset() { + if (echo_reverb_) { + echo_reverb_->Reset(); + } else { + RTC_DCHECK(echo_reverb_fallback); + echo_reverb_fallback->Reset(); + } X2_noise_floor_counter_.fill(config_.echo_model.noise_floor_hold); X2_noise_floor_.fill(config_.echo_model.min_noise_floor_power); - R2_reverb_.fill(0.f); R2_old_.fill(0.f); R2_hold_counter_.fill(0.f); - for (auto& S2_k : S2_old_) { - S2_k.fill(0.f); - } } void ResidualEchoEstimator::LinearEstimate( @@ -207,41 +237,6 @@ void ResidualEchoEstimator::NonLinearEstimate( } } -void ResidualEchoEstimator::AddEchoReverb( - const std::array& S2, - size_t delay, - float reverb_decay_factor, - std::array* R2) { - // Compute the decay factor for how much the echo has decayed before leaving - // the region covered by the linear model. - auto integer_power = [](float base, int exp) { - float result = 1.f; - for (int k = 0; k < exp; ++k) { - result *= base; - } - return result; - }; - RTC_DCHECK_LE(delay, S2_old_.size()); - const float reverb_decay_for_delay = - integer_power(reverb_decay_factor, S2_old_.size() - delay); - - // Update the estimate of the reverberant residual echo power. - S2_old_index_ = S2_old_index_ > 0 ? S2_old_index_ - 1 : S2_old_.size() - 1; - const auto& S2_end = S2_old_[S2_old_index_]; - std::transform( - S2_end.begin(), S2_end.end(), R2_reverb_.begin(), R2_reverb_.begin(), - [reverb_decay_for_delay, reverb_decay_factor](float a, float b) { - return (b + a * reverb_decay_for_delay) * reverb_decay_factor; - }); - - // Update the buffer of old echo powers. - std::copy(S2.begin(), S2.end(), S2_old_[S2_old_index_].begin()); - - // Add the power of the echo reverb to the residual echo power. - std::transform(R2->begin(), R2->end(), R2_reverb_.begin(), R2->begin(), - std::plus()); -} - void ResidualEchoEstimator::EchoGeneratingPower( const VectorBuffer& spectrum_buffer, const EchoCanceller3Config::EchoModel& echo_model, diff --git a/modules/audio_processing/aec3/residual_echo_estimator.h b/modules/audio_processing/aec3/residual_echo_estimator.h index 79699fcfc1..5e2affe11b 100644 --- a/modules/audio_processing/aec3/residual_echo_estimator.h +++ b/modules/audio_processing/aec3/residual_echo_estimator.h @@ -13,6 +13,7 @@ #include #include +#include #include #include "api/array_view.h" @@ -20,6 +21,8 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec_state.h" #include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/reverb_model.h" +#include "modules/audio_processing/aec3/reverb_model_fallback.h" #include "rtc_base/constructormagic.h" namespace webrtc { @@ -35,6 +38,16 @@ class ResidualEchoEstimator { const std::array& Y2, std::array* R2); + // Returns the reverberant power spectrum contributions to the echo residual. + const std::array& GetReverbPowerSpectrum() const { + if (echo_reverb_) { + return echo_reverb_->GetPowerSpectrum(); + } else { + RTC_DCHECK(echo_reverb_fallback); + return echo_reverb_fallback->GetPowerSpectrum(); + } + } + private: // Resets the state. void Reset(); @@ -52,12 +65,6 @@ class ResidualEchoEstimator { const std::array& Y2, std::array* R2); - // Adds the estimated unmodelled echo power to the residual echo power - // estimate. - void AddEchoReverb(const std::array& S2, - size_t delay, - float reverb_decay_factor, - std::array* R2); // Estimates the echo generating signal power as gated maximal power over a // time window. @@ -79,14 +86,12 @@ class ResidualEchoEstimator { const EchoCanceller3Config config_; std::array R2_old_; std::array R2_hold_counter_; - std::array R2_reverb_; - int S2_old_index_ = 0; - std::vector> S2_old_; std::array X2_noise_floor_; std::array X2_noise_floor_counter_; const bool soft_transparent_mode_; const bool override_estimated_echo_path_gain_; - + std::unique_ptr echo_reverb_; + std::unique_ptr echo_reverb_fallback; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(ResidualEchoEstimator); }; diff --git a/modules/audio_processing/aec3/reverb_model.cc b/modules/audio_processing/aec3/reverb_model.cc new file mode 100644 index 0000000000..5daca74c8f --- /dev/null +++ b/modules/audio_processing/aec3/reverb_model.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/reverb_model.h" + +#include + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" + +namespace webrtc { + +ReverbModel::ReverbModel() { + Reset(); +} + +ReverbModel::~ReverbModel() = default; + +void ReverbModel::Reset() { + reverb_.fill(0.); +} + +void ReverbModel::UpdateReverbContributions( + rtc::ArrayView power_spectrum, + float power_spectrum_scaling, + float reverb_decay) { + if (reverb_decay > 0) { + // Update the estimate of the reverberant power. + std::transform(power_spectrum.begin(), power_spectrum.end(), + reverb_.begin(), reverb_.begin(), + [reverb_decay, power_spectrum_scaling](float a, float b) { + return (b + a * power_spectrum_scaling) * reverb_decay; + }); + } +} + +void ReverbModel::AddReverb(rtc::ArrayView power_spectrum, + float power_spectrum_scaling, + float reverb_decay, + rtc::ArrayView reverb_power_spectrum) { + UpdateReverbContributions(power_spectrum, power_spectrum_scaling, + reverb_decay); + + // Add the power of the echo reverb to the residual echo power. + std::transform(reverb_power_spectrum.begin(), reverb_power_spectrum.end(), + reverb_.begin(), reverb_power_spectrum.begin(), + std::plus()); +} + +} // namespace webrtc diff --git a/modules/audio_processing/aec3/reverb_model.h b/modules/audio_processing/aec3/reverb_model.h new file mode 100644 index 0000000000..6836c1d529 --- /dev/null +++ b/modules/audio_processing/aec3/reverb_model.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_REVERB_MODEL_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_REVERB_MODEL_H_ + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" + +namespace webrtc { + +// The ReverbModel class describes an exponential reverberant model +// that can be applied over power spectrums. +class ReverbModel { + public: + ReverbModel(); + ~ReverbModel(); + + // Resets the state. + void Reset(); + + // Updates the reverberation contributions. + void UpdateReverbContributions(rtc::ArrayView power_spectrum, + float power_spectrum_scaling, + float reverb_decay); + + // Adds the reverberation contributions to an input/output power spectrum. + // - power_spectrum: Input to the exponential reverberation model. + // - power_spectrum_scaling: A pre-scaling of the power_spectrum used + // before applying the exponential reverberation model. + // - reverb_decay: Parameter used by the expontial reververation model. + void AddReverb(rtc::ArrayView power_spectrum, + float power_spectrum_scaling, + float reverb_decay, + rtc::ArrayView reverb_power_spectrum); + + // Returns the current power spectrum reverberation contributions. + const std::array& GetPowerSpectrum() const { + return reverb_; + } + + private: + std::array reverb_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_REVERB_MODEL_H_ diff --git a/modules/audio_processing/aec3/reverb_model_fallback.cc b/modules/audio_processing/aec3/reverb_model_fallback.cc new file mode 100644 index 0000000000..67df37b3cf --- /dev/null +++ b/modules/audio_processing/aec3/reverb_model_fallback.cc @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/reverb_model_fallback.h" + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +ReverbModelFallback::ReverbModelFallback(size_t length_blocks) + : S2_old_(length_blocks) { + Reset(); +} + +ReverbModelFallback::~ReverbModelFallback() = default; + +void ReverbModelFallback::Reset() { + R2_reverb_.fill(0.f); + for (auto& S2_k : S2_old_) { + S2_k.fill(0.f); + } +} + +void ReverbModelFallback::AddEchoReverb( + const std::array& S2, + size_t delay, + float reverb_decay_factor, + std::array* R2) { + // Compute the decay factor for how much the echo has decayed before leaving + // the region covered by the linear model. + auto integer_power = [](float base, int exp) { + float result = 1.f; + for (int k = 0; k < exp; ++k) { + result *= base; + } + return result; + }; + RTC_DCHECK_LE(delay, S2_old_.size()); + const float reverb_decay_for_delay = + integer_power(reverb_decay_factor, S2_old_.size() - delay); + + // Update the estimate of the reverberant residual echo power. + S2_old_index_ = S2_old_index_ > 0 ? S2_old_index_ - 1 : S2_old_.size() - 1; + const auto& S2_end = S2_old_[S2_old_index_]; + std::transform( + S2_end.begin(), S2_end.end(), R2_reverb_.begin(), R2_reverb_.begin(), + [reverb_decay_for_delay, reverb_decay_factor](float a, float b) { + return (b + a * reverb_decay_for_delay) * reverb_decay_factor; + }); + + // Update the buffer of old echo powers. + std::copy(S2.begin(), S2.end(), S2_old_[S2_old_index_].begin()); + + // Add the power of the echo reverb to the residual echo power. + std::transform(R2->begin(), R2->end(), R2_reverb_.begin(), R2->begin(), + std::plus()); +} + +} // namespace webrtc diff --git a/modules/audio_processing/aec3/reverb_model_fallback.h b/modules/audio_processing/aec3/reverb_model_fallback.h new file mode 100644 index 0000000000..1b2a953ea6 --- /dev/null +++ b/modules/audio_processing/aec3/reverb_model_fallback.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_REVERB_MODEL_FALLBACK_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_REVERB_MODEL_FALLBACK_H_ + +#include +#include + +#include "modules/audio_processing/aec3/aec3_common.h" + +namespace webrtc { + +// The ReverbModelFallback class describes an exponential reverberant model. +// This model is expected to be applied over the echo power spectrum that +// is estimated by the linear filter. + +class ReverbModelFallback { + public: + explicit ReverbModelFallback(size_t length_blocks); + ~ReverbModelFallback(); + + // Resets the state + void Reset(); + + // Adds the estimated unmodelled echo power to the residual echo power + // estimate. + void AddEchoReverb(const std::array& S2, + size_t delay, + float reverb_decay_factor, + std::array* R2); + + // Returns the current power spectrum reverberation contributions. + const std::array& GetPowerSpectrum() const { + return R2_reverb_; + } + + private: + std::array R2_reverb_; + int S2_old_index_ = 0; + std::vector> S2_old_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_REVERB_MODEL_FALLBACK_H_ diff --git a/modules/audio_processing/aec3/stationarity_estimator.cc b/modules/audio_processing/aec3/stationarity_estimator.cc index 8e065d2422..d4e32f56ab 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.cc +++ b/modules/audio_processing/aec3/stationarity_estimator.cc @@ -40,6 +40,7 @@ void StationarityEstimator::Reset() { noise_.Reset(); hangovers_.fill(0); stationarity_flags_.fill(false); + render_reverb_.Reset(); } // Update just the noise estimator. Usefull until the delay is known @@ -52,7 +53,8 @@ void StationarityEstimator::UpdateNoiseEstimator( void StationarityEstimator::UpdateStationarityFlags( const VectorBuffer& spectrum_buffer, int idx_current, - int num_lookahead) { + int num_lookahead, + float reverb_decay) { std::array indexes; int num_lookahead_bounded = std::min(num_lookahead, kWindowLength - 1); int idx = idx_current; @@ -75,9 +77,12 @@ void StationarityEstimator::UpdateStationarityFlags( spectrum_buffer.DecIndex(indexes[kWindowLength - 1]), spectrum_buffer.OffsetIndex(idx_current, -(num_lookahead_bounded + 1))); + int idx_past = spectrum_buffer.IncIndex(idx_current); + render_reverb_.UpdateReverbContributions(spectrum_buffer.buffer[idx_past], 1., + reverb_decay); for (size_t k = 0; k < stationarity_flags_.size(); ++k) { - stationarity_flags_[k] = - EstimateBandStationarity(spectrum_buffer, indexes, k); + stationarity_flags_[k] = EstimateBandStationarity( + spectrum_buffer, render_reverb_.GetPowerSpectrum(), indexes, k); } UpdateHangover(); SmoothStationaryPerFreq(); @@ -86,6 +91,7 @@ void StationarityEstimator::UpdateStationarityFlags( bool StationarityEstimator::EstimateBandStationarity( const VectorBuffer& spectrum_buffer, + const std::array& reverb, const std::array& indexes, size_t band) const { constexpr float kThrStationarity = 10.f; @@ -93,6 +99,7 @@ bool StationarityEstimator::EstimateBandStationarity( for (auto idx : indexes) { acum_power += spectrum_buffer.buffer[idx][band]; } + acum_power += reverb[band]; float noise = kWindowLength * GetStationarityPowerBand(band); RTC_CHECK_LT(0.f, noise); bool stationary = acum_power < kThrStationarity * noise; diff --git a/modules/audio_processing/aec3/stationarity_estimator.h b/modules/audio_processing/aec3/stationarity_estimator.h index 8f0179009f..d5fcd007bf 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.h +++ b/modules/audio_processing/aec3/stationarity_estimator.h @@ -17,6 +17,7 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/reverb_model.h" #include "modules/audio_processing/aec3/vector_buffer.h" namespace webrtc { @@ -38,7 +39,8 @@ class StationarityEstimator { // getting a more robust estimation, it looks at future and/or past frames. void UpdateStationarityFlags(const VectorBuffer& spectrum_buffer, int idx_current, - int num_lookahead); + int num_lookahead, + float reverb_decay); // Returns true if the current band is stationary. bool IsBandStationary(size_t band) const { @@ -52,9 +54,11 @@ class StationarityEstimator { // Get an estimation of the stationarity for the current band by looking // at the past/present/future available data. - bool EstimateBandStationarity(const VectorBuffer& spectrum_buffer, - const std::array& indexes, - size_t band) const; + bool EstimateBandStationarity( + const VectorBuffer& spectrum_buffer, + const std::array& reverb, + const std::array& indexes, + size_t band) const; // True if all bands at the current point are stationary. bool AreAllBandsStationary(); @@ -104,6 +108,7 @@ class StationarityEstimator { NoiseSpectrum noise_; std::array hangovers_; std::array stationarity_flags_; + ReverbModel render_reverb_; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/suppression_gain.cc b/modules/audio_processing/aec3/suppression_gain.cc index a9f564a686..1328601c4e 100644 --- a/modules/audio_processing/aec3/suppression_gain.cc +++ b/modules/audio_processing/aec3/suppression_gain.cc @@ -182,6 +182,8 @@ void GainToNoAudibleEcho( : config.gain_mask.m8; for (size_t k = 0; k < gain->size(); ++k) { + // TODO(devicentepena): Experiment by removing the reverberation estimation + // from the nearend signal before computing the gains. const float unity_gain_masker = std::max(nearend[k], masker[k]); RTC_DCHECK_LE(0.f, nearend_masking_margin * unity_gain_masker); if (weighted_echo[k] <= nearend_masking_margin * unity_gain_masker || diff --git a/modules/audio_processing/test/audio_processing_simulator.cc b/modules/audio_processing/test/audio_processing_simulator.cc index 577274f8af..ed30230748 100644 --- a/modules/audio_processing/test/audio_processing_simulator.cc +++ b/modules/audio_processing/test/audio_processing_simulator.cc @@ -201,6 +201,8 @@ EchoCanceller3Config ParseAec3Parameters(const std::string& filename) { ReadParam(section, "mf", &cfg.ep_strength.mf); ReadParam(section, "hf", &cfg.ep_strength.hf); ReadParam(section, "default_len", &cfg.ep_strength.default_len); + ReadParam(section, "reverb_based_on_render", + &cfg.ep_strength.reverb_based_on_render); ReadParam(section, "echo_can_saturate", &cfg.ep_strength.echo_can_saturate); ReadParam(section, "bounded_erl", &cfg.ep_strength.bounded_erl); }