From dc872b6be1827e5adc40a0318c17197e0fdbe06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20de=20Vicente=20Pe=C3=B1a?= Date: Wed, 25 Apr 2018 16:11:42 +0200 Subject: [PATCH] AEC3: Audibility: improvements on the initial noise estimation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:9193,chromium:836790 Change-Id: I589082a18a4a5d1ba5abc170b6cf49d1f545b6cc Reviewed-on: https://webrtc-review.googlesource.com/72480 Reviewed-by: Per Åhgren Commit-Queue: Per Åhgren Cr-Commit-Position: refs/heads/master@{#23027} --- modules/audio_processing/aec3/aec_state.cc | 4 +- .../audio_processing/aec3/echo_audibility.cc | 38 +++++++++++++++++-- .../audio_processing/aec3/echo_audibility.h | 12 +++++- modules/audio_processing/aec3/render_buffer.h | 14 +++++++ .../aec3/stationarity_estimator.cc | 7 ++++ .../aec3/stationarity_estimator.h | 3 ++ 6 files changed, 72 insertions(+), 6 deletions(-) diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index d9b3ae1c08..1f1eaceaeb 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -132,10 +132,10 @@ void AecState::Update( suppression_gain_limiter_.Update(render_buffer.GetRenderActivity(), transparent_mode_); - if (UseStationaryProperties() && external_delay_seen_) { + if (UseStationaryProperties()) { // Update the echo audibility evaluator. echo_audibility_.Update(render_buffer, FilterDelayBlocks(), - capture_block_counter_); + capture_block_counter_, external_delay_seen_); } // Update the ERL and ERLE measures. diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc index 29c6908a6c..a32e8a0812 100644 --- a/modules/audio_processing/aec3/echo_audibility.cc +++ b/modules/audio_processing/aec3/echo_audibility.cc @@ -27,9 +27,9 @@ void EchoAudibility::Reset() { EchoAudibility::~EchoAudibility() = default; -void EchoAudibility::Update(const RenderBuffer& render_buffer, - size_t delay_blocks, - size_t capture_block_counter) { +void EchoAudibility::UpdateAfterStableDelay(const RenderBuffer& render_buffer, + size_t delay_blocks, + size_t capture_block_counter) { RTC_DCHECK_GT(capture_block_counter, delay_blocks); size_t num_lookahead = std::min(StationarityEstimator::GetMaxNumLookAhead(), @@ -47,4 +47,36 @@ void EchoAudibility::Update(const RenderBuffer& render_buffer, num_lookahead); } +void EchoAudibility::UpdateBeforeStableDelay( + const RenderBuffer& render_buffer) { + // If the delay is not set, the read position in the buffer cannot be trust + // and the write position in the render buffer should be used instead + + if (first_update_) { + render_write_prev_ = render_buffer.GetWritePositionSpectrum(); + first_update_ = false; + return; + } + int render_write_current = render_buffer.GetWritePositionSpectrum(); + + for (int idx = render_write_prev_; idx != render_write_current; + idx = render_buffer.DecIdx(idx)) { + render_stationarity_.UpdateNoiseEstimator( + render_buffer.SpectrumFromPosition(idx)); + } + + render_write_prev_ = render_write_current; +} + +void EchoAudibility::Update(const RenderBuffer& render_buffer, + size_t delay_blocks, + size_t capture_block_counter, + bool external_delay_seen) { + if (external_delay_seen) { + UpdateAfterStableDelay(render_buffer, delay_blocks, capture_block_counter); + } else { + UpdateBeforeStableDelay(render_buffer); + } +} + } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_audibility.h b/modules/audio_processing/aec3/echo_audibility.h index 165c879b6c..f4749df97e 100644 --- a/modules/audio_processing/aec3/echo_audibility.h +++ b/modules/audio_processing/aec3/echo_audibility.h @@ -34,7 +34,8 @@ class EchoAudibility { // Feed new render data to the echo audibility estimator. void Update(const RenderBuffer& render_buffer, size_t delay_blocks, - size_t capture_block_counter_); + size_t capture_block_counter_, + bool external_delay_seen); // Get the residual echo scaling. void GetResidualEchoScaling(rtc::ArrayView residual_scaling) const { @@ -53,6 +54,15 @@ class EchoAudibility { // Compute the residual scaling per frequency for the current frame. void ComputeResidualScaling(); + + void UpdateAfterStableDelay(const RenderBuffer& render_buffer, + size_t delay_blocks, + size_t capture_block_counter); + + void UpdateBeforeStableDelay(const RenderBuffer& render_buffer); + + bool first_update_ = true; + int render_write_prev_; StationarityEstimator render_stationarity_; RTC_DISALLOW_COPY_AND_ASSIGN(EchoAudibility); }; diff --git a/modules/audio_processing/aec3/render_buffer.h b/modules/audio_processing/aec3/render_buffer.h index a1cd0e3a53..9d6b8c9802 100644 --- a/modules/audio_processing/aec3/render_buffer.h +++ b/modules/audio_processing/aec3/render_buffer.h @@ -45,6 +45,14 @@ class RenderBuffer { return spectrum_buffer_->buffer[position]; } + // Get the spectrum directly from a position in the buffer. + rtc::ArrayView SpectrumFromPosition(int position) const { + RTC_CHECK_LT(position, spectrum_buffer_->size); + int position_bound = std::min(position, spectrum_buffer_->size - 1); + position_bound = std::max(0, position_bound); + return spectrum_buffer_->buffer[position_bound]; + } + // Returns the circular fft buffer. rtc::ArrayView GetFftBuffer() const { return fft_buffer_->buffer; @@ -57,6 +65,9 @@ class RenderBuffer { return fft_buffer_->read; } + // Returns the write postion in the circular buffer. + int GetWritePositionSpectrum() const { return spectrum_buffer_->write; } + // Returns the sum of the spectrums for a certain number of FFTs. void SpectralSum(size_t num_spectra, std::array* X2) const; @@ -82,6 +93,9 @@ class RenderBuffer { return headroom; } + // Decrease an index that is used for accessing the buffer. + int DecIdx(int idx) const { return spectrum_buffer_->DecIndex(idx); } + private: const MatrixBuffer* const block_buffer_; const VectorBuffer* const spectrum_buffer_; diff --git a/modules/audio_processing/aec3/stationarity_estimator.cc b/modules/audio_processing/aec3/stationarity_estimator.cc index 078abce69c..61381a075f 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.cc +++ b/modules/audio_processing/aec3/stationarity_estimator.cc @@ -55,6 +55,13 @@ void StationarityEstimator::Update(rtc::ArrayView spectrum, } } +// Update just the noise estimator. Usefull until the delay is known +void StationarityEstimator::UpdateNoiseEstimator( + rtc::ArrayView spectrum) { + noise_.Update(spectrum); + data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum()); +} + void StationarityEstimator::UpdateStationarityFlags(size_t current_block_number, size_t num_lookahead) { RTC_DCHECK_GE(idx_lookahead_.capacity(), diff --git a/modules/audio_processing/aec3/stationarity_estimator.h b/modules/audio_processing/aec3/stationarity_estimator.h index 31291a9342..b91956626d 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.h +++ b/modules/audio_processing/aec3/stationarity_estimator.h @@ -33,6 +33,9 @@ class StationarityEstimator { // Update the stationarity estimator. void Update(rtc::ArrayView spectrum, int block_number); + // Update just the noise estimator. Usefull until the delay is known + void UpdateNoiseEstimator(rtc::ArrayView spectrum); + // Update the flag indicating whether this current frame is stationary. For // getting a more robust estimation, it looks at future and/or past frames. void UpdateStationarityFlags(size_t current_block_number,