From 95581927115711dad49d5962387282f1394f6176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20de=20Vicente=20Pe=C3=B1a?= Date: Mon, 30 Apr 2018 08:37:57 +0200 Subject: [PATCH] AEC3: Removing the need of a buffer for the stationarity estimator of the render signal. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I6983e1d8bdd048a5d92209e3023c687f82d383d5 Bug: webrtc:9193,chromium:836790 Change-Id: I6983e1d8bdd048a5d92209e3023c687f82d383d5 Reviewed-on: https://webrtc-review.googlesource.com/72760 Reviewed-by: Per Åhgren Commit-Queue: Jesus de Vicente Pena Cr-Commit-Position: refs/heads/master@{#23065} --- .../audio_processing/aec3/echo_audibility.cc | 49 +++----- .../audio_processing/aec3/echo_audibility.h | 11 +- modules/audio_processing/aec3/render_buffer.h | 20 ++- .../aec3/stationarity_estimator.cc | 117 ++++++------------ .../aec3/stationarity_estimator.h | 67 ++-------- 5 files changed, 86 insertions(+), 178 deletions(-) diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc index a32e8a0812..6f2a8f121c 100644 --- a/modules/audio_processing/aec3/echo_audibility.cc +++ b/modules/audio_processing/aec3/echo_audibility.cc @@ -10,51 +10,40 @@ #include "modules/audio_processing/aec3/echo_audibility.h" -#include - #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/stationarity_estimator.h" namespace webrtc { +namespace { +constexpr int kUnitializedIndex = -1; +} // namespace + EchoAudibility::EchoAudibility() { Reset(); } void EchoAudibility::Reset() { render_stationarity_.Reset(); + render_write_prev_ = kUnitializedIndex; } EchoAudibility::~EchoAudibility() = default; -void EchoAudibility::UpdateAfterStableDelay(const RenderBuffer& render_buffer, - size_t delay_blocks, - size_t capture_block_counter) { - RTC_DCHECK_GT(capture_block_counter, delay_blocks); - - size_t num_lookahead = std::min(StationarityEstimator::GetMaxNumLookAhead(), - render_buffer.Headroom() - delay_blocks + 1); - int render_block_number = capture_block_counter - delay_blocks; - - for (size_t k = 0; k < (num_lookahead + 1); ++k) { - // Delay changes can potentially make that not all the farend blocks - // are seen. That effect is assumed to have a minimum effect in the - // estimation. - render_stationarity_.Update(render_buffer.Spectrum(delay_blocks - k), - render_block_number + k); - } - render_stationarity_.UpdateStationarityFlags(render_block_number, - num_lookahead); +void EchoAudibility::UpdateRenderStationarityFlags( + const RenderBuffer& render_buffer, + size_t delay_blocks) { + int idx_at_delay = + render_buffer.OffsetSpectrumIndex(render_buffer.Position(), delay_blocks); + size_t num_lookahead = render_buffer.Headroom() - delay_blocks + 1; + render_stationarity_.UpdateStationarityFlags( + render_buffer.GetSpectrumBuffer(), idx_at_delay, num_lookahead); } -void EchoAudibility::UpdateBeforeStableDelay( +void EchoAudibility::UpdateRenderNoiseEstimator( const RenderBuffer& render_buffer) { - // If the delay is not set, the read position in the buffer cannot be trust - // and the write position in the render buffer should be used instead - - if (first_update_) { + if (render_write_prev_ == kUnitializedIndex) { render_write_prev_ = render_buffer.GetWritePositionSpectrum(); - first_update_ = false; return; } int render_write_current = render_buffer.GetWritePositionSpectrum(); @@ -62,7 +51,7 @@ void EchoAudibility::UpdateBeforeStableDelay( for (int idx = render_write_prev_; idx != render_write_current; idx = render_buffer.DecIdx(idx)) { render_stationarity_.UpdateNoiseEstimator( - render_buffer.SpectrumFromPosition(idx)); + render_buffer.SpectrumAtIndex(idx)); } render_write_prev_ = render_write_current; @@ -72,10 +61,10 @@ void EchoAudibility::Update(const RenderBuffer& render_buffer, size_t delay_blocks, size_t capture_block_counter, bool external_delay_seen) { + UpdateRenderNoiseEstimator(render_buffer); + if (external_delay_seen) { - UpdateAfterStableDelay(render_buffer, delay_blocks, capture_block_counter); - } else { - UpdateBeforeStableDelay(render_buffer); + UpdateRenderStationarityFlags(render_buffer, delay_blocks); } } diff --git a/modules/audio_processing/aec3/echo_audibility.h b/modules/audio_processing/aec3/echo_audibility.h index f4749df97e..7b9f283257 100644 --- a/modules/audio_processing/aec3/echo_audibility.h +++ b/modules/audio_processing/aec3/echo_audibility.h @@ -55,13 +55,14 @@ class EchoAudibility { // Compute the residual scaling per frequency for the current frame. void ComputeResidualScaling(); - void UpdateAfterStableDelay(const RenderBuffer& render_buffer, - size_t delay_blocks, - size_t capture_block_counter); + // Updates the render stationarity flags for the current frame. + void UpdateRenderStationarityFlags(const RenderBuffer& render_buffer, + size_t delay_blocks); - void UpdateBeforeStableDelay(const RenderBuffer& render_buffer); + // Updates the noise estimator with the new render data since the previous + // call to this method. + void UpdateRenderNoiseEstimator(const RenderBuffer& render_buffer); - bool first_update_ = true; int render_write_prev_; StationarityEstimator render_stationarity_; RTC_DISALLOW_COPY_AND_ASSIGN(EchoAudibility); diff --git a/modules/audio_processing/aec3/render_buffer.h b/modules/audio_processing/aec3/render_buffer.h index 9d6b8c9802..292cb9d2c3 100644 --- a/modules/audio_processing/aec3/render_buffer.h +++ b/modules/audio_processing/aec3/render_buffer.h @@ -45,10 +45,10 @@ class RenderBuffer { return spectrum_buffer_->buffer[position]; } - // Get the spectrum directly from a position in the buffer. - rtc::ArrayView SpectrumFromPosition(int position) const { - RTC_CHECK_LT(position, spectrum_buffer_->size); - int position_bound = std::min(position, spectrum_buffer_->size - 1); + // Get the spectrum directly from an index in the buffer. + rtc::ArrayView SpectrumAtIndex(int index) const { + RTC_CHECK_LT(index, spectrum_buffer_->size); + int position_bound = std::min(index, spectrum_buffer_->size - 1); position_bound = std::max(0, position_bound); return spectrum_buffer_->buffer[position_bound]; } @@ -65,6 +65,11 @@ class RenderBuffer { return fft_buffer_->read; } + // Applies an offset to a buffer index and returns it. + int OffsetSpectrumIndex(int index, int offset) const { + return spectrum_buffer_->OffsetIndex(index, offset); + } + // Returns the write postion in the circular buffer. int GetWritePositionSpectrum() const { return spectrum_buffer_->write; } @@ -79,7 +84,7 @@ class RenderBuffer { void SetRenderActivity(bool activity) { render_activity_ = activity; } // Returns the headroom between the write and the read positions in the - // buffer; + // buffer. int Headroom() const { // The write and read indices are decreased over time. int headroom = @@ -93,9 +98,12 @@ class RenderBuffer { return headroom; } - // Decrease an index that is used for accessing the buffer. + // Decreases an index that is used for accessing the buffer. int DecIdx(int idx) const { return spectrum_buffer_->DecIndex(idx); } + // Returns a reference to the spectrum buffer. + const VectorBuffer& GetSpectrumBuffer() const { return *spectrum_buffer_; } + private: const MatrixBuffer* const block_buffer_; const VectorBuffer* const spectrum_buffer_; diff --git a/modules/audio_processing/aec3/stationarity_estimator.cc b/modules/audio_processing/aec3/stationarity_estimator.cc index 61381a075f..8e065d2422 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.cc +++ b/modules/audio_processing/aec3/stationarity_estimator.cc @@ -15,6 +15,7 @@ #include #include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/vector_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/atomicops.h" @@ -25,17 +26,11 @@ constexpr float kMinNoisePower = 10.f; constexpr int kHangoverBlocks = kNumBlocksPerSecond / 20; constexpr int kNBlocksAverageInitPhase = 20; constexpr int kNBlocksInitialPhase = kNumBlocksPerSecond * 2.; -constexpr size_t kLongWindowSize = 13; } // namespace StationarityEstimator::StationarityEstimator() : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), - idx_lookahead_(kLongWindowSize, 0), - idx_lookback_(kLongWindowSize, 0) { - static_assert(StationarityEstimator::CircularBuffer::GetBufferSize() >= - (kLongWindowSize + 1), - "Mismatch between the window size and the buffer size."); + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))) { Reset(); } @@ -47,14 +42,6 @@ void StationarityEstimator::Reset() { stationarity_flags_.fill(false); } -void StationarityEstimator::Update(rtc::ArrayView spectrum, - int block_number) { - if (!buffer_.IsBlockNumberAlreadyUpdated(block_number)) { - noise_.Update(spectrum); - WriteInfoFrameInSlot(block_number, spectrum); - } -} - // Update just the noise estimator. Usefull until the delay is known void StationarityEstimator::UpdateNoiseEstimator( rtc::ArrayView spectrum) { @@ -62,48 +49,54 @@ void StationarityEstimator::UpdateNoiseEstimator( data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum()); } -void StationarityEstimator::UpdateStationarityFlags(size_t current_block_number, - size_t num_lookahead) { - RTC_DCHECK_GE(idx_lookahead_.capacity(), - std::min(num_lookahead + 1, kLongWindowSize)); - idx_lookahead_.resize(std::min(num_lookahead + 1, kLongWindowSize)); - idx_lookback_.resize(0); - GetSlotsAheadBack(current_block_number); +void StationarityEstimator::UpdateStationarityFlags( + const VectorBuffer& spectrum_buffer, + int idx_current, + int num_lookahead) { + std::array indexes; + int num_lookahead_bounded = std::min(num_lookahead, kWindowLength - 1); + int idx = idx_current; + + if (num_lookahead_bounded < kWindowLength - 1) { + int num_lookback = (kWindowLength - 1) - num_lookahead_bounded; + idx = spectrum_buffer.OffsetIndex(idx_current, num_lookback); + } + // For estimating the stationarity properties of the current frame, the + // power for each band is accumulated for several consecutive spectra in the + // method EstimateBandStationarity. + // In order to avoid getting the indexes of the spectra for every band with + // its associated overhead, those indexes are stored in an array and then use + // when the estimation is done. + indexes[0] = idx; + for (size_t k = 1; k < indexes.size(); ++k) { + indexes[k] = spectrum_buffer.DecIndex(indexes[k - 1]); + } + RTC_DCHECK_EQ( + spectrum_buffer.DecIndex(indexes[kWindowLength - 1]), + spectrum_buffer.OffsetIndex(idx_current, -(num_lookahead_bounded + 1))); for (size_t k = 0; k < stationarity_flags_.size(); ++k) { - stationarity_flags_[k] = EstimateBandStationarity(k); + stationarity_flags_[k] = + EstimateBandStationarity(spectrum_buffer, indexes, k); } UpdateHangover(); SmoothStationaryPerFreq(); - data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum()); } -void StationarityEstimator::WriteInfoFrameInSlot( - int block_number, - rtc::ArrayView spectrum) { - size_t slot = buffer_.SetBlockNumberInSlot(block_number); - for (size_t k = 0; k < spectrum.size(); ++k) { - buffer_.SetElementProperties(spectrum[k], slot, k); - } -} - -bool StationarityEstimator::EstimateBandStationarity(size_t band) const { +bool StationarityEstimator::EstimateBandStationarity( + const VectorBuffer& spectrum_buffer, + const std::array& indexes, + size_t band) const { constexpr float kThrStationarity = 10.f; - float acumPower = 0.f; - for (auto slot : idx_lookahead_) { - acumPower += buffer_.GetPowerBand(slot, band); + float acum_power = 0.f; + for (auto idx : indexes) { + acum_power += spectrum_buffer.buffer[idx][band]; } - for (auto slot : idx_lookback_) { - acumPower += buffer_.GetPowerBand(slot, band); - } - - // Generally windowSize is equal to kLongWindowSize - float windowSize = idx_lookahead_.size() + idx_lookback_.size(); - float noise = windowSize * GetStationarityPowerBand(band); + float noise = kWindowLength * GetStationarityPowerBand(band); RTC_CHECK_LT(0.f, noise); - bool stationary = acumPower < kThrStationarity * noise; - data_dumper_->DumpRaw("aec3_stationarity_long_ratio", acumPower / noise); + bool stationary = acum_power < kThrStationarity * noise; + data_dumper_->DumpRaw("aec3_stationarity_long_ratio", acum_power / noise); return stationary; } @@ -126,32 +119,6 @@ void StationarityEstimator::UpdateHangover() { } } -void StationarityEstimator::GetSlotsAheadBack(size_t current_block_number) { - for (size_t block = 0; block < idx_lookahead_.size(); ++block) { - idx_lookahead_[block] = buffer_.GetSlotNumber(current_block_number + block); - } - size_t num_lookback_blocks; - if (idx_lookahead_.size() >= kLongWindowSize) { - RTC_CHECK_EQ(idx_lookahead_.size(), kLongWindowSize); - num_lookback_blocks = 0; - } else { - num_lookback_blocks = kLongWindowSize - idx_lookahead_.size(); - } - if (current_block_number < num_lookback_blocks) { - idx_lookback_.resize(0); - } else { - for (size_t block = 0; block < num_lookback_blocks; ++block) { - int block_number = current_block_number - block - 1; - if (!buffer_.IsBlockNumberAlreadyUpdated(block_number)) { - break; - } else { - RTC_DCHECK_GE(idx_lookback_.capacity(), idx_lookback_.size() + 1); - idx_lookback_.push_back(buffer_.GetSlotNumber(block_number)); - } - } - } -} - void StationarityEstimator::SmoothStationaryPerFreq() { std::array all_ahead_stationary_smooth; for (size_t k = 1; k < kFftLengthBy2Plus1 - 1; ++k) { @@ -230,10 +197,4 @@ float StationarityEstimator::NoiseSpectrum::UpdateBandBySmoothing( return power_band_noise_updated; } -StationarityEstimator::CircularBuffer::CircularBuffer() { - for (auto slot : slots_) { - slot.block_number_ = -1; - } -} - } // namespace webrtc diff --git a/modules/audio_processing/aec3/stationarity_estimator.h b/modules/audio_processing/aec3/stationarity_estimator.h index b91956626d..8f0179009f 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.h +++ b/modules/audio_processing/aec3/stationarity_estimator.h @@ -17,6 +17,7 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/vector_buffer.h" namespace webrtc { @@ -30,38 +31,30 @@ class StationarityEstimator { // Reset the stationarity estimator. void Reset(); - // Update the stationarity estimator. - void Update(rtc::ArrayView spectrum, int block_number); - // Update just the noise estimator. Usefull until the delay is known void UpdateNoiseEstimator(rtc::ArrayView spectrum); // Update the flag indicating whether this current frame is stationary. For // getting a more robust estimation, it looks at future and/or past frames. - void UpdateStationarityFlags(size_t current_block_number, - size_t num_lookahead); + void UpdateStationarityFlags(const VectorBuffer& spectrum_buffer, + int idx_current, + int num_lookahead); // Returns true if the current band is stationary. bool IsBandStationary(size_t band) const { return stationarity_flags_[band] && (hangovers_[band] == 0); } - static constexpr size_t GetMaxNumLookAhead() { - return CircularBuffer::GetBufferSize() - 2; - } - private: + static constexpr int kWindowLength = 13; // Returns the power of the stationary noise spectrum at a band. float GetStationarityPowerBand(size_t k) const { return noise_.Power(k); } - // Write into the slot the information about the current frame that - // is needed for the stationarity detection. - void WriteInfoFrameInSlot(int block_number, - rtc::ArrayView spectrum); - // Get an estimation of the stationarity for the current band by looking // at the past/present/future available data. - bool EstimateBandStationarity(size_t band) const; + bool EstimateBandStationarity(const VectorBuffer& spectrum_buffer, + const std::array& indexes, + size_t band) const; // True if all bands at the current point are stationary. bool AreAllBandsStationary(); @@ -70,9 +63,6 @@ class StationarityEstimator { // frame. void UpdateHangover(); - // Get the slots that contain past/present and future data. - void GetSlotsAheadBack(size_t current_block_number); - // Smooth the stationarity detection by looking at neighbouring frequency // bands. void SmoothStationaryPerFreq(); @@ -109,52 +99,11 @@ class StationarityEstimator { size_t block_counter_; }; - // The class circular buffer stores the data needed to take a decission - // on whether the current frame is stationary by looking at data from the - // future, present and/or past. This buffer stores that data that is - // represented by the struct Element a bit bellow. - class CircularBuffer { - public: - static constexpr int kCircularBufferSize = 16; - struct Element { - int block_number_; - std::array power_spectrum_; - }; - CircularBuffer(); - - static constexpr int GetBufferSize() { return kCircularBufferSize; } - - bool IsBlockNumberAlreadyUpdated(int block_number) const { - size_t slot_number = GetSlotNumber(block_number); - return slots_[slot_number].block_number_ == block_number; - } - - size_t GetSlotNumber(int block_number) const { - return block_number & (kCircularBufferSize - 1); - } - size_t SetBlockNumberInSlot(int block_number) { - size_t slot = GetSlotNumber(block_number); - slots_[slot].block_number_ = block_number; - return slot; - } - void SetElementProperties(float band_power, int slot, int band) { - slots_[slot].power_spectrum_[band] = band_power; - } - float GetPowerBand(size_t slot, size_t band) const { - return slots_[slot].power_spectrum_[band]; - } - - private: - std::array slots_; - }; static int instance_count_; std::unique_ptr data_dumper_; NoiseSpectrum noise_; - std::vector idx_lookahead_; - std::vector idx_lookback_; std::array hangovers_; std::array stationarity_flags_; - CircularBuffer buffer_; }; } // namespace webrtc