admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AEC3: Removing the need of a buffer for the stationarity estimator of the render signal.

Browse Source 
Change-Id: I6983e1d8bdd048a5d92209e3023c687f82d383d5

Bug: webrtc:9193,chromium:836790
Change-Id: I6983e1d8bdd048a5d92209e3023c687f82d383d5
Reviewed-on: https://webrtc-review.googlesource.com/72760
Reviewed-by: Per Åhgren <peah@webrtc.org>
Commit-Queue: Jesus de Vicente Pena <devicentepena@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23065}
This commit is contained in:
Jesús de Vicente Peña 2018-04-30 08:37:57 +02:00 committed by Commit Bot
parent 28899d0791
commit 9558192711
5 changed files with 86 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
modules/audio_processing/aec3/echo_audibility.cc
View File

@ -10,51 +10,40 @@
#include "modules/audio_processing/aec3/echo_audibility.h"
#include <algorithm>
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/stationarity_estimator.h"
namespace webrtc {
namespace {
constexpr int kUnitializedIndex = -1;
} // namespace
EchoAudibility::EchoAudibility() {
Reset();
}
void EchoAudibility::Reset() {
render_stationarity_.Reset();
render_write_prev_ = kUnitializedIndex;
}
EchoAudibility::~EchoAudibility() = default;
void EchoAudibility::UpdateAfterStableDelay(const RenderBuffer& render_buffer,
size_t delay_blocks,
size_t capture_block_counter) {
RTC_DCHECK_GT(capture_block_counter, delay_blocks);
size_t num_lookahead = std::min(StationarityEstimator::GetMaxNumLookAhead(),
render_buffer.Headroom() - delay_blocks + 1);
int render_block_number = capture_block_counter - delay_blocks;
for (size_t k = 0; k < (num_lookahead + 1); ++k) {
// Delay changes can potentially make that not all the farend blocks
// are seen. That effect is assumed to have a minimum effect in the
// estimation.
render_stationarity_.Update(render_buffer.Spectrum(delay_blocks - k),
render_block_number + k);
}
render_stationarity_.UpdateStationarityFlags(render_block_number,
num_lookahead);
void EchoAudibility::UpdateRenderStationarityFlags(
const RenderBuffer& render_buffer,
size_t delay_blocks) {
int idx_at_delay =
render_buffer.OffsetSpectrumIndex(render_buffer.Position(), delay_blocks);
size_t num_lookahead = render_buffer.Headroom() - delay_blocks + 1;
render_stationarity_.UpdateStationarityFlags(
render_buffer.GetSpectrumBuffer(), idx_at_delay, num_lookahead);
}
void EchoAudibility::UpdateBeforeStableDelay(
void EchoAudibility::UpdateRenderNoiseEstimator(
const RenderBuffer& render_buffer) {
// If the delay is not set, the read position in the buffer cannot be trust
// and the write position in the render buffer should be used instead
if (first_update_) {
if (render_write_prev_ == kUnitializedIndex) {
render_write_prev_ = render_buffer.GetWritePositionSpectrum();
first_update_ = false;
return;
}
int render_write_current = render_buffer.GetWritePositionSpectrum();
@ -62,7 +51,7 @@ void EchoAudibility::UpdateBeforeStableDelay(
for (int idx = render_write_prev_; idx != render_write_current;
idx = render_buffer.DecIdx(idx)) {
render_stationarity_.UpdateNoiseEstimator(
render_buffer.SpectrumFromPosition(idx));
render_buffer.SpectrumAtIndex(idx));
}
render_write_prev_ = render_write_current;
@ -72,10 +61,10 @@ void EchoAudibility::Update(const RenderBuffer& render_buffer,
size_t delay_blocks,
size_t capture_block_counter,
bool external_delay_seen) {
UpdateRenderNoiseEstimator(render_buffer);
if (external_delay_seen) {
UpdateAfterStableDelay(render_buffer, delay_blocks, capture_block_counter);
} else {
UpdateBeforeStableDelay(render_buffer);
UpdateRenderStationarityFlags(render_buffer, delay_blocks);
}
}

11
modules/audio_processing/aec3/echo_audibility.h
View File

@ -55,13 +55,14 @@ class EchoAudibility {
// Compute the residual scaling per frequency for the current frame.
void ComputeResidualScaling();
void UpdateAfterStableDelay(const RenderBuffer& render_buffer,
size_t delay_blocks,
size_t capture_block_counter);
// Updates the render stationarity flags for the current frame.
void UpdateRenderStationarityFlags(const RenderBuffer& render_buffer,
size_t delay_blocks);
void UpdateBeforeStableDelay(const RenderBuffer& render_buffer);
// Updates the noise estimator with the new render data since the previous
// call to this method.
void UpdateRenderNoiseEstimator(const RenderBuffer& render_buffer);
bool first_update_ = true;
int render_write_prev_;
StationarityEstimator render_stationarity_;
RTC_DISALLOW_COPY_AND_ASSIGN(EchoAudibility);

20
modules/audio_processing/aec3/render_buffer.h
View File

@ -45,10 +45,10 @@ class RenderBuffer {
return spectrum_buffer_->buffer[position];
}
// Get the spectrum directly from a position in the buffer.
rtc::ArrayView<const float> SpectrumFromPosition(int position) const {
RTC_CHECK_LT(position, spectrum_buffer_->size);
int position_bound = std::min(position, spectrum_buffer_->size - 1);
// Get the spectrum directly from an index in the buffer.
rtc::ArrayView<const float> SpectrumAtIndex(int index) const {
RTC_CHECK_LT(index, spectrum_buffer_->size);
int position_bound = std::min(index, spectrum_buffer_->size - 1);
position_bound = std::max(0, position_bound);
return spectrum_buffer_->buffer[position_bound];
}
@ -65,6 +65,11 @@ class RenderBuffer {
return fft_buffer_->read;
}
// Applies an offset to a buffer index and returns it.
int OffsetSpectrumIndex(int index, int offset) const {
return spectrum_buffer_->OffsetIndex(index, offset);
}
// Returns the write postion in the circular buffer.
int GetWritePositionSpectrum() const { return spectrum_buffer_->write; }
@ -79,7 +84,7 @@ class RenderBuffer {
void SetRenderActivity(bool activity) { render_activity_ = activity; }
// Returns the headroom between the write and the read positions in the
// buffer;
// buffer.
int Headroom() const {
// The write and read indices are decreased over time.
int headroom =
@ -93,9 +98,12 @@ class RenderBuffer {
return headroom;
}
// Decrease an index that is used for accessing the buffer.
// Decreases an index that is used for accessing the buffer.
int DecIdx(int idx) const { return spectrum_buffer_->DecIndex(idx); }
// Returns a reference to the spectrum buffer.
const VectorBuffer& GetSpectrumBuffer() const { return *spectrum_buffer_; }
private:
const MatrixBuffer* const block_buffer_;
const VectorBuffer* const spectrum_buffer_;

117
modules/audio_processing/aec3/stationarity_estimator.cc
View File

@ -15,6 +15,7 @@
#include <vector>
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/vector_buffer.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomicops.h"
@ -25,17 +26,11 @@ constexpr float kMinNoisePower = 10.f;
constexpr int kHangoverBlocks = kNumBlocksPerSecond / 20;
constexpr int kNBlocksAverageInitPhase = 20;
constexpr int kNBlocksInitialPhase = kNumBlocksPerSecond * 2.;
constexpr size_t kLongWindowSize = 13;
} // namespace
StationarityEstimator::StationarityEstimator()
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
idx_lookahead_(kLongWindowSize, 0),
idx_lookback_(kLongWindowSize, 0) {
static_assert(StationarityEstimator::CircularBuffer::GetBufferSize() >=
(kLongWindowSize + 1),
"Mismatch between the window size and the buffer size.");
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))) {
Reset();
}
@ -47,14 +42,6 @@ void StationarityEstimator::Reset() {
stationarity_flags_.fill(false);
}
void StationarityEstimator::Update(rtc::ArrayView<const float> spectrum,
int block_number) {
if (!buffer_.IsBlockNumberAlreadyUpdated(block_number)) {
noise_.Update(spectrum);
WriteInfoFrameInSlot(block_number, spectrum);
}
}
// Update just the noise estimator. Usefull until the delay is known
void StationarityEstimator::UpdateNoiseEstimator(
rtc::ArrayView<const float> spectrum) {
@ -62,48 +49,54 @@ void StationarityEstimator::UpdateNoiseEstimator(
data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum());
}
void StationarityEstimator::UpdateStationarityFlags(size_t current_block_number,
size_t num_lookahead) {
RTC_DCHECK_GE(idx_lookahead_.capacity(),
std::min(num_lookahead + 1, kLongWindowSize));
idx_lookahead_.resize(std::min(num_lookahead + 1, kLongWindowSize));
idx_lookback_.resize(0);
GetSlotsAheadBack(current_block_number);
void StationarityEstimator::UpdateStationarityFlags(
const VectorBuffer& spectrum_buffer,
int idx_current,
int num_lookahead) {
std::array<int, kWindowLength> indexes;
int num_lookahead_bounded = std::min(num_lookahead, kWindowLength - 1);
int idx = idx_current;
if (num_lookahead_bounded < kWindowLength - 1) {
int num_lookback = (kWindowLength - 1) - num_lookahead_bounded;
idx = spectrum_buffer.OffsetIndex(idx_current, num_lookback);
}
// For estimating the stationarity properties of the current frame, the
// power for each band is accumulated for several consecutive spectra in the
// method EstimateBandStationarity.
// In order to avoid getting the indexes of the spectra for every band with
// its associated overhead, those indexes are stored in an array and then use
// when the estimation is done.
indexes[0] = idx;
for (size_t k = 1; k < indexes.size(); ++k) {
indexes[k] = spectrum_buffer.DecIndex(indexes[k - 1]);
}
RTC_DCHECK_EQ(
spectrum_buffer.DecIndex(indexes[kWindowLength - 1]),
spectrum_buffer.OffsetIndex(idx_current, -(num_lookahead_bounded + 1)));
for (size_t k = 0; k < stationarity_flags_.size(); ++k) {
stationarity_flags_[k] = EstimateBandStationarity(k);
stationarity_flags_[k] =
EstimateBandStationarity(spectrum_buffer, indexes, k);
}
UpdateHangover();
SmoothStationaryPerFreq();
data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum());
}
void StationarityEstimator::WriteInfoFrameInSlot(
int block_number,
rtc::ArrayView<const float> spectrum) {
size_t slot = buffer_.SetBlockNumberInSlot(block_number);
for (size_t k = 0; k < spectrum.size(); ++k) {
buffer_.SetElementProperties(spectrum[k], slot, k);
}
}
bool StationarityEstimator::EstimateBandStationarity(size_t band) const {
bool StationarityEstimator::EstimateBandStationarity(
const VectorBuffer& spectrum_buffer,
const std::array<int, kWindowLength>& indexes,
size_t band) const {
constexpr float kThrStationarity = 10.f;
float acumPower = 0.f;
for (auto slot : idx_lookahead_) {
acumPower += buffer_.GetPowerBand(slot, band);
float acum_power = 0.f;
for (auto idx : indexes) {
acum_power += spectrum_buffer.buffer[idx][band];
}
for (auto slot : idx_lookback_) {
acumPower += buffer_.GetPowerBand(slot, band);
}
// Generally windowSize is equal to kLongWindowSize
float windowSize = idx_lookahead_.size() + idx_lookback_.size();
float noise = windowSize * GetStationarityPowerBand(band);
float noise = kWindowLength * GetStationarityPowerBand(band);
RTC_CHECK_LT(0.f, noise);
bool stationary = acumPower < kThrStationarity * noise;
data_dumper_->DumpRaw("aec3_stationarity_long_ratio", acumPower / noise);
bool stationary = acum_power < kThrStationarity * noise;
data_dumper_->DumpRaw("aec3_stationarity_long_ratio", acum_power / noise);
return stationary;
}
@ -126,32 +119,6 @@ void StationarityEstimator::UpdateHangover() {
}
}
void StationarityEstimator::GetSlotsAheadBack(size_t current_block_number) {
for (size_t block = 0; block < idx_lookahead_.size(); ++block) {
idx_lookahead_[block] = buffer_.GetSlotNumber(current_block_number + block);
}
size_t num_lookback_blocks;
if (idx_lookahead_.size() >= kLongWindowSize) {
RTC_CHECK_EQ(idx_lookahead_.size(), kLongWindowSize);
num_lookback_blocks = 0;
} else {
num_lookback_blocks = kLongWindowSize - idx_lookahead_.size();
}
if (current_block_number < num_lookback_blocks) {
idx_lookback_.resize(0);
} else {
for (size_t block = 0; block < num_lookback_blocks; ++block) {
int block_number = current_block_number - block - 1;
if (!buffer_.IsBlockNumberAlreadyUpdated(block_number)) {
break;
} else {
RTC_DCHECK_GE(idx_lookback_.capacity(), idx_lookback_.size() + 1);
idx_lookback_.push_back(buffer_.GetSlotNumber(block_number));
}
}
}
}
void StationarityEstimator::SmoothStationaryPerFreq() {
std::array<bool, kFftLengthBy2Plus1> all_ahead_stationary_smooth;
for (size_t k = 1; k < kFftLengthBy2Plus1 - 1; ++k) {
@ -230,10 +197,4 @@ float StationarityEstimator::NoiseSpectrum::UpdateBandBySmoothing(
return power_band_noise_updated;
}
StationarityEstimator::CircularBuffer::CircularBuffer() {
for (auto slot : slots_) {
slot.block_number_ = -1;
}
}
} // namespace webrtc

67
modules/audio_processing/aec3/stationarity_estimator.h
View File

@ -17,6 +17,7 @@
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/vector_buffer.h"
namespace webrtc {
@ -30,38 +31,30 @@ class StationarityEstimator {
// Reset the stationarity estimator.
void Reset();
// Update the stationarity estimator.
void Update(rtc::ArrayView<const float> spectrum, int block_number);
// Update just the noise estimator. Usefull until the delay is known
void UpdateNoiseEstimator(rtc::ArrayView<const float> spectrum);
// Update the flag indicating whether this current frame is stationary. For
// getting a more robust estimation, it looks at future and/or past frames.
void UpdateStationarityFlags(size_t current_block_number,
size_t num_lookahead);
void UpdateStationarityFlags(const VectorBuffer& spectrum_buffer,
int idx_current,
int num_lookahead);
// Returns true if the current band is stationary.
bool IsBandStationary(size_t band) const {
return stationarity_flags_[band] && (hangovers_[band] == 0);
}
static constexpr size_t GetMaxNumLookAhead() {
return CircularBuffer::GetBufferSize() - 2;
}
private:
static constexpr int kWindowLength = 13;
// Returns the power of the stationary noise spectrum at a band.
float GetStationarityPowerBand(size_t k) const { return noise_.Power(k); }
// Write into the slot the information about the current frame that
// is needed for the stationarity detection.
void WriteInfoFrameInSlot(int block_number,
rtc::ArrayView<const float> spectrum);
// Get an estimation of the stationarity for the current band by looking
// at the past/present/future available data.
bool EstimateBandStationarity(size_t band) const;
bool EstimateBandStationarity(const VectorBuffer& spectrum_buffer,
const std::array<int, kWindowLength>& indexes,
size_t band) const;
// True if all bands at the current point are stationary.
bool AreAllBandsStationary();
@ -70,9 +63,6 @@ class StationarityEstimator {
// frame.
void UpdateHangover();
// Get the slots that contain past/present and future data.
void GetSlotsAheadBack(size_t current_block_number);
// Smooth the stationarity detection by looking at neighbouring frequency
// bands.
void SmoothStationaryPerFreq();
@ -109,52 +99,11 @@ class StationarityEstimator {
size_t block_counter_;
};
// The class circular buffer stores the data needed to take a decission
// on whether the current frame is stationary by looking at data from the
// future, present and/or past. This buffer stores that data that is
// represented by the struct Element a bit bellow.
class CircularBuffer {
public:
static constexpr int kCircularBufferSize = 16;
struct Element {
int block_number_;
std::array<float, kFftLengthBy2Plus1> power_spectrum_;
};
CircularBuffer();
static constexpr int GetBufferSize() { return kCircularBufferSize; }
bool IsBlockNumberAlreadyUpdated(int block_number) const {
size_t slot_number = GetSlotNumber(block_number);
return slots_[slot_number].block_number_ == block_number;
}
size_t GetSlotNumber(int block_number) const {
return block_number & (kCircularBufferSize - 1);
}
size_t SetBlockNumberInSlot(int block_number) {
size_t slot = GetSlotNumber(block_number);
slots_[slot].block_number_ = block_number;
return slot;
}
void SetElementProperties(float band_power, int slot, int band) {
slots_[slot].power_spectrum_[band] = band_power;
}
float GetPowerBand(size_t slot, size_t band) const {
return slots_[slot].power_spectrum_[band];
}
private:
std::array<Element, kCircularBufferSize> slots_;
};
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
NoiseSpectrum noise_;
std::vector<size_t> idx_lookahead_;
std::vector<size_t> idx_lookback_;
std::array<int, kFftLengthBy2Plus1> hangovers_;
std::array<bool, kFftLengthBy2Plus1> stationarity_flags_;
CircularBuffer buffer_;
};
} // namespace webrtc