admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AEC3: Made EchoAudibility multichannel

Browse Source 
This CL corrects the EchoAudibility and StationarityEstimator
code to work properly with multiple channels.

It also changes the naming of the FilterDelayBlocks() method
to better reflect what it does.

The changes have been verified to be bitexact over a large number
of recordings.

Bug: webrtc:10913
Change-Id: I070b531efcdff4c33f70fd5b37fbb556dcebe5b4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/156565
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Commit-Queue: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29482}
This commit is contained in:
Per Åhgren 2019-10-15 10:31:35 +02:00 committed by Commit Bot
parent 41478c7c1b
commit 8718afb283
12 changed files with 115 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc
View File

@ -441,7 +441,7 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) {
auto* const render_buffer = render_delay_buffer->GetRenderBuffer();
render_signal_analyzer.Update(*render_buffer,
aec_state.FilterDelayBlocks());
aec_state.MinDirectPathFilterDelay());
filter.Filter(*render_buffer, &S);
fft.Ifft(S, &s_scratch);

29
modules/audio_processing/aec3/aec_state.cc
View File

@ -29,7 +29,7 @@ namespace {
constexpr size_t kBlocksSinceConvergencedFilterInit = 10000;
constexpr size_t kBlocksSinceConsistentEstimateInit = 10000;
void UpdateAndComputeReverb(
void ComputeAvgRenderReverb(
const SpectrumBuffer& spectrum_buffer,
int delay_blocks,
float reverb_decay,
@ -211,16 +211,16 @@ void AecState::Update(
strong_not_saturated_render_blocks_ +=
active_render && !SaturatedCapture() ? 1 : 0;
std::array<float, kFftLengthBy2Plus1> X2_reverb;
std::array<float, kFftLengthBy2Plus1> avg_render_spectrum_with_reverb;
UpdateAndComputeReverb(render_buffer.GetSpectrumBuffer(),
delay_state_.DirectPathFilterDelays()[0],
ReverbDecay(), &reverb_model_, X2_reverb);
ComputeAvgRenderReverb(render_buffer.GetSpectrumBuffer(),
delay_state_.MinDirectPathFilterDelay(), ReverbDecay(),
&avg_render_reverb_, avg_render_spectrum_with_reverb);
if (config_.echo_audibility.use_stationarity_properties) {
// Update the echo audibility evaluator.
echo_audibility_.Update(render_buffer, reverb_model_.reverb(),
delay_state_.DirectPathFilterDelays()[0],
echo_audibility_.Update(render_buffer, avg_render_reverb_.reverb(),
delay_state_.MinDirectPathFilterDelay(),
delay_state_.ExternalDelayReported());
}
@ -229,17 +229,15 @@ void AecState::Update(
erle_estimator_.Reset(false);
}
// TODO(bugs.webrtc.org/10913): Take all channels into account.
const auto& X2 =
render_buffer.Spectrum(delay_state_.DirectPathFilterDelays()[0],
/*channel=*/0);
const auto& X2_input_erle = X2_reverb;
erle_estimator_.Update(render_buffer, adaptive_filter_frequency_responses[0],
X2_input_erle, Y2[0], E2_main[0],
avg_render_spectrum_with_reverb, Y2[0], E2_main[0],
subtractor_output_analyzers_[0].ConvergedFilter(),
config_.erle.onset_detection);
// TODO(bugs.webrtc.org/10913): Take all channels into account.
const auto& X2 =
render_buffer.Spectrum(delay_state_.MinDirectPathFilterDelay(),
/*channel=*/0);
erl_estimator_.Update(subtractor_output_analyzers_[0].ConvergedFilter(), X2,
Y2[0]);
@ -357,6 +355,9 @@ void AecState::FilterDelay::Update(
analyzer_filter_delay_estimates_blocks.end(),
filter_delays_blocks_.begin());
}
min_filter_delay_ = *std::min_element(filter_delays_blocks_.begin(),
filter_delays_blocks_.end());
}
AecState::TransparentMode::TransparentMode(const EchoCanceller3Config& config)

11
modules/audio_processing/aec3/aec_state.h
View File

@ -91,8 +91,8 @@ class AecState {
float ErlTimeDomain() const { return erl_estimator_.ErlTimeDomain(); }
// Returns the delay estimate based on the linear filter.
int FilterDelayBlocks() const {
return delay_state_.DirectPathFilterDelays()[0];
int MinDirectPathFilterDelay() const {
return delay_state_.MinDirectPathFilterDelay();
}
// Returns whether the capture signal is saturated.
@ -194,6 +194,10 @@ class AecState {
return filter_delays_blocks_;
}
// Returns the minimum delay among the direct path delays relative to the
// beginning of the filter
int MinDirectPathFilterDelay() const { return min_filter_delay_; }
// Updates the delay estimates based on new data.
void Update(
rtc::ArrayView<const int> analyzer_filter_delay_estimates_blocks,
@ -204,6 +208,7 @@ class AecState {
const int delay_headroom_samples_;
bool external_delay_reported_ = false;
std::vector<int> filter_delays_blocks_;
int min_filter_delay_ = 0;
absl::optional<DelayEstimate> external_delay_;
} delay_state_;
@ -308,7 +313,7 @@ class AecState {
absl::optional<DelayEstimate> external_delay_;
EchoAudibility echo_audibility_;
ReverbModelEstimator reverb_model_estimator_;
ReverbModel reverb_model_;
ReverbModel avg_render_reverb_;
std::vector<SubtractorOutputAnalyzer> subtractor_output_analyzers_;
};

46
modules/audio_processing/aec3/echo_audibility.cc
View File

@ -29,18 +29,16 @@ EchoAudibility::EchoAudibility(bool use_render_stationarity_at_init)
EchoAudibility::~EchoAudibility() = default;
void EchoAudibility::Update(
const RenderBuffer& render_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int delay_blocks,
bool external_delay_seen) {
void EchoAudibility::Update(const RenderBuffer& render_buffer,
rtc::ArrayView<const float> average_reverb,
int delay_blocks,
bool external_delay_seen) {
UpdateRenderNoiseEstimator(render_buffer.GetSpectrumBuffer(),
render_buffer.GetBlockBuffer(),
external_delay_seen);
if (external_delay_seen || use_render_stationarity_at_init_) {
UpdateRenderStationarityFlags(
render_buffer, render_reverb_contribution_spectrum, delay_blocks);
UpdateRenderStationarityFlags(render_buffer, average_reverb, delay_blocks);
}
}
@ -52,18 +50,17 @@ void EchoAudibility::Reset() {
void EchoAudibility::UpdateRenderStationarityFlags(
const RenderBuffer& render_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int delay_blocks) {
rtc::ArrayView<const float> average_reverb,
int min_channel_delay_blocks) {
const SpectrumBuffer& spectrum_buffer = render_buffer.GetSpectrumBuffer();
int idx_at_delay =
spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks);
int idx_at_delay = spectrum_buffer.OffsetIndex(spectrum_buffer.read,
min_channel_delay_blocks);
int num_lookahead = render_buffer.Headroom() - delay_blocks + 1;
int num_lookahead = render_buffer.Headroom() - min_channel_delay_blocks + 1;
num_lookahead = std::max(0, num_lookahead);
render_stationarity_.UpdateStationarityFlags(
spectrum_buffer, render_reverb_contribution_spectrum, idx_at_delay,
num_lookahead);
render_stationarity_.UpdateStationarityFlags(spectrum_buffer, average_reverb,
idx_at_delay, num_lookahead);
}
void EchoAudibility::UpdateRenderNoiseEstimator(
@ -83,14 +80,15 @@ void EchoAudibility::UpdateRenderNoiseEstimator(
for (int idx = render_spectrum_write_prev_.value();
idx != render_spectrum_write_current;
idx = spectrum_buffer.DecIndex(idx)) {
render_stationarity_.UpdateNoiseEstimator(
spectrum_buffer.buffer[idx][/*channel=*/0]);
render_stationarity_.UpdateNoiseEstimator(spectrum_buffer.buffer[idx]);
}
}
render_spectrum_write_prev_ = render_spectrum_write_current;
}
bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) {
const int num_render_channels =
static_cast<int>(block_buffer.buffer[0].size());
bool too_low = false;
const int render_block_write_current = block_buffer.write;
if (render_block_write_current == render_block_write_prev_) {
@ -98,10 +96,16 @@ bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) {
} else {
for (int idx = render_block_write_prev_; idx != render_block_write_current;
idx = block_buffer.IncIndex(idx)) {
auto block = block_buffer.buffer[idx][0][0];
auto r = std::minmax_element(block.cbegin(), block.cend());
float max_abs = std::max(std::fabs(*r.first), std::fabs(*r.second));
if (max_abs < 10) {
float max_abs_over_channels = 0.f;
for (int ch = 0; ch < num_render_channels; ++ch) {
auto block = block_buffer.buffer[idx][0][ch];
auto r = std::minmax_element(block.cbegin(), block.cend());
float max_abs_channel =
std::max(std::fabs(*r.first), std::fabs(*r.second));
max_abs_over_channels =
std::max(max_abs_over_channels, max_abs_channel);
}
if (max_abs_over_channels < 10.f) {
too_low = true; // Discards all blocks if one of them is too low.
break;
}

15
modules/audio_processing/aec3/echo_audibility.h
View File

@ -28,10 +28,13 @@ class EchoAudibility {
explicit EchoAudibility(bool use_render_stationarity_at_init);
~EchoAudibility();
EchoAudibility(const EchoAudibility&) = delete;
EchoAudibility& operator=(const EchoAudibility&) = delete;
// Feed new render data to the echo audibility estimator.
void Update(const RenderBuffer& render_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int delay_blocks,
rtc::ArrayView<const float> average_reverb,
int min_channel_delay_blocks,
bool external_delay_seen);
// Get the residual echo scaling.
void GetResidualEchoScaling(bool filter_has_had_time_to_converge,
@ -57,10 +60,9 @@ class EchoAudibility {
void Reset();
// Updates the render stationarity flags for the current frame.
void UpdateRenderStationarityFlags(
const RenderBuffer& render_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int delay_blocks);
void UpdateRenderStationarityFlags(const RenderBuffer& render_buffer,
rtc::ArrayView<const float> average_reverb,
int delay_blocks);
// Updates the noise estimator with the new render data since the previous
// call to this method.
@ -77,7 +79,6 @@ class EchoAudibility {
bool non_zero_render_seen_;
const bool use_render_stationarity_at_init_;
StationarityEstimator render_stationarity_;
RTC_DISALLOW_COPY_AND_ASSIGN(EchoAudibility);
};
} // namespace webrtc

9
modules/audio_processing/aec3/echo_remover.cc
View File

@ -356,7 +356,7 @@ void EchoRemoverImpl::ProcessCapture(
// Analyze the render signal.
render_signal_analyzer_.Update(*render_buffer,
aec_state_.FilterDelayBlocks());
aec_state_.MinDirectPathFilterDelay());
// State transition.
if (aec_state_.TransitionTriggered()) {
@ -457,10 +457,11 @@ void EchoRemoverImpl::ProcessCapture(
data_dumper_->DumpRaw("aec3_S2_linear", S2_linear[0]);
data_dumper_->DumpRaw("aec3_Y2", Y2[0]);
data_dumper_->DumpRaw(
"aec3_X2",
render_buffer->Spectrum(aec_state_.FilterDelayBlocks(), /*channel=*/0));
"aec3_X2", render_buffer->Spectrum(aec_state_.MinDirectPathFilterDelay(),
/*channel=*/0));
data_dumper_->DumpRaw("aec3_R2", R2[0]);
data_dumper_->DumpRaw("aec3_filter_delay", aec_state_.FilterDelayBlocks());
data_dumper_->DumpRaw("aec3_filter_delay",
aec_state_.MinDirectPathFilterDelay());
data_dumper_->DumpRaw("aec3_capture_saturation",
aec_state_.SaturatedCapture() ? 1 : 0);
}

3
modules/audio_processing/aec3/echo_remover_metrics.cc
View File

@ -241,7 +241,8 @@ void EchoRemoverMetrics::Update(
static_cast<int>(
active_render_count_ > kMetricsCollectionBlocksBy2 ? 1 : 0));
RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.FilterDelay",
aec_state.FilterDelayBlocks(), 0, 30, 31);
aec_state.MinDirectPathFilterDelay(), 0, 30,
31);
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.EchoCanceller.CaptureSaturation",
static_cast<int>(saturated_capture_ ? 1 : 0));
break;

2
modules/audio_processing/aec3/main_filter_update_gain_unittest.cc
View File

@ -150,7 +150,7 @@ void RunFilterUpdateTest(int num_blocks_to_process,
render_delay_buffer->PrepareCaptureProcessing();
render_signal_analyzer.Update(*render_delay_buffer->GetRenderBuffer(),
aec_state.FilterDelayBlocks());
aec_state.MinDirectPathFilterDelay());
// Apply the main filter.
main_filter.Filter(*render_delay_buffer->GetRenderBuffer(), &S);

9
modules/audio_processing/aec3/residual_echo_estimator.cc
View File

@ -203,7 +203,7 @@ void ResidualEchoEstimator::Estimate(
std::array<float, kFftLengthBy2Plus1> X2;
EchoGeneratingPower(num_render_channels_,
render_buffer.GetSpectrumBuffer(), config_.echo_model,
aec_state.FilterDelayBlocks(), X2);
aec_state.MinDirectPathFilterDelay(), X2);
if (!aec_state.UseStationarityProperties()) {
ApplyNoiseGate(config_.echo_model, X2);
}
@ -288,9 +288,10 @@ void ResidualEchoEstimator::AddReverb(
const size_t num_capture_channels = R2.size();
// Choose reverb partition based on what type of echo power model is used.
const size_t first_reverb_partition = reverb_type == ReverbType::kLinear
? aec_state.FilterLengthBlocks() + 1
: aec_state.FilterDelayBlocks() + 1;
const size_t first_reverb_partition =
reverb_type == ReverbType::kLinear
? aec_state.FilterLengthBlocks() + 1
: aec_state.MinDirectPathFilterDelay() + 1;
// Compute render power for the reverb.
std::array<float, kFftLengthBy2Plus1> render_power_data;

49
modules/audio_processing/aec3/stationarity_estimator.cc
View File

@ -45,7 +45,7 @@ void StationarityEstimator::Reset() {
// Update just the noise estimator. Usefull until the delay is known
void StationarityEstimator::UpdateNoiseEstimator(
rtc::ArrayView<const float> spectrum) {
rtc::ArrayView<const std::vector<float>> spectrum) {
noise_.Update(spectrum);
data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum());
data_dumper_->DumpRaw("aec3_stationarity_is_block_stationary",
@ -99,15 +99,20 @@ bool StationarityEstimator::IsBlockStationary() const {
bool StationarityEstimator::EstimateBandStationarity(
const SpectrumBuffer& spectrum_buffer,
rtc::ArrayView<const float> reverb,
rtc::ArrayView<const float> average_reverb,
const std::array<int, kWindowLength>& indexes,
size_t band) const {
constexpr float kThrStationarity = 10.f;
float acum_power = 0.f;
const int num_render_channels =
static_cast<int>(spectrum_buffer.buffer[0].size());
const float one_by_num_channels = 1.f / num_render_channels;
for (auto idx : indexes) {
acum_power += spectrum_buffer.buffer[idx][/*channel=*/0][band];
for (int ch = 0; ch < num_render_channels; ++ch) {
acum_power += spectrum_buffer.buffer[idx][ch][band] * one_by_num_channels;
}
}
acum_power += reverb[band];
acum_power += average_reverb[band];
float noise = kWindowLength * GetStationarityPowerBand(band);
RTC_CHECK_LT(0.f, noise);
bool stationary = acum_power < kThrStationarity * noise;
@ -163,16 +168,42 @@ void StationarityEstimator::NoiseSpectrum::Reset() {
}
void StationarityEstimator::NoiseSpectrum::Update(
rtc::ArrayView<const float> spectrum) {
RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum.size());
rtc::ArrayView<const std::vector<float>> spectrum) {
RTC_DCHECK_LE(1, spectrum[0].size());
const int num_render_channels = static_cast<int>(spectrum.size());
std::array<float, kFftLengthBy2Plus1> avg_spectrum_data;
rtc::ArrayView<const float> avg_spectrum;
RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum[0].size());
if (num_render_channels == 1) {
avg_spectrum = spectrum[0];
} else {
// For multiple channels, average the channel spectra before passing to the
// noise spectrum estimator.
avg_spectrum = avg_spectrum_data;
std::copy(spectrum[0].begin(), spectrum[0].end(),
avg_spectrum_data.begin());
for (int ch = 1; ch < num_render_channels; ++ch) {
RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum[ch].size());
for (size_t k = 1; k < kFftLengthBy2Plus1; ++k) {
avg_spectrum_data[k] += spectrum[ch][k];
}
}
const float one_by_num_channels = 1.f / num_render_channels;
for (size_t k = 1; k < kFftLengthBy2Plus1; ++k) {
avg_spectrum_data[k] *= one_by_num_channels;
}
}
++block_counter_;
float alpha = GetAlpha();
for (size_t k = 0; k < spectrum.size(); ++k) {
for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) {
if (block_counter_ <= kNBlocksAverageInitPhase) {
noise_spectrum_[k] += (1.f / kNBlocksAverageInitPhase) * spectrum[k];
noise_spectrum_[k] += (1.f / kNBlocksAverageInitPhase) * avg_spectrum[k];
} else {
noise_spectrum_[k] =
UpdateBandBySmoothing(spectrum[k], noise_spectrum_[k], alpha);
UpdateBandBySmoothing(avg_spectrum[k], noise_spectrum_[k], alpha);
}
}
}

7
modules/audio_processing/aec3/stationarity_estimator.h
View File

@ -15,6 +15,7 @@
#include <array>
#include <memory>
#include <vector>
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h" // kFftLengthBy2Plus1...
@ -35,7 +36,7 @@ class StationarityEstimator {
void Reset();
// Update just the noise estimator. Usefull until the delay is known
void UpdateNoiseEstimator(rtc::ArrayView<const float> spectrum);
void UpdateNoiseEstimator(rtc::ArrayView<const std::vector<float>> spectrum);
// Update the flag indicating whether this current frame is stationary. For
// getting a more robust estimation, it looks at future and/or past frames.
@ -61,7 +62,7 @@ class StationarityEstimator {
// Get an estimation of the stationarity for the current band by looking
// at the past/present/future available data.
bool EstimateBandStationarity(const SpectrumBuffer& spectrum_buffer,
rtc::ArrayView<const float> reverb,
rtc::ArrayView<const float> average_reverb,
const std::array<int, kWindowLength>& indexes,
size_t band) const;
@ -85,7 +86,7 @@ class StationarityEstimator {
void Reset();
// Update the noise power spectrum with a new frame.
void Update(rtc::ArrayView<const float> spectrum);
void Update(rtc::ArrayView<const std::vector<float>> spectrum);
// Get the noise estimation power spectrum.
rtc::ArrayView<const float> Spectrum() const { return noise_spectrum_; }

2
modules/audio_processing/aec3/subtractor_unittest.cc
View File

@ -135,7 +135,7 @@ std::vector<float> RunSubtractorTest(
}
render_delay_buffer->PrepareCaptureProcessing();
render_signal_analyzer.Update(*render_delay_buffer->GetRenderBuffer(),
aec_state.FilterDelayBlocks());
aec_state.MinDirectPathFilterDelay());
// Handle echo path changes.
if (std::find(blocks_with_echo_path_changes.begin(),