From a81c09d5b6a23fbc28656117d808d18d00e75e4d Mon Sep 17 00:00:00 2001 From: Sam Zackrisson Date: Thu, 5 Sep 2019 09:35:10 +0200 Subject: [PATCH] Make VectorBuffer in AEC3 multi-channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All dependent modules are hardcoded to do their regular mono processing on the first channel. This _almost_ makes RenderBuffer multi-channel: FftData is still only mono. Bug: webrtc:10913 Change-Id: Id5cc34dbabfe59e1cc72a9675dc7979794e870ed Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/151139 Commit-Queue: Sam Zackrisson Reviewed-by: Per Åhgren Cr-Commit-Position: refs/heads/master@{#29074} --- modules/audio_processing/aec3/aec_state.cc | 3 ++- .../audio_processing/aec3/echo_audibility.cc | 3 ++- modules/audio_processing/aec3/echo_remover.cc | 3 ++- .../aec3/erle_estimator_unittest.cc | 3 ++- .../aec3/mock/mock_render_delay_buffer.cc | 4 +++- .../audio_processing/aec3/render_buffer.cc | 23 +++++++++++-------- modules/audio_processing/aec3/render_buffer.h | 5 ++-- .../aec3/render_buffer_unittest.cc | 4 ++-- .../aec3/render_delay_buffer.cc | 6 +++-- .../aec3/render_reverb_model.cc | 7 +++--- .../aec3/render_signal_analyzer.cc | 5 ++-- .../aec3/residual_echo_estimator.cc | 11 +++++---- .../aec3/signal_dependent_erle_estimator.cc | 7 +++--- ...ignal_dependent_erle_estimator_unittest.cc | 4 ++-- .../aec3/stationarity_estimator.cc | 2 +- .../audio_processing/aec3/vector_buffer.cc | 15 ++++++++---- modules/audio_processing/aec3/vector_buffer.h | 5 ++-- 17 files changed, 69 insertions(+), 41 deletions(-) diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index 566c62fbfe..8ff293089d 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -153,7 +153,8 @@ void AecState::Update( erle_estimator_.Reset(false); } - const auto& X2 = render_buffer.Spectrum(delay_state_.DirectPathFilterDelay()); + const auto& X2 = render_buffer.Spectrum(delay_state_.DirectPathFilterDelay(), + /*channel=*/0); const auto& X2_input_erle = X2_reverb; erle_estimator_.Update(render_buffer, adaptive_filter_frequency_response, diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc index 4154e539b5..f1b0760907 100644 --- a/modules/audio_processing/aec3/echo_audibility.cc +++ b/modules/audio_processing/aec3/echo_audibility.cc @@ -83,7 +83,8 @@ void EchoAudibility::UpdateRenderNoiseEstimator( for (int idx = render_spectrum_write_prev_.value(); idx != render_spectrum_write_current; idx = spectrum_buffer.DecIndex(idx)) { - render_stationarity_.UpdateNoiseEstimator(spectrum_buffer.buffer[idx]); + render_stationarity_.UpdateNoiseEstimator( + spectrum_buffer.buffer[idx][/*channel=*/0]); } } render_spectrum_write_prev_ = render_spectrum_write_current; diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index 8f288de1f5..21eb12ffed 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -346,7 +346,8 @@ void EchoRemoverImpl::ProcessCapture( data_dumper_->DumpRaw("aec3_S2_linear", S2_linear); data_dumper_->DumpRaw("aec3_Y2", Y2); data_dumper_->DumpRaw( - "aec3_X2", render_buffer->Spectrum(aec_state_.FilterDelayBlocks())); + "aec3_X2", + render_buffer->Spectrum(aec_state_.FilterDelayBlocks(), /*channel=*/0)); data_dumper_->DumpRaw("aec3_R2", R2); data_dumper_->DumpRaw("aec3_R2_reverb", residual_echo_estimator_.GetReverbPowerSpectrum()); diff --git a/modules/audio_processing/aec3/erle_estimator_unittest.cc b/modules/audio_processing/aec3/erle_estimator_unittest.cc index 18ba25a53b..dc8ed34bfc 100644 --- a/modules/audio_processing/aec3/erle_estimator_unittest.cc +++ b/modules/audio_processing/aec3/erle_estimator_unittest.cc @@ -70,7 +70,8 @@ void FormFarendFrame(const RenderBuffer& render_buffer, std::array* Y2, float erle) { const auto& spectrum_buffer = render_buffer.GetSpectrumBuffer(); - const auto& X2_from_buffer = spectrum_buffer.buffer[spectrum_buffer.write]; + const auto& X2_from_buffer = + spectrum_buffer.buffer[spectrum_buffer.write][/*channel=*/0]; std::copy(X2_from_buffer.begin(), X2_from_buffer.end(), X2->begin()); std::transform(X2->begin(), X2->end(), Y2->begin(), [](float a) { return a * kEchoPathGain * kEchoPathGain; }); diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc index de87000128..56569a2b88 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc +++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc @@ -19,7 +19,9 @@ MockRenderDelayBuffer::MockRenderDelayBuffer(int sample_rate_hz, NumBandsForRate(sample_rate_hz), num_channels, kBlockSize), - spectrum_buffer_(block_buffer_.buffer.size(), kFftLengthBy2Plus1), + spectrum_buffer_(block_buffer_.buffer.size(), + num_channels, + kFftLengthBy2Plus1), fft_buffer_(block_buffer_.buffer.size()), render_buffer_(&block_buffer_, &spectrum_buffer_, &fft_buffer_), downsampled_render_buffer_(GetDownSampledBufferSize(4, 4)) { diff --git a/modules/audio_processing/aec3/render_buffer.cc b/modules/audio_processing/aec3/render_buffer.cc index f6ffa046e6..bef2b2f31c 100644 --- a/modules/audio_processing/aec3/render_buffer.cc +++ b/modules/audio_processing/aec3/render_buffer.cc @@ -41,9 +41,10 @@ void RenderBuffer::SpectralSum( X2->fill(0.f); int position = spectrum_buffer_->read; for (size_t j = 0; j < num_spectra; ++j) { - std::transform(X2->begin(), X2->end(), - spectrum_buffer_->buffer[position].begin(), X2->begin(), - std::plus()); + for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) { + std::transform(X2->begin(), X2->end(), channel_spectrum.begin(), + X2->begin(), std::plus()); + } position = spectrum_buffer_->IncIndex(position); } } @@ -58,16 +59,20 @@ void RenderBuffer::SpectralSums( int position = spectrum_buffer_->read; size_t j = 0; for (; j < num_spectra_shorter; ++j) { - std::transform(X2_shorter->begin(), X2_shorter->end(), - spectrum_buffer_->buffer[position].begin(), - X2_shorter->begin(), std::plus()); + for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) { + std::transform(X2_shorter->begin(), X2_shorter->end(), + channel_spectrum.begin(), X2_shorter->begin(), + std::plus()); + } position = spectrum_buffer_->IncIndex(position); } std::copy(X2_shorter->begin(), X2_shorter->end(), X2_longer->begin()); for (; j < num_spectra_longer; ++j) { - std::transform(X2_longer->begin(), X2_longer->end(), - spectrum_buffer_->buffer[position].begin(), - X2_longer->begin(), std::plus()); + for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) { + std::transform(X2_longer->begin(), X2_longer->end(), + channel_spectrum.begin(), X2_longer->begin(), + std::plus()); + } position = spectrum_buffer_->IncIndex(position); } } diff --git a/modules/audio_processing/aec3/render_buffer.h b/modules/audio_processing/aec3/render_buffer.h index 8759760ada..7c70e1b737 100644 --- a/modules/audio_processing/aec3/render_buffer.h +++ b/modules/audio_processing/aec3/render_buffer.h @@ -44,10 +44,11 @@ class RenderBuffer { } // Get the spectrum from one of the FFTs in the buffer. - rtc::ArrayView Spectrum(int buffer_offset_ffts) const { + rtc::ArrayView Spectrum(int buffer_offset_ffts, + size_t channel) const { int position = spectrum_buffer_->OffsetIndex(spectrum_buffer_->read, buffer_offset_ffts); - return spectrum_buffer_->buffer[position]; + return spectrum_buffer_->buffer[position][channel]; } // Returns the circular fft buffer. diff --git a/modules/audio_processing/aec3/render_buffer_unittest.cc b/modules/audio_processing/aec3/render_buffer_unittest.cc index 4437178305..e7e964cc85 100644 --- a/modules/audio_processing/aec3/render_buffer_unittest.cc +++ b/modules/audio_processing/aec3/render_buffer_unittest.cc @@ -23,7 +23,7 @@ namespace webrtc { // Verifies the check for non-null fft buffer. TEST(RenderBuffer, NullExternalFftBuffer) { MatrixBuffer block_buffer(10, 3, 1, kBlockSize); - VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1); + VectorBuffer spectrum_buffer(10, 1, kFftLengthBy2Plus1); EXPECT_DEATH(RenderBuffer(&block_buffer, &spectrum_buffer, nullptr), ""); } @@ -37,7 +37,7 @@ TEST(RenderBuffer, NullExternalSpectrumBuffer) { // Verifies the check for non-null block buffer. TEST(RenderBuffer, NullExternalBlockBuffer) { FftBuffer fft_buffer(10); - VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1); + VectorBuffer spectrum_buffer(10, 1, kFftLengthBy2Plus1); EXPECT_DEATH(RenderBuffer(nullptr, &spectrum_buffer, &fft_buffer), ""); } diff --git a/modules/audio_processing/aec3/render_delay_buffer.cc b/modules/audio_processing/aec3/render_delay_buffer.cc index 379f5a1eb0..9d47bec1de 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.cc +++ b/modules/audio_processing/aec3/render_delay_buffer.cc @@ -127,7 +127,7 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, NumBandsForRate(sample_rate_hz), num_render_channels, kBlockSize), - spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1), + spectra_(blocks_.buffer.size(), num_render_channels, kFftLengthBy2Plus1), ffts_(blocks_.buffer.size()), delay_(config_.delay.default_delay), echo_remover_buffer_(&blocks_, &spectra_, &ffts_), @@ -381,7 +381,9 @@ void RenderDelayBufferImpl::InsertBlock( std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write); fft_.PaddedFft(block[0][0], b.buffer[previous_write][0][0], &f.buffer[f.write]); - f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write]); + // TODO(http://bugs.webrtc.org/10913): Loop over all channels when FftBuffer + // supports multi-channel. + f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write][/*channel=*/0]); } bool RenderDelayBufferImpl::DetectActiveRender( diff --git a/modules/audio_processing/aec3/render_reverb_model.cc b/modules/audio_processing/aec3/render_reverb_model.cc index 8ad54c0cc3..72f650636f 100644 --- a/modules/audio_processing/aec3/render_reverb_model.cc +++ b/modules/audio_processing/aec3/render_reverb_model.cc @@ -34,11 +34,12 @@ void RenderReverbModel::Apply(const VectorBuffer& spectrum_buffer, int idx_at_delay = spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks); int idx_past = spectrum_buffer.IncIndex(idx_at_delay); - const auto& X2 = spectrum_buffer.buffer[idx_at_delay]; + const auto& X2 = spectrum_buffer.buffer[idx_at_delay][/*channel=*/0]; RTC_DCHECK_EQ(X2.size(), reverb_power_spectrum.size()); std::copy(X2.begin(), X2.end(), reverb_power_spectrum.begin()); - render_reverb_.AddReverbNoFreqShaping(spectrum_buffer.buffer[idx_past], 1.0f, - reverb_decay, reverb_power_spectrum); + render_reverb_.AddReverbNoFreqShaping( + spectrum_buffer.buffer[idx_past][/*channel=*/0], 1.0f, reverb_decay, + reverb_power_spectrum); } } // namespace webrtc diff --git a/modules/audio_processing/aec3/render_signal_analyzer.cc b/modules/audio_processing/aec3/render_signal_analyzer.cc index 88bacaf79a..8db874ce38 100644 --- a/modules/audio_processing/aec3/render_signal_analyzer.cc +++ b/modules/audio_processing/aec3/render_signal_analyzer.cc @@ -34,7 +34,8 @@ void IdentifySmallNarrowBandRegions( return; } - rtc::ArrayView X2 = render_buffer.Spectrum(*delay_partitions); + rtc::ArrayView X2 = + render_buffer.Spectrum(*delay_partitions, /*channel=*/0); RTC_DCHECK_EQ(kFftLengthBy2Plus1, X2.size()); for (size_t k = 1; k < (X2.size() - 1); ++k) { @@ -49,7 +50,7 @@ void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer, int strong_peak_freeze_duration, absl::optional* narrow_peak_band, size_t* narrow_peak_counter) { - const auto X2_latest = render_buffer.Spectrum(0); + const auto X2_latest = render_buffer.Spectrum(0, /*channel=*/0); // Identify the spectral peak. const int peak_bin = static_cast( diff --git a/modules/audio_processing/aec3/residual_echo_estimator.cc b/modules/audio_processing/aec3/residual_echo_estimator.cc index d863d8bb45..a278240b31 100644 --- a/modules/audio_processing/aec3/residual_echo_estimator.cc +++ b/modules/audio_processing/aec3/residual_echo_estimator.cc @@ -77,7 +77,8 @@ void ResidualEchoEstimator::Estimate( // Adds the estimated unmodelled echo power to the residual echo power // estimate. echo_reverb_.AddReverb( - render_buffer.Spectrum(aec_state.FilterLengthBlocks() + 1), + render_buffer.Spectrum(aec_state.FilterLengthBlocks() + 1, + /*channel=*/0), aec_state.GetReverbFrequencyResponse(), aec_state.ReverbDecay(), *R2); } else { // Estimate the echo generating signal power. @@ -108,7 +109,8 @@ void ResidualEchoEstimator::Estimate( if (!(aec_state.TransparentMode())) { echo_reverb_.AddReverbNoFreqShaping( - render_buffer.Spectrum(aec_state.FilterDelayBlocks() + 1), + render_buffer.Spectrum(aec_state.FilterDelayBlocks() + 1, + /*channel=*/0), echo_path_gain * echo_path_gain, aec_state.ReverbDecay(), *R2); } } @@ -171,7 +173,8 @@ void ResidualEchoEstimator::EchoGeneratingPower( X2->fill(0.f); for (int k = idx_start; k != idx_stop; k = spectrum_buffer.IncIndex(k)) { - std::transform(X2->begin(), X2->end(), spectrum_buffer.buffer[k].begin(), + std::transform(X2->begin(), X2->end(), + spectrum_buffer.buffer[k][/*channel=*/0].begin(), X2->begin(), [](float a, float b) { return std::max(a, b); }); } @@ -194,7 +197,7 @@ void ResidualEchoEstimator::RenderNoisePower( RTC_DCHECK(X2_noise_floor); RTC_DCHECK(X2_noise_floor_counter); - const auto render_power = render_buffer.Spectrum(0); + const auto render_power = render_buffer.Spectrum(0, /*channel=*/0); RTC_DCHECK_EQ(X2_noise_floor->size(), render_power.size()); RTC_DCHECK_EQ(X2_noise_floor_counter->size(), render_power.size()); diff --git a/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc b/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc index dbe8e487e1..b16b70e0e7 100644 --- a/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc +++ b/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc @@ -332,9 +332,10 @@ void SignalDependentErleEstimator::ComputeEchoEstimatePerFilterSection( filter_frequency_response.size()); for (size_t block = section_boundaries_blocks_[section]; block < block_limit; ++block) { - std::transform(X2_section.begin(), X2_section.end(), - spectrum_render_buffer.buffer[idx_render].begin(), - X2_section.begin(), std::plus()); + std::transform( + X2_section.begin(), X2_section.end(), + spectrum_render_buffer.buffer[idx_render][/*channel=*/0].begin(), + X2_section.begin(), std::plus()); std::transform(H2_section.begin(), H2_section.end(), filter_frequency_response[block].begin(), H2_section.begin(), std::plus()); diff --git a/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc b/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc index b8c83f767b..51a2857d73 100644 --- a/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc +++ b/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc @@ -99,8 +99,8 @@ void TestInputs::UpdateCurrentPowerSpectra() { render_buffer_->GetSpectrumBuffer(); size_t idx = render_buffer_->Position(); size_t prev_idx = spectrum_render_buffer.OffsetIndex(idx, 1); - auto& X2 = spectrum_render_buffer.buffer[idx]; - auto& X2_prev = spectrum_render_buffer.buffer[prev_idx]; + auto& X2 = spectrum_render_buffer.buffer[idx][/*channel=*/0]; + auto& X2_prev = spectrum_render_buffer.buffer[prev_idx][/*channel=*/0]; std::copy(X2.begin(), X2.end(), X2_.begin()); RTC_DCHECK_EQ(X2.size(), Y2_.size()); for (size_t k = 0; k < X2.size(); ++k) { diff --git a/modules/audio_processing/aec3/stationarity_estimator.cc b/modules/audio_processing/aec3/stationarity_estimator.cc index 47f04bf0c1..f6a7d997e9 100644 --- a/modules/audio_processing/aec3/stationarity_estimator.cc +++ b/modules/audio_processing/aec3/stationarity_estimator.cc @@ -105,7 +105,7 @@ bool StationarityEstimator::EstimateBandStationarity( constexpr float kThrStationarity = 10.f; float acum_power = 0.f; for (auto idx : indexes) { - acum_power += spectrum_buffer.buffer[idx][band]; + acum_power += spectrum_buffer.buffer[idx][/*channel=*/0][band]; } acum_power += reverb[band]; float noise = kWindowLength * GetStationarityPowerBand(band); diff --git a/modules/audio_processing/aec3/vector_buffer.cc b/modules/audio_processing/aec3/vector_buffer.cc index 0682885c0c..74b1261068 100644 --- a/modules/audio_processing/aec3/vector_buffer.cc +++ b/modules/audio_processing/aec3/vector_buffer.cc @@ -14,11 +14,18 @@ namespace webrtc { -VectorBuffer::VectorBuffer(size_t size, size_t height) +VectorBuffer::VectorBuffer(size_t size, + size_t num_channels, + size_t spectrum_length) : size(static_cast(size)), - buffer(size, std::vector(height, 0.f)) { - for (auto& c : buffer) { - std::fill(c.begin(), c.end(), 0.f); + buffer(size, + std::vector>( + num_channels, + std::vector(spectrum_length, 0.f))) { + for (auto& channel : buffer) { + for (auto& c : channel) { + std::fill(c.begin(), c.end(), 0.f); + } } } diff --git a/modules/audio_processing/aec3/vector_buffer.h b/modules/audio_processing/aec3/vector_buffer.h index 9d1539f639..36dfa80e37 100644 --- a/modules/audio_processing/aec3/vector_buffer.h +++ b/modules/audio_processing/aec3/vector_buffer.h @@ -21,8 +21,9 @@ namespace webrtc { // Struct for bundling a circular buffer of one dimensional vector objects // together with the read and write indices. +// TODO(peah): Change name of this class to be more specific to what it does. struct VectorBuffer { - VectorBuffer(size_t size, size_t height); + VectorBuffer(size_t size, size_t num_channels, size_t spectrum_length); ~VectorBuffer(); int IncIndex(int index) const { @@ -50,7 +51,7 @@ struct VectorBuffer { void DecReadIndex() { read = DecIndex(read); } const int size; - std::vector> buffer; + std::vector>> buffer; int write = 0; int read = 0; };