From 260c788d77ca1501b327f03472825c46f18ced93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=85hgren?= Date: Tue, 28 Jan 2020 15:38:41 +0100 Subject: [PATCH] AEC3: Added multi-channel support for the capture delay functionality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL adds the missing support for multi-channel in the code that provides an optional and configurable delay to be added to the microphone signal. The CL also makes the creation of the delay object conditional on the need for that support (this is important since this adds a significant heap memory footprint) Bug: webrtc:11314,chromium:1045910 Change-Id: I92d577e31af830945fe9d5ca2032000aad4266be Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167525 Commit-Queue: Per Åhgren Reviewed-by: Sam Zackrisson Cr-Commit-Position: refs/heads/master@{#30392} --- .../aec3/block_delay_buffer.cc | 37 +++++++--- .../aec3/block_delay_buffer.h | 7 +- .../aec3/block_delay_buffer_unittest.cc | 71 ++++++++++++------- .../audio_processing/aec3/echo_canceller3.cc | 14 ++-- .../audio_processing/aec3/echo_canceller3.h | 3 +- 5 files changed, 86 insertions(+), 46 deletions(-) diff --git a/modules/audio_processing/aec3/block_delay_buffer.cc b/modules/audio_processing/aec3/block_delay_buffer.cc index 6c1df7c9f9..b9eb3c9f93 100644 --- a/modules/audio_processing/aec3/block_delay_buffer.cc +++ b/modules/audio_processing/aec3/block_delay_buffer.cc @@ -9,35 +9,50 @@ */ #include "modules/audio_processing/aec3/block_delay_buffer.h" +#include "api/array_view.h" #include "rtc_base/checks.h" namespace webrtc { -BlockDelayBuffer::BlockDelayBuffer(size_t num_bands, +BlockDelayBuffer::BlockDelayBuffer(size_t num_channels, + size_t num_bands, size_t frame_length, size_t delay_samples) : frame_length_(frame_length), delay_(delay_samples), - buf_(num_bands, std::vector(delay_, 0.f)) {} + buf_(num_channels, + std::vector>(num_bands, + std::vector(delay_, 0.f))) {} BlockDelayBuffer::~BlockDelayBuffer() = default; void BlockDelayBuffer::DelaySignal(AudioBuffer* frame) { - RTC_DCHECK_EQ(1, frame->num_channels()); - RTC_DCHECK_EQ(buf_.size(), frame->num_bands()); + RTC_DCHECK_EQ(buf_.size(), frame->num_channels()); if (delay_ == 0) { return; } + const size_t num_bands = buf_[0].size(); + const size_t num_channels = buf_.size(); + const size_t i_start = last_insert_; size_t i = 0; - for (size_t j = 0; j < buf_.size(); ++j) { - i = i_start; - for (size_t k = 0; k < frame_length_; ++k) { - const float tmp = buf_[j][i]; - buf_[j][i] = frame->split_bands(0)[j][k]; - frame->split_bands(0)[j][k] = tmp; - i = i < buf_[0].size() - 1 ? i + 1 : 0; + for (size_t ch = 0; ch < num_channels; ++ch) { + RTC_DCHECK_EQ(buf_[ch].size(), frame->num_bands()); + RTC_DCHECK_EQ(buf_[ch].size(), num_bands); + rtc::ArrayView frame_ch(frame->split_bands(ch), num_bands); + + for (size_t band = 0; band < num_bands; ++band) { + RTC_DCHECK_EQ(delay_, buf_[ch][band].size()); + i = i_start; + + for (size_t k = 0; k < frame_length_; ++k) { + const float tmp = buf_[ch][band][i]; + buf_[ch][band][i] = frame_ch[band][k]; + frame_ch[band][k] = tmp; + + i = i < delay_ - 1 ? i + 1 : 0; + } } } diff --git a/modules/audio_processing/aec3/block_delay_buffer.h b/modules/audio_processing/aec3/block_delay_buffer.h index dd57759489..711a790bfe 100644 --- a/modules/audio_processing/aec3/block_delay_buffer.h +++ b/modules/audio_processing/aec3/block_delay_buffer.h @@ -23,7 +23,10 @@ namespace webrtc { // the audiobuffer band-splitting scheme. class BlockDelayBuffer { public: - BlockDelayBuffer(size_t num_bands, size_t frame_length, size_t delay_samples); + BlockDelayBuffer(size_t num_channels, + size_t num_bands, + size_t frame_length, + size_t delay_samples); ~BlockDelayBuffer(); // Delays the samples by the specified delay. @@ -32,7 +35,7 @@ class BlockDelayBuffer { private: const size_t frame_length_; const size_t delay_; - std::vector> buf_; + std::vector>> buf_; size_t last_insert_ = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/block_delay_buffer_unittest.cc b/modules/audio_processing/aec3/block_delay_buffer_unittest.cc index bda1821d0e..011ab49651 100644 --- a/modules/audio_processing/aec3/block_delay_buffer_unittest.cc +++ b/modules/audio_processing/aec3/block_delay_buffer_unittest.cc @@ -47,37 +47,54 @@ std::string ProduceDebugText(int sample_rate_hz, size_t delay) { } // namespace +class BlockDelayBufferTest + : public ::testing::Test, + public ::testing::WithParamInterface> {}; + +INSTANTIATE_TEST_SUITE_P( + ParameterCombinations, + BlockDelayBufferTest, + ::testing::Combine(::testing::Values(0, 1, 27, 160, 4321, 7021), + ::testing::Values(16000, 32000, 48000), + ::testing::Values(1, 2, 4))); + // Verifies that the correct signal delay is achived. -TEST(BlockDelayBuffer, CorrectDelayApplied) { - for (size_t delay : {0, 1, 27, 160, 4321, 7021}) { - for (auto rate : {16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate, delay)); - size_t num_bands = NumBandsForRate(rate); - size_t subband_frame_length = 160; +TEST_P(BlockDelayBufferTest, CorrectDelayApplied) { + const size_t delay = std::get<0>(GetParam()); + const int rate = std::get<1>(GetParam()); + const size_t num_channels = std::get<2>(GetParam()); - BlockDelayBuffer delay_buffer(num_bands, subband_frame_length, delay); + SCOPED_TRACE(ProduceDebugText(rate, delay)); + size_t num_bands = NumBandsForRate(rate); + size_t subband_frame_length = 160; - static constexpr size_t kNumFramesToProcess = 20; - for (size_t frame_index = 0; frame_index < kNumFramesToProcess; - ++frame_index) { - AudioBuffer audio_buffer(rate, 1, rate, 1, rate, 1); - if (rate > 16000) { - audio_buffer.SplitIntoFrequencyBands(); - } - size_t first_sample_index = frame_index * subband_frame_length; - PopulateInputFrame(subband_frame_length, num_bands, first_sample_index, - &audio_buffer.split_bands(0)[0]); - delay_buffer.DelaySignal(&audio_buffer); + BlockDelayBuffer delay_buffer(num_channels, num_bands, subband_frame_length, + delay); - for (size_t k = 0; k < num_bands; ++k) { - size_t sample_index = first_sample_index; - for (size_t i = 0; i < subband_frame_length; ++i, ++sample_index) { - if (sample_index < delay) { - EXPECT_EQ(0.f, audio_buffer.split_bands(0)[k][i]); - } else { - EXPECT_EQ(SampleValue(sample_index - delay), - audio_buffer.split_bands(0)[k][i]); - } + static constexpr size_t kNumFramesToProcess = 20; + for (size_t frame_index = 0; frame_index < kNumFramesToProcess; + ++frame_index) { + AudioBuffer audio_buffer(rate, num_channels, rate, num_channels, rate, + num_channels); + if (rate > 16000) { + audio_buffer.SplitIntoFrequencyBands(); + } + size_t first_sample_index = frame_index * subband_frame_length; + for (size_t ch = 0; ch < num_channels; ++ch) { + PopulateInputFrame(subband_frame_length, num_bands, first_sample_index, + &audio_buffer.split_bands(ch)[0]); + } + delay_buffer.DelaySignal(&audio_buffer); + + for (size_t ch = 0; ch < num_channels; ++ch) { + for (size_t band = 0; band < num_bands; ++band) { + size_t sample_index = first_sample_index; + for (size_t i = 0; i < subband_frame_length; ++i, ++sample_index) { + if (sample_index < delay) { + EXPECT_EQ(0.f, audio_buffer.split_bands(ch)[band][i]); + } else { + EXPECT_EQ(SampleValue(sample_index - delay), + audio_buffer.split_bands(ch)[band][i]); } } } diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index dafb14f544..0a4c61e392 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -338,12 +338,15 @@ EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, std::vector>(num_render_channels_)), capture_sub_frame_view_( num_bands_, - std::vector>(num_capture_channels_)), - block_delay_buffer_(num_bands_, - AudioBuffer::kSplitBandSize, - config_.delay.fixed_capture_delay_samples) { + std::vector>(num_capture_channels_)) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); + if (config_.delay.fixed_capture_delay_samples > 0) { + block_delay_buffer_.reset(new BlockDelayBuffer( + num_capture_channels_, num_bands_, AudioBuffer::kSplitBandSize, + config_.delay.fixed_capture_delay_samples)); + } + render_writer_.reset(new RenderWriter(data_dumper_.get(), &render_transfer_queue_, num_bands_, num_render_channels_)); @@ -417,7 +420,8 @@ void EchoCanceller3::ProcessCapture(AudioBuffer* capture, // Optionally delay the capture signal. if (config_.delay.fixed_capture_delay_samples > 0) { - block_delay_buffer_.DelaySignal(capture); + RTC_DCHECK(block_delay_buffer_); + block_delay_buffer_->DelaySignal(capture); } rtc::ArrayView capture_lower_band = rtc::ArrayView( diff --git a/modules/audio_processing/aec3/echo_canceller3.h b/modules/audio_processing/aec3/echo_canceller3.h index f346f189bf..4bad488d51 100644 --- a/modules/audio_processing/aec3/echo_canceller3.h +++ b/modules/audio_processing/aec3/echo_canceller3.h @@ -182,7 +182,8 @@ class EchoCanceller3 : public EchoControl { RTC_GUARDED_BY(capture_race_checker_); std::vector>> capture_sub_frame_view_ RTC_GUARDED_BY(capture_race_checker_); - BlockDelayBuffer block_delay_buffer_ RTC_GUARDED_BY(capture_race_checker_); + std::unique_ptr block_delay_buffer_ + RTC_GUARDED_BY(capture_race_checker_); ApiCallJitterMetrics api_call_metrics_ RTC_GUARDED_BY(capture_race_checker_); }; } // namespace webrtc