From 398689f5811f691a42b1f628a35c864bdf0ecc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=85hgren?= Date: Thu, 23 Aug 2018 11:38:27 +0200 Subject: [PATCH] AEC3: Adding the option for applying a fixed delay to the capture signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL adds functionality for applying an optional fixed delay in AEC3 to the capture signal Bug: webrtc:9647 Change-Id: Id3b3f896bcf203e6611298dc804c3c80da9f1883 Reviewed-on: https://webrtc-review.googlesource.com/95142 Commit-Queue: Per Åhgren Reviewed-by: Gustaf Ullberg Cr-Commit-Position: refs/heads/master@{#24399} --- api/audio/echo_canceller3_config.cc | 4 + api/audio/echo_canceller3_config.h | 3 + modules/audio_processing/aec3/BUILD.gn | 3 + .../aec3/block_delay_buffer.cc | 47 ++++++++++ .../aec3/block_delay_buffer.h | 38 ++++++++ .../aec3/block_delay_buffer_unittest.cc | 91 +++++++++++++++++++ .../audio_processing/aec3/echo_canceller3.cc | 11 ++- .../audio_processing/aec3/echo_canceller3.h | 3 + .../test/audio_processing_simulator.cc | 2 + 9 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 modules/audio_processing/aec3/block_delay_buffer.cc create mode 100644 modules/audio_processing/aec3/block_delay_buffer.h create mode 100644 modules/audio_processing/aec3/block_delay_buffer_unittest.cc diff --git a/api/audio/echo_canceller3_config.cc b/api/audio/echo_canceller3_config.cc index 9af7f11c57..e955b2fd95 100644 --- a/api/audio/echo_canceller3_config.cc +++ b/api/audio/echo_canceller3_config.cc @@ -14,6 +14,10 @@ namespace webrtc { EchoCanceller3Config::EchoCanceller3Config() = default; EchoCanceller3Config::EchoCanceller3Config(const EchoCanceller3Config& e) = default; +EchoCanceller3Config::Delay::Delay() = default; +EchoCanceller3Config::Delay::Delay(const EchoCanceller3Config::Delay& e) = + default; + EchoCanceller3Config::Mask::Mask() = default; EchoCanceller3Config::Mask::Mask(const EchoCanceller3Config::Mask& m) = default; diff --git a/api/audio/echo_canceller3_config.h b/api/audio/echo_canceller3_config.h index 7b3ced38fd..c649a0944d 100644 --- a/api/audio/echo_canceller3_config.h +++ b/api/audio/echo_canceller3_config.h @@ -20,6 +20,8 @@ struct EchoCanceller3Config { EchoCanceller3Config(); EchoCanceller3Config(const EchoCanceller3Config& e); struct Delay { + Delay(); + Delay(const Delay& e); size_t default_delay = 5; size_t down_sampling_factor = 4; size_t num_filters = 6; @@ -29,6 +31,7 @@ struct EchoCanceller3Config { size_t hysteresis_limit_1_blocks = 1; size_t hysteresis_limit_2_blocks = 1; size_t skew_hysteresis_blocks = 3; + size_t fixed_capture_delay_samples = 0; } delay; struct Filter { diff --git a/modules/audio_processing/aec3/BUILD.gn b/modules/audio_processing/aec3/BUILD.gn index d0df5f7b36..8f8245dfa7 100644 --- a/modules/audio_processing/aec3/BUILD.gn +++ b/modules/audio_processing/aec3/BUILD.gn @@ -20,6 +20,8 @@ rtc_static_library("aec3") { "aec3_fft.h", "aec_state.cc", "aec_state.h", + "block_delay_buffer.cc", + "block_delay_buffer.h", "block_framer.cc", "block_framer.h", "block_processor.cc", @@ -179,6 +181,7 @@ if (rtc_include_tests) { "adaptive_fir_filter_unittest.cc", "aec3_fft_unittest.cc", "aec_state_unittest.cc", + "block_delay_buffer_unittest.cc", "block_framer_unittest.cc", "block_processor_metrics_unittest.cc", "block_processor_unittest.cc", diff --git a/modules/audio_processing/aec3/block_delay_buffer.cc b/modules/audio_processing/aec3/block_delay_buffer.cc new file mode 100644 index 0000000000..0a242eede7 --- /dev/null +++ b/modules/audio_processing/aec3/block_delay_buffer.cc @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/aec3/block_delay_buffer.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +BlockDelayBuffer::BlockDelayBuffer(size_t num_bands, + size_t frame_length, + size_t delay_samples) + : frame_length_(frame_length), + delay_(delay_samples), + buf_(num_bands, std::vector(delay_, 0.f)) {} + +BlockDelayBuffer::~BlockDelayBuffer() = default; + +void BlockDelayBuffer::DelaySignal(AudioBuffer* frame) { + RTC_DCHECK_EQ(1, frame->num_channels()); + RTC_DCHECK_EQ(buf_.size(), frame->num_bands()); + if (delay_ == 0) { + return; + } + + const size_t i_start = last_insert_; + size_t i = 0; + for (size_t j = 0; j < buf_.size(); ++j) { + i = i_start; + for (size_t k = 0; k < frame_length_; ++k) { + const float tmp = buf_[j][i]; + buf_[j][i] = frame->split_bands_f(0)[j][k]; + frame->split_bands_f(0)[j][k] = tmp; + i = i < buf_[0].size() - 1 ? i + 1 : 0; + } + } + + last_insert_ = i; +} + +} // namespace webrtc diff --git a/modules/audio_processing/aec3/block_delay_buffer.h b/modules/audio_processing/aec3/block_delay_buffer.h new file mode 100644 index 0000000000..6e5fd5c09d --- /dev/null +++ b/modules/audio_processing/aec3/block_delay_buffer.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_BLOCK_DELAY_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_BLOCK_DELAY_BUFFER_H_ + +#include + +#include "modules/audio_processing/audio_buffer.h" + +namespace webrtc { + +// Class for applying a fixed delay to the samples in a signal partitioned using +// the audiobuffer band-splitting scheme. +class BlockDelayBuffer { + public: + BlockDelayBuffer(size_t num_bands, size_t frame_length, size_t delay_samples); + ~BlockDelayBuffer(); + + // Delays the samples by the specified delay. + void DelaySignal(AudioBuffer* frame); + + private: + const size_t frame_length_; + const size_t delay_; + std::vector> buf_; + size_t last_insert_ = 0; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_BLOCK_DELAY_BUFFER_H_ diff --git a/modules/audio_processing/aec3/block_delay_buffer_unittest.cc b/modules/audio_processing/aec3/block_delay_buffer_unittest.cc new file mode 100644 index 0000000000..778d43d857 --- /dev/null +++ b/modules/audio_processing/aec3/block_delay_buffer_unittest.cc @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/block_delay_buffer.h" + +#include + +#include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +float SampleValue(size_t sample_index) { + return sample_index % 32768; +} + +// Populates the frame with linearly increasing sample values for each band. +void PopulateInputFrame(size_t frame_length, + size_t num_bands, + size_t first_sample_index, + float* const* frame) { + for (size_t k = 0; k < num_bands; ++k) { + for (size_t i = 0; i < frame_length; ++i) { + frame[k][i] = SampleValue(first_sample_index + i); + } + } +} + +std::string ProduceDebugText(int sample_rate_hz, size_t delay) { + char log_stream_buffer[8 * 1024]; + rtc::SimpleStringBuilder ss(log_stream_buffer); + ss << "Sample rate: " << sample_rate_hz; + ss << ", Delay: " << delay; + return ss.str(); +} + +} // namespace + +// Verifies that the correct signal delay is achived. +TEST(BlockDelayBuffer, CorrectDelayApplied) { + for (size_t delay : {0, 1, 27, 160, 4321, 7021}) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate, delay)); + size_t num_bands = NumBandsForRate(rate); + size_t fullband_frame_length = rate / 100; + size_t subband_frame_length = rate == 8000 ? 80 : 160; + + BlockDelayBuffer delay_buffer(num_bands, subband_frame_length, delay); + + static constexpr size_t kNumFramesToProcess = 20; + for (size_t frame_index = 0; frame_index < kNumFramesToProcess; + ++frame_index) { + AudioBuffer audio_buffer(fullband_frame_length, 1, + fullband_frame_length, 1, + fullband_frame_length); + if (rate > 16000) { + audio_buffer.SplitIntoFrequencyBands(); + } + size_t first_sample_index = frame_index * subband_frame_length; + PopulateInputFrame(subband_frame_length, num_bands, first_sample_index, + &audio_buffer.split_bands_f(0)[0]); + delay_buffer.DelaySignal(&audio_buffer); + + for (size_t k = 0; k < num_bands; ++k) { + size_t sample_index = first_sample_index; + for (size_t i = 0; i < subband_frame_length; ++i, ++sample_index) { + if (sample_index < delay) { + EXPECT_EQ(0.f, audio_buffer.split_bands_f(0)[k][i]); + } else { + EXPECT_EQ(SampleValue(sample_index - delay), + audio_buffer.split_bands_f(0)[k][i]); + } + } + } + } + } + } +} + +} // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index c02bc14863..e5219c72ab 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -342,6 +342,7 @@ EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, std::unique_ptr block_processor) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + config_(config), sample_rate_hz_(sample_rate_hz), num_bands_(NumBandsForRate(sample_rate_hz_)), frame_length_(rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)), @@ -358,7 +359,10 @@ EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, render_queue_output_frame_(num_bands_, std::vector(frame_length_, 0.f)), block_(num_bands_, std::vector(kBlockSize, 0.f)), - sub_frame_view_(num_bands_) { + sub_frame_view_(num_bands_), + block_delay_buffer_(num_bands_, + frame_length_, + config_.delay.fixed_capture_delay_samples) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); std::unique_ptr render_highpass_filter; @@ -421,6 +425,11 @@ void EchoCanceller3::ProcessCapture(AudioBuffer* capture, bool level_change) { data_dumper_->DumpRaw("aec3_call_order", static_cast(EchoCanceller3ApiCall::kCapture)); + // Optionally delay the capture signal. + if (config_.delay.fixed_capture_delay_samples > 0) { + block_delay_buffer_.DelaySignal(capture); + } + rtc::ArrayView capture_lower_band = rtc::ArrayView(&capture->split_bands_f(0)[0][0], frame_length_); diff --git a/modules/audio_processing/aec3/echo_canceller3.h b/modules/audio_processing/aec3/echo_canceller3.h index b66b8b1dd1..f5520ba3ac 100644 --- a/modules/audio_processing/aec3/echo_canceller3.h +++ b/modules/audio_processing/aec3/echo_canceller3.h @@ -12,6 +12,7 @@ #define MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ #include "api/audio/echo_canceller3_config.h" +#include "modules/audio_processing/aec3/block_delay_buffer.h" #include "modules/audio_processing/aec3/block_framer.h" #include "modules/audio_processing/aec3/block_processor.h" #include "modules/audio_processing/aec3/cascaded_biquad_filter.h" @@ -110,6 +111,7 @@ class EchoCanceller3 : public EchoControl { // State that may be accessed by the capture thread. static int instance_count_; std::unique_ptr data_dumper_; + const EchoCanceller3Config config_; const int sample_rate_hz_; const int num_bands_; const size_t frame_length_; @@ -129,6 +131,7 @@ class EchoCanceller3 : public EchoControl { std::vector> block_ RTC_GUARDED_BY(capture_race_checker_); std::vector> sub_frame_view_ RTC_GUARDED_BY(capture_race_checker_); + BlockDelayBuffer block_delay_buffer_ RTC_GUARDED_BY(capture_race_checker_); RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(EchoCanceller3); }; diff --git a/modules/audio_processing/test/audio_processing_simulator.cc b/modules/audio_processing/test/audio_processing_simulator.cc index 160798dd1d..6d0c15a108 100644 --- a/modules/audio_processing/test/audio_processing_simulator.cc +++ b/modules/audio_processing/test/audio_processing_simulator.cc @@ -203,6 +203,8 @@ EchoCanceller3Config ParseAec3Parameters(const std::string& filename) { &cfg.delay.hysteresis_limit_2_blocks); ReadParam(section, "skew_hysteresis_blocks", &cfg.delay.skew_hysteresis_blocks); + ReadParam(section, "fixed_capture_delay_samples", + &cfg.delay.fixed_capture_delay_samples); } if (rtc::GetValueFromJsonObject(root, "filter", §ion)) {