diff --git a/api/audio/echo_canceller3_factory.cc b/api/audio/echo_canceller3_factory.cc index e83e552270..d8d39bc37e 100644 --- a/api/audio/echo_canceller3_factory.cc +++ b/api/audio/echo_canceller3_factory.cc @@ -22,6 +22,17 @@ EchoCanceller3Factory::EchoCanceller3Factory(const EchoCanceller3Config& config) : config_(config) {} std::unique_ptr EchoCanceller3Factory::Create(int sample_rate_hz) { - return absl::make_unique(config_, sample_rate_hz); + return absl::make_unique(config_, sample_rate_hz, + /*num_render_channels=*/1, + /*num_capture_channels=*/1); } + +std::unique_ptr EchoCanceller3Factory::Create( + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) { + return absl::make_unique( + config_, sample_rate_hz, num_render_channels, num_capture_channels); +} + } // namespace webrtc diff --git a/api/audio/echo_canceller3_factory.h b/api/audio/echo_canceller3_factory.h index 9052d99bb1..4637c4502e 100644 --- a/api/audio/echo_canceller3_factory.h +++ b/api/audio/echo_canceller3_factory.h @@ -28,9 +28,16 @@ class RTC_EXPORT EchoCanceller3Factory : public EchoControlFactory { // configuration. explicit EchoCanceller3Factory(const EchoCanceller3Config& config); - // Creates an EchoCanceller3 running at the specified sampling rate. + // Creates an EchoCanceller3 running at the specified sampling rate using a + // mono setup std::unique_ptr Create(int sample_rate_hz) override; + // Creates an EchoCanceller3 running at the specified sampling rate and a + // specified number of channels. + std::unique_ptr Create(int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) override; + private: const EchoCanceller3Config config_; }; diff --git a/api/audio/echo_control.h b/api/audio/echo_control.h index f549f40fbb..44960496b1 100644 --- a/api/audio/echo_control.h +++ b/api/audio/echo_control.h @@ -48,6 +48,11 @@ class EchoControl { class EchoControlFactory { public: virtual std::unique_ptr Create(int sample_rate_hz) = 0; + virtual std::unique_ptr Create(int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) { + return Create(sample_rate_hz); + } virtual ~EchoControlFactory() = default; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc index 821573639e..64d84cdc9e 100644 --- a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc +++ b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc @@ -53,10 +53,17 @@ std::string ProduceDebugText(size_t delay) { // Verifies that the optimized methods for filter adaptation are similar to // their reference counterparts. TEST(AdaptiveFirFilter, FilterAdaptationNeonOptimizations) { + constexpr size_t kNumRenderChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 48000)); + RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz, + kNumRenderChannels)); Random random_generator(42U); - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumRenderChannels, std::vector(kBlockSize, 0.f))); FftData S_C; FftData S_NEON; FftData G; @@ -71,7 +78,11 @@ TEST(AdaptiveFirFilter, FilterAdaptationNeonOptimizations) { } for (size_t k = 0; k < 30; ++k) { - RandomizeSampleVector(&random_generator, x[0]); + for (size_t band = 0; band < x.size(); ++band) { + for (size_t channel = 0; channel < x[band].size(); ++channel) { + RandomizeSampleVector(&random_generator, x[band][channel]); + } + } render_delay_buffer->Insert(x); if (k == 0) { render_delay_buffer->Reset(); @@ -162,12 +173,20 @@ TEST(AdaptiveFirFilter, UpdateErlNeonOptimization) { // Verifies that the optimized methods for filter adaptation are bitexact to // their reference counterparts. TEST(AdaptiveFirFilter, FilterAdaptationSse2Optimizations) { + constexpr size_t kNumRenderChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + bool use_sse2 = (WebRtc_GetCPUInfo(kSSE2) != 0); if (use_sse2) { std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 48000)); + RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz, + kNumRenderChannels)); Random random_generator(42U); - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, + std::vector>(kNumRenderChannels, + std::vector(kBlockSize, 0.f))); FftData S_C; FftData S_SSE2; FftData G; @@ -182,7 +201,11 @@ TEST(AdaptiveFirFilter, FilterAdaptationSse2Optimizations) { } for (size_t k = 0; k < 500; ++k) { - RandomizeSampleVector(&random_generator, x[0]); + for (size_t band = 0; band < x.size(); ++band) { + for (size_t channel = 0; channel < x[band].size(); ++channel) { + RandomizeSampleVector(&random_generator, x[band][channel]); + } + } render_delay_buffer->Insert(x); if (k == 0) { render_delay_buffer->Reset(); @@ -281,7 +304,7 @@ TEST(AdaptiveFirFilter, NullFilterOutput) { ApmDataDumper data_dumper(42); AdaptiveFirFilter filter(9, 9, 250, DetectOptimization(), &data_dumper); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 48000)); + RenderDelayBuffer::Create(EchoCanceller3Config(), 48000, 1)); EXPECT_DEATH(filter.Filter(*render_delay_buffer->GetRenderBuffer(), nullptr), ""); } @@ -310,6 +333,10 @@ TEST(AdaptiveFirFilter, FilterSize) { // Verifies that the filter is being able to properly filter a signal and to // adapt its coefficients. TEST(AdaptiveFirFilter, FilterAndAdapt) { + constexpr size_t kNumRenderChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + constexpr size_t kNumBlocksToProcess = 1000; ApmDataDumper data_dumper(42); EchoCanceller3Config config; @@ -320,11 +347,13 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) { Aec3Fft fft; config.delay.default_delay = 1; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumRenderChannels)); ShadowFilterUpdateGain gain(config.filter.shadow, config.filter.config_change_duration_blocks); Random random_generator(42U); - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumRenderChannels, std::vector(kBlockSize, 0.f))); std::vector n(kBlockSize, 0.f); std::vector y(kBlockSize, 0.f); AecState aec_state(EchoCanceller3Config{}); @@ -357,15 +386,15 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) { SCOPED_TRACE(ProduceDebugText(delay_samples)); for (size_t j = 0; j < kNumBlocksToProcess; ++j) { - RandomizeSampleVector(&random_generator, x[0]); - delay_buffer.Delay(x[0], y); + RandomizeSampleVector(&random_generator, x[0][0]); + delay_buffer.Delay(x[0][0], y); RandomizeSampleVector(&random_generator, n); static constexpr float kNoiseScaling = 1.f / 100.f; std::transform(y.begin(), y.end(), n.begin(), y.begin(), [](float a, float b) { return a + b * kNoiseScaling; }); - x_hp_filter.Process(x[0]); + x_hp_filter.Process(x[0][0]); y_hp_filter.Process(y); render_delay_buffer->Insert(x); diff --git a/modules/audio_processing/aec3/aec3_common.h b/modules/audio_processing/aec3/aec3_common.h index 56c7a9024a..bf554e315b 100644 --- a/modules/audio_processing/aec3/aec3_common.h +++ b/modules/audio_processing/aec3/aec3_common.h @@ -54,16 +54,12 @@ constexpr size_t kMatchedFilterAlignmentShiftSizeSubBlocks = // TODO(peah): Integrate this with how it is done inside audio_processing_impl. constexpr size_t NumBandsForRate(int sample_rate_hz) { - return static_cast(sample_rate_hz == 8000 ? 1 - : sample_rate_hz / 16000); -} -constexpr int LowestBandRate(int sample_rate_hz) { - return sample_rate_hz == 8000 ? sample_rate_hz : 16000; + return static_cast(sample_rate_hz / 16000); } constexpr bool ValidFullBandRate(int sample_rate_hz) { - return sample_rate_hz == 8000 || sample_rate_hz == 16000 || - sample_rate_hz == 32000 || sample_rate_hz == 48000; + return sample_rate_hz == 16000 || sample_rate_hz == 32000 || + sample_rate_hz == 48000; } constexpr int GetTimeDomainLength(int filter_length_blocks) { @@ -100,21 +96,10 @@ static_assert(1 << kBlockSizeLog2 == kBlockSize, static_assert(1 << kFftLengthBy2Log2 == kFftLengthBy2, "Proper number of shifts for the fft length"); -static_assert(1 == NumBandsForRate(8000), "Number of bands for 8 kHz"); static_assert(1 == NumBandsForRate(16000), "Number of bands for 16 kHz"); static_assert(2 == NumBandsForRate(32000), "Number of bands for 32 kHz"); static_assert(3 == NumBandsForRate(48000), "Number of bands for 48 kHz"); -static_assert(8000 == LowestBandRate(8000), "Sample rate of band 0 for 8 kHz"); -static_assert(16000 == LowestBandRate(16000), - "Sample rate of band 0 for 16 kHz"); -static_assert(16000 == LowestBandRate(32000), - "Sample rate of band 0 for 32 kHz"); -static_assert(16000 == LowestBandRate(48000), - "Sample rate of band 0 for 48 kHz"); - -static_assert(ValidFullBandRate(8000), - "Test that 8 kHz is a valid sample rate"); static_assert(ValidFullBandRate(16000), "Test that 16 kHz is a valid sample rate"); static_assert(ValidFullBandRate(32000), diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index eab009444f..566c62fbfe 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -121,7 +121,7 @@ void AecState::Update( } const std::vector& aligned_render_block = - render_buffer.Block(-delay_state_.DirectPathFilterDelay())[0]; + render_buffer.Block(-delay_state_.DirectPathFilterDelay())[0][0]; // Update render counters. const float render_energy = std::inner_product( diff --git a/modules/audio_processing/aec3/aec_state_unittest.cc b/modules/audio_processing/aec3/aec_state_unittest.cc index bf47a05f30..4631eac39e 100644 --- a/modules/audio_processing/aec3/aec_state_unittest.cc +++ b/modules/audio_processing/aec3/aec_state_unittest.cc @@ -19,16 +19,21 @@ namespace webrtc { // Verify the general functionality of AecState TEST(AecState, NormalUsage) { + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); ApmDataDumper data_dumper(42); EchoCanceller3Config config; AecState state(config); absl::optional delay_estimate = DelayEstimate(DelayEstimate::Quality::kRefined, 10); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); std::array E2_main = {}; std::array Y2 = {}; - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); EchoPathVariability echo_path_variability( false, EchoPathVariability::DelayAdjustment::kNone, false); SubtractorOutput output; @@ -53,7 +58,11 @@ TEST(AecState, NormalUsage) { GetTimeDomainLength(config.filter.main.length_blocks), 0.f); // Verify that linear AEC usability is true when the filter is converged - std::fill(x[0].begin(), x[0].end(), 101.f); + for (size_t band = 0; band < kNumBands; ++band) { + for (size_t channel = 0; channel < kNumChannels; ++channel) { + std::fill(x[band][channel].begin(), x[band][channel].end(), 101.f); + } + } for (int k = 0; k < 3000; ++k) { render_delay_buffer->Insert(x); output.ComputeMetrics(y); @@ -74,7 +83,7 @@ TEST(AecState, NormalUsage) { EXPECT_FALSE(state.UsableLinearEstimate()); // Verify that the active render detection works as intended. - std::fill(x[0].begin(), x[0].end(), 101.f); + std::fill(x[0][0].begin(), x[0][0].end(), 101.f); render_delay_buffer->Insert(x); output.ComputeMetrics(y); state.HandleEchoPathChange(EchoPathVariability( @@ -94,11 +103,13 @@ TEST(AecState, NormalUsage) { EXPECT_TRUE(state.ActiveRender()); // Verify that the ERL is properly estimated - for (auto& x_k : x) { - x_k = std::vector(kBlockSize, 0.f); + for (auto& band : x) { + for (auto& channel : band) { + channel = std::vector(kBlockSize, 0.f); + } } - x[0][0] = 5000.f; + x[0][0][0] = 5000.f; for (size_t k = 0; k < render_delay_buffer->GetRenderBuffer()->GetFftBuffer().size(); ++k) { render_delay_buffer->Insert(x); @@ -179,7 +190,7 @@ TEST(AecState, ConvergedFilterDelay) { EchoCanceller3Config config; AecState state(config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, 48000, 1)); absl::optional delay_estimate; std::array E2_main; std::array Y2; diff --git a/modules/audio_processing/aec3/block_delay_buffer_unittest.cc b/modules/audio_processing/aec3/block_delay_buffer_unittest.cc index ec825baea3..bda1821d0e 100644 --- a/modules/audio_processing/aec3/block_delay_buffer_unittest.cc +++ b/modules/audio_processing/aec3/block_delay_buffer_unittest.cc @@ -50,10 +50,10 @@ std::string ProduceDebugText(int sample_rate_hz, size_t delay) { // Verifies that the correct signal delay is achived. TEST(BlockDelayBuffer, CorrectDelayApplied) { for (size_t delay : {0, 1, 27, 160, 4321, 7021}) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate, delay)); size_t num_bands = NumBandsForRate(rate); - size_t subband_frame_length = rate == 8000 ? 80 : 160; + size_t subband_frame_length = 160; BlockDelayBuffer delay_buffer(num_bands, subband_frame_length, delay); diff --git a/modules/audio_processing/aec3/block_framer.cc b/modules/audio_processing/aec3/block_framer.cc index ca7667c24f..8241ce64f2 100644 --- a/modules/audio_processing/aec3/block_framer.cc +++ b/modules/audio_processing/aec3/block_framer.cc @@ -17,9 +17,16 @@ namespace webrtc { -BlockFramer::BlockFramer(size_t num_bands) +BlockFramer::BlockFramer(size_t num_bands, size_t num_channels) : num_bands_(num_bands), - buffer_(num_bands_, std::vector(kBlockSize, 0.f)) {} + num_channels_(num_channels), + buffer_(num_bands_, + std::vector>( + num_channels, + std::vector(kBlockSize, 0.f))) { + RTC_DCHECK_LT(0, num_bands); + RTC_DCHECK_LT(0, num_channels); +} BlockFramer::~BlockFramer() = default; @@ -27,33 +34,52 @@ BlockFramer::~BlockFramer() = default; // samples for InsertBlockAndExtractSubFrame to produce a frame. In order to // achieve this, the InsertBlockAndExtractSubFrame and InsertBlock methods need // to be called in the correct order. -void BlockFramer::InsertBlock(const std::vector>& block) { +void BlockFramer::InsertBlock( + const std::vector>>& block) { RTC_DCHECK_EQ(num_bands_, block.size()); - for (size_t i = 0; i < num_bands_; ++i) { - RTC_DCHECK_EQ(kBlockSize, block[i].size()); - RTC_DCHECK_EQ(0, buffer_[i].size()); - buffer_[i].insert(buffer_[i].begin(), block[i].begin(), block[i].end()); + for (size_t band = 0; band < num_bands_; ++band) { + RTC_DCHECK_EQ(num_channels_, block[band].size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + RTC_DCHECK_EQ(kBlockSize, block[band][channel].size()); + RTC_DCHECK_EQ(0, buffer_[band][channel].size()); + + buffer_[band][channel].insert(buffer_[band][channel].begin(), + block[band][channel].begin(), + block[band][channel].end()); + } } } void BlockFramer::InsertBlockAndExtractSubFrame( - const std::vector>& block, - std::vector>* sub_frame) { + const std::vector>>& block, + std::vector>>* sub_frame) { RTC_DCHECK(sub_frame); RTC_DCHECK_EQ(num_bands_, block.size()); RTC_DCHECK_EQ(num_bands_, sub_frame->size()); - for (size_t i = 0; i < num_bands_; ++i) { - RTC_DCHECK_LE(kSubFrameLength, buffer_[i].size() + kBlockSize); - RTC_DCHECK_EQ(kBlockSize, block[i].size()); - RTC_DCHECK_GE(kBlockSize, buffer_[i].size()); - RTC_DCHECK_EQ(kSubFrameLength, (*sub_frame)[i].size()); - const int samples_to_frame = kSubFrameLength - buffer_[i].size(); - std::copy(buffer_[i].begin(), buffer_[i].end(), (*sub_frame)[i].begin()); - std::copy(block[i].begin(), block[i].begin() + samples_to_frame, - (*sub_frame)[i].begin() + buffer_[i].size()); - buffer_[i].clear(); - buffer_[i].insert(buffer_[i].begin(), block[i].begin() + samples_to_frame, - block[i].end()); + for (size_t band = 0; band < num_bands_; ++band) { + RTC_DCHECK_EQ(num_channels_, block[band].size()); + RTC_DCHECK_EQ(num_channels_, (*sub_frame)[0].size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + RTC_DCHECK_LE(kSubFrameLength, + buffer_[band][channel].size() + kBlockSize); + RTC_DCHECK_EQ(kBlockSize, block[band][channel].size()); + RTC_DCHECK_GE(kBlockSize, buffer_[band][channel].size()); + RTC_DCHECK_EQ(kSubFrameLength, (*sub_frame)[band][channel].size()); + + const int samples_to_frame = + kSubFrameLength - buffer_[band][channel].size(); + std::copy(buffer_[band][channel].begin(), buffer_[band][channel].end(), + (*sub_frame)[band][channel].begin()); + std::copy( + block[band][channel].begin(), + block[band][channel].begin() + samples_to_frame, + (*sub_frame)[band][channel].begin() + buffer_[band][channel].size()); + buffer_[band][channel].clear(); + buffer_[band][channel].insert( + buffer_[band][channel].begin(), + block[band][channel].begin() + samples_to_frame, + block[band][channel].end()); + } } } diff --git a/modules/audio_processing/aec3/block_framer.h b/modules/audio_processing/aec3/block_framer.h index fae4b2990d..1d378660c3 100644 --- a/modules/audio_processing/aec3/block_framer.h +++ b/modules/audio_processing/aec3/block_framer.h @@ -15,11 +15,10 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { -// Class for producing frames consisting of 1 or 2 subframes of 80 samples each +// Class for producing frames consisting of 2 subframes of 80 samples each // from 64 sample blocks. The class is designed to work together with the // FrameBlocker class which performs the reverse conversion. Used together with // that, this class produces output frames are the same rate as frames are @@ -27,20 +26,22 @@ namespace webrtc { // overrun if any other rate of packets insertion is used. class BlockFramer { public: - explicit BlockFramer(size_t num_bands); + BlockFramer(size_t num_bands, size_t num_channels); ~BlockFramer(); + BlockFramer(const BlockFramer&) = delete; + BlockFramer& operator=(const BlockFramer&) = delete; + // Adds a 64 sample block into the data that will form the next output frame. - void InsertBlock(const std::vector>& block); + void InsertBlock(const std::vector>>& block); // Adds a 64 sample block and extracts an 80 sample subframe. void InsertBlockAndExtractSubFrame( - const std::vector>& block, - std::vector>* sub_frame); + const std::vector>>& block, + std::vector>>* sub_frame); private: const size_t num_bands_; - std::vector> buffer_; - - RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockFramer); + const size_t num_channels_; + std::vector>> buffer_; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/block_framer_unittest.cc b/modules/audio_processing/aec3/block_framer_unittest.cc index 9baade98d4..e9a16d06d5 100644 --- a/modules/audio_processing/aec3/block_framer_unittest.cc +++ b/modules/audio_processing/aec3/block_framer_unittest.cc @@ -20,66 +20,87 @@ namespace webrtc { namespace { -void SetupSubFrameView(std::vector>* sub_frame, - std::vector>* sub_frame_view) { - for (size_t k = 0; k < sub_frame_view->size(); ++k) { - (*sub_frame_view)[k] = - rtc::ArrayView((*sub_frame)[k].data(), (*sub_frame)[k].size()); +void SetupSubFrameView( + std::vector>>* sub_frame, + std::vector>>* sub_frame_view) { + for (size_t band = 0; band < sub_frame_view->size(); ++band) { + for (size_t channel = 0; channel < (*sub_frame_view)[band].size(); + ++channel) { + (*sub_frame_view)[band][channel] = + rtc::ArrayView((*sub_frame)[band][channel].data(), + (*sub_frame)[band][channel].size()); + } } } float ComputeSampleValue(size_t chunk_counter, size_t chunk_size, size_t band, + size_t channel, size_t sample_index, int offset) { - float value = - static_cast(chunk_counter * chunk_size + sample_index) + offset; - return value > 0 ? 5000 * band + value : 0; + float value = static_cast(100 + chunk_counter * chunk_size + + sample_index + channel) + + offset; + return 5000 * band + value; } -bool VerifySubFrame(size_t sub_frame_counter, - int offset, - const std::vector>& sub_frame_view) { - for (size_t k = 0; k < sub_frame_view.size(); ++k) { - for (size_t i = 0; i < sub_frame_view[k].size(); ++i) { - const float reference_value = - ComputeSampleValue(sub_frame_counter, kSubFrameLength, k, i, offset); - if (reference_value != sub_frame_view[k][i]) { - return false; +bool VerifySubFrame( + size_t sub_frame_counter, + int offset, + const std::vector>>& sub_frame_view) { + for (size_t band = 0; band < sub_frame_view.size(); ++band) { + for (size_t channel = 0; channel < sub_frame_view[band].size(); ++channel) { + for (size_t sample = 0; sample < sub_frame_view[band][channel].size(); + ++sample) { + const float reference_value = ComputeSampleValue( + sub_frame_counter, kSubFrameLength, band, channel, sample, offset); + if (reference_value != sub_frame_view[band][channel][sample]) { + return false; + } } } } return true; } -void FillBlock(size_t block_counter, std::vector>* block) { - for (size_t k = 0; k < block->size(); ++k) { - for (size_t i = 0; i < (*block)[0].size(); ++i) { - (*block)[k][i] = ComputeSampleValue(block_counter, kBlockSize, k, i, 0); +void FillBlock(size_t block_counter, + std::vector>>* block) { + for (size_t band = 0; band < block->size(); ++band) { + for (size_t channel = 0; channel < (*block)[band].size(); ++channel) { + for (size_t sample = 0; sample < (*block)[band][channel].size(); + ++sample) { + (*block)[band][channel][sample] = ComputeSampleValue( + block_counter, kBlockSize, band, channel, sample, 0); + } } } } // Verifies that the BlockFramer is able to produce the expected frame content. -void RunFramerTest(int sample_rate_hz) { - constexpr size_t kNumSubFramesToProcess = 2; +void RunFramerTest(int sample_rate_hz, size_t num_channels) { + constexpr size_t kNumSubFramesToProcess = 10; const size_t num_bands = NumBandsForRate(sample_rate_hz); - std::vector> block(num_bands, - std::vector(kBlockSize, 0.f)); - std::vector> output_sub_frame( - num_bands, std::vector(kSubFrameLength, 0.f)); - std::vector> output_sub_frame_view(num_bands); + std::vector>> block( + num_bands, std::vector>( + num_channels, std::vector(kBlockSize, 0.f))); + std::vector>> output_sub_frame( + num_bands, std::vector>( + num_channels, std::vector(kSubFrameLength, 0.f))); + std::vector>> output_sub_frame_view( + num_bands, std::vector>(num_channels)); SetupSubFrameView(&output_sub_frame, &output_sub_frame_view); - BlockFramer framer(num_bands); + BlockFramer framer(num_bands, num_channels); size_t block_index = 0; for (size_t sub_frame_index = 0; sub_frame_index < kNumSubFramesToProcess; ++sub_frame_index) { FillBlock(block_index++, &block); framer.InsertBlockAndExtractSubFrame(block, &output_sub_frame_view); - EXPECT_TRUE(VerifySubFrame(sub_frame_index, -64, output_sub_frame_view)); + if (sub_frame_index > 1) { + EXPECT_TRUE(VerifySubFrame(sub_frame_index, -64, output_sub_frame_view)); + } if ((sub_frame_index + 1) % 4 == 0) { FillBlock(block_index++, &block); @@ -91,21 +112,30 @@ void RunFramerTest(int sample_rate_hz) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the BlockFramer crashes if the InsertBlockAndExtractSubFrame // method is called for inputs with the wrong number of bands or band lengths. -void RunWronglySizedInsertAndExtractParametersTest(int sample_rate_hz, - size_t num_block_bands, - size_t block_length, - size_t num_sub_frame_bands, - size_t sub_frame_length) { +void RunWronglySizedInsertAndExtractParametersTest( + int sample_rate_hz, + size_t correct_num_channels, + size_t num_block_bands, + size_t num_block_channels, + size_t block_length, + size_t num_sub_frame_bands, + size_t num_sub_frame_channels, + size_t sub_frame_length) { const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); - std::vector> block(num_block_bands, - std::vector(block_length, 0.f)); - std::vector> output_sub_frame( - num_sub_frame_bands, std::vector(sub_frame_length, 0.f)); - std::vector> output_sub_frame_view( - output_sub_frame.size()); + std::vector>> block( + num_block_bands, + std::vector>(num_block_channels, + std::vector(block_length, 0.f))); + std::vector>> output_sub_frame( + num_sub_frame_bands, + std::vector>( + num_sub_frame_channels, std::vector(sub_frame_length, 0.f))); + std::vector>> output_sub_frame_view( + output_sub_frame.size(), + std::vector>(num_sub_frame_channels)); SetupSubFrameView(&output_sub_frame, &output_sub_frame_view); - BlockFramer framer(correct_num_bands); + BlockFramer framer(correct_num_bands, correct_num_channels); EXPECT_DEATH( framer.InsertBlockAndExtractSubFrame(block, &output_sub_frame_view), ""); } @@ -113,20 +143,29 @@ void RunWronglySizedInsertAndExtractParametersTest(int sample_rate_hz, // Verifies that the BlockFramer crashes if the InsertBlock method is called for // inputs with the wrong number of bands or band lengths. void RunWronglySizedInsertParameterTest(int sample_rate_hz, + size_t correct_num_channels, size_t num_block_bands, + size_t num_block_channels, size_t block_length) { const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); - std::vector> correct_block( - correct_num_bands, std::vector(kBlockSize, 0.f)); - std::vector> wrong_block( - num_block_bands, std::vector(block_length, 0.f)); - std::vector> output_sub_frame( - correct_num_bands, std::vector(kSubFrameLength, 0.f)); - std::vector> output_sub_frame_view( - output_sub_frame.size()); + std::vector>> correct_block( + correct_num_bands, + std::vector>(correct_num_channels, + std::vector(kBlockSize, 0.f))); + std::vector>> wrong_block( + num_block_bands, + std::vector>(num_block_channels, + std::vector(block_length, 0.f))); + std::vector>> output_sub_frame( + correct_num_bands, + std::vector>( + correct_num_channels, std::vector(kSubFrameLength, 0.f))); + std::vector>> output_sub_frame_view( + output_sub_frame.size(), + std::vector>(correct_num_channels)); SetupSubFrameView(&output_sub_frame, &output_sub_frame_view); - BlockFramer framer(correct_num_bands); + BlockFramer framer(correct_num_bands, correct_num_channels); framer.InsertBlockAndExtractSubFrame(correct_block, &output_sub_frame_view); framer.InsertBlockAndExtractSubFrame(correct_block, &output_sub_frame_view); framer.InsertBlockAndExtractSubFrame(correct_block, &output_sub_frame_view); @@ -138,18 +177,25 @@ void RunWronglySizedInsertParameterTest(int sample_rate_hz, // Verifies that the BlockFramer crashes if the InsertBlock method is called // after a wrong number of previous InsertBlockAndExtractSubFrame method calls // have been made. + void RunWronglyInsertOrderTest(int sample_rate_hz, + size_t num_channels, size_t num_preceeding_api_calls) { const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); - std::vector> block(correct_num_bands, - std::vector(kBlockSize, 0.f)); - std::vector> output_sub_frame( - correct_num_bands, std::vector(kSubFrameLength, 0.f)); - std::vector> output_sub_frame_view( - output_sub_frame.size()); + std::vector>> block( + correct_num_bands, + std::vector>(num_channels, + std::vector(kBlockSize, 0.f))); + std::vector>> output_sub_frame( + correct_num_bands, + std::vector>( + num_channels, std::vector(kSubFrameLength, 0.f))); + std::vector>> output_sub_frame_view( + output_sub_frame.size(), + std::vector>(num_channels)); SetupSubFrameView(&output_sub_frame, &output_sub_frame_view); - BlockFramer framer(correct_num_bands); + BlockFramer framer(correct_num_bands, num_channels); for (size_t k = 0; k < num_preceeding_api_calls; ++k) { framer.InsertBlockAndExtractSubFrame(block, &output_sub_frame_view); } @@ -158,9 +204,10 @@ void RunWronglyInsertOrderTest(int sample_rate_hz, } #endif -std::string ProduceDebugText(int sample_rate_hz) { +std::string ProduceDebugText(int sample_rate_hz, size_t num_channels) { rtc::StringBuilder ss; ss << "Sample rate: " << sample_rate_hz; + ss << ", number of channels: " << num_channels; return ss.Release(); } @@ -168,83 +215,157 @@ std::string ProduceDebugText(int sample_rate_hz) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) TEST(BlockFramer, WrongNumberOfBandsInBlockForInsertBlockAndExtractSubFrame) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - const size_t wrong_num_bands = (correct_num_bands % 3) + 1; - RunWronglySizedInsertAndExtractParametersTest( - rate, wrong_num_bands, kBlockSize, correct_num_bands, kSubFrameLength); + for (auto rate : {16000, 32000, 48000}) { + for (auto correct_num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, wrong_num_bands, correct_num_channels, + kBlockSize, correct_num_bands, correct_num_channels, kSubFrameLength); + } + } +} + +TEST(BlockFramer, + WrongNumberOfChannelsInBlockForInsertBlockAndExtractSubFrame) { + for (auto rate : {16000, 32000, 48000}) { + for (auto correct_num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_channels = correct_num_channels + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, wrong_num_channels, + kBlockSize, correct_num_bands, correct_num_channels, kSubFrameLength); + } } } TEST(BlockFramer, WrongNumberOfBandsInSubFrameForInsertBlockAndExtractSubFrame) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - const size_t wrong_num_bands = (correct_num_bands % 3) + 1; - RunWronglySizedInsertAndExtractParametersTest( - rate, correct_num_bands, kBlockSize, wrong_num_bands, kSubFrameLength); + for (auto rate : {16000, 32000, 48000}) { + for (auto correct_num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, correct_num_channels, + kBlockSize, wrong_num_bands, correct_num_channels, kSubFrameLength); + } + } +} + +TEST(BlockFramer, + WrongNumberOfChannelsInSubFrameForInsertBlockAndExtractSubFrame) { + for (auto rate : {16000, 32000, 48000}) { + for (auto correct_num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_channels = correct_num_channels + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, correct_num_channels, + kBlockSize, correct_num_bands, wrong_num_channels, kSubFrameLength); + } } } TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlockAndExtractSubFrame) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - RunWronglySizedInsertAndExtractParametersTest( - rate, correct_num_bands, kBlockSize - 1, correct_num_bands, - kSubFrameLength); + for (auto rate : {16000, 32000, 48000}) { + for (auto correct_num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, correct_num_channels, + kBlockSize - 1, correct_num_bands, correct_num_channels, + kSubFrameLength); + } } } TEST(BlockFramer, WrongNumberOfSamplesInSubFrameForInsertBlockAndExtractSubFrame) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); + const size_t correct_num_channels = 1; + for (auto rate : {16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); const size_t correct_num_bands = NumBandsForRate(rate); - RunWronglySizedInsertAndExtractParametersTest(rate, correct_num_bands, - kBlockSize, correct_num_bands, - kSubFrameLength - 1); + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, correct_num_channels, + kBlockSize, correct_num_bands, correct_num_channels, + kSubFrameLength - 1); } } TEST(BlockFramer, WrongNumberOfBandsInBlockForInsertBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - const size_t wrong_num_bands = (correct_num_bands % 3) + 1; - RunWronglySizedInsertParameterTest(rate, wrong_num_bands, kBlockSize); - } -} - -TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - RunWronglySizedInsertParameterTest(rate, correct_num_bands, kBlockSize - 1); - } -} - -TEST(BlockFramer, WrongNumberOfPreceedingApiCallsForInsertBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - for (size_t num_calls = 0; num_calls < 4; ++num_calls) { - rtc::StringBuilder ss; - ss << "Sample rate: " << rate; - ss << ", Num preceeding InsertBlockAndExtractSubFrame calls: " - << num_calls; - - SCOPED_TRACE(ss.str()); - RunWronglyInsertOrderTest(rate, num_calls); + for (auto rate : {16000, 32000, 48000}) { + for (auto correct_num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertParameterTest(rate, correct_num_channels, + wrong_num_bands, correct_num_channels, + kBlockSize); } } } -// Verifiers that the verification for null sub_frame pointer works. +TEST(BlockFramer, WrongNumberOfChannelsInBlockForInsertBlock) { + for (auto rate : {16000, 32000, 48000}) { + for (auto correct_num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_channels = correct_num_channels + 1; + RunWronglySizedInsertParameterTest(rate, correct_num_channels, + correct_num_bands, wrong_num_channels, + kBlockSize); + } + } +} + +TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlock) { + for (auto rate : {16000, 32000, 48000}) { + for (auto correct_num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertParameterTest(rate, correct_num_channels, + correct_num_bands, + correct_num_channels, kBlockSize - 1); + } + } +} + +TEST(BlockFramer, WrongNumberOfPreceedingApiCallsForInsertBlock) { + for (size_t num_channels : {1, 2, 8}) { + for (auto rate : {16000, 32000, 48000}) { + for (size_t num_calls = 0; num_calls < 4; ++num_calls) { + rtc::StringBuilder ss; + ss << "Sample rate: " << rate; + ss << ", Num channels: " << num_channels; + ss << ", Num preceeding InsertBlockAndExtractSubFrame calls: " + << num_calls; + + SCOPED_TRACE(ss.str()); + RunWronglyInsertOrderTest(rate, num_channels, num_calls); + } + } + } +} + +// Verifies that the verification for 0 number of channels works. +TEST(BlockFramer, ZeroNumberOfChannelsParameter) { + EXPECT_DEATH(BlockFramer(16000, 0), ""); +} + +// Verifies that the verification for 0 number of bands works. +TEST(BlockFramer, ZeroNumberOfBandsParameter) { + EXPECT_DEATH(BlockFramer(0, 1), ""); +} + +// Verifies that the verification for null sub_frame pointer works. TEST(BlockFramer, NullSubFrameParameter) { - EXPECT_DEATH(BlockFramer(1).InsertBlockAndExtractSubFrame( - std::vector>( - 1, std::vector(kBlockSize, 0.f)), + EXPECT_DEATH(BlockFramer(1, 1).InsertBlockAndExtractSubFrame( + std::vector>>( + 1, std::vector>( + 1, std::vector(kBlockSize, 0.f))), nullptr), ""); } @@ -252,9 +373,11 @@ TEST(BlockFramer, NullSubFrameParameter) { #endif TEST(BlockFramer, FrameBitexactness) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - RunFramerTest(rate); + for (auto rate : {16000, 32000, 48000}) { + for (auto num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, num_channels)); + RunFramerTest(rate, num_channels); + } } } diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index 184248fc23..33b6b9bcd2 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -39,6 +39,8 @@ class BlockProcessorImpl final : public BlockProcessor { public: BlockProcessorImpl(const EchoCanceller3Config& config, int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover); @@ -47,11 +49,13 @@ class BlockProcessorImpl final : public BlockProcessor { ~BlockProcessorImpl() override; - void ProcessCapture(bool echo_path_gain_change, - bool capture_signal_saturation, - std::vector>* capture_block) override; + void ProcessCapture( + bool echo_path_gain_change, + bool capture_signal_saturation, + std::vector>>* capture_block) override; - void BufferRender(const std::vector>& block) override; + void BufferRender( + const std::vector>>& block) override; void UpdateEchoLeakageStatus(bool leakage_detected) override; @@ -80,6 +84,8 @@ int BlockProcessorImpl::instance_count_ = 0; BlockProcessorImpl::BlockProcessorImpl( const EchoCanceller3Config& config, int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover) @@ -99,18 +105,17 @@ BlockProcessorImpl::~BlockProcessorImpl() = default; void BlockProcessorImpl::ProcessCapture( bool echo_path_gain_change, bool capture_signal_saturation, - std::vector>* capture_block) { + std::vector>>* capture_block) { RTC_DCHECK(capture_block); RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size()); - RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size()); + RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0][0].size()); capture_call_counter_++; data_dumper_->DumpRaw("aec3_processblock_call_order", static_cast(BlockProcessorApiCall::kCapture)); data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize, - &(*capture_block)[0][0], - LowestBandRate(sample_rate_hz_), 1); + &(*capture_block)[0][0][0], 16000, 1); if (render_properly_started_) { if (!capture_properly_started_) { @@ -151,8 +156,7 @@ void BlockProcessorImpl::ProcessCapture( } data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, - &(*capture_block)[0][0], - LowestBandRate(sample_rate_hz_), 1); + &(*capture_block)[0][0][0], 16000, 1); bool has_delay_estimator = !config_.delay.use_external_delay_estimator; if (has_delay_estimator) { @@ -161,7 +165,7 @@ void BlockProcessorImpl::ProcessCapture( // alignment. estimated_delay_ = delay_controller_->GetDelay( render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(), - (*capture_block)[0]); + (*capture_block)[0][0]); if (estimated_delay_) { bool delay_change = @@ -192,15 +196,15 @@ void BlockProcessorImpl::ProcessCapture( } void BlockProcessorImpl::BufferRender( - const std::vector>& block) { + const std::vector>>& block) { RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size()); - RTC_DCHECK_EQ(kBlockSize, block[0].size()); + RTC_DCHECK_EQ(kBlockSize, block[0][0].size()); data_dumper_->DumpRaw("aec3_processblock_call_order", static_cast(BlockProcessorApiCall::kRender)); data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize, - &block[0][0], LowestBandRate(sample_rate_hz_), 1); + &block[0][0][0], 16000, 1); data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize, - &block[0][0], LowestBandRate(sample_rate_hz_), 1); + &block[0][0][0], 16000, 1); render_event_ = render_buffer_->Insert(block); @@ -218,7 +222,7 @@ void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const { echo_remover_->GetMetrics(metrics); - const int block_size_ms = sample_rate_hz_ == 8000 ? 8 : 4; + constexpr int block_size_ms = 4; absl::optional delay = render_buffer_->Delay(); metrics->delay_ms = delay ? static_cast(*delay) * block_size_ms : 0; } @@ -230,44 +234,53 @@ void BlockProcessorImpl::SetAudioBufferDelay(size_t delay_ms) { } // namespace BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, - int sample_rate_hz) { + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) { std::unique_ptr render_buffer( - RenderDelayBuffer::Create(config, sample_rate_hz)); + RenderDelayBuffer::Create(config, sample_rate_hz, num_render_channels)); std::unique_ptr delay_controller; if (!config.delay.use_external_delay_estimator) { delay_controller.reset( RenderDelayController::Create(config, sample_rate_hz)); } - std::unique_ptr echo_remover( - EchoRemover::Create(config, sample_rate_hz)); - return Create(config, sample_rate_hz, std::move(render_buffer), + std::unique_ptr echo_remover(EchoRemover::Create( + config, sample_rate_hz, num_render_channels, num_capture_channels)); + return Create(config, sample_rate_hz, num_render_channels, + num_capture_channels, std::move(render_buffer), std::move(delay_controller), std::move(echo_remover)); } BlockProcessor* BlockProcessor::Create( const EchoCanceller3Config& config, int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels, std::unique_ptr render_buffer) { std::unique_ptr delay_controller; if (!config.delay.use_external_delay_estimator) { delay_controller.reset( RenderDelayController::Create(config, sample_rate_hz)); } - std::unique_ptr echo_remover( - EchoRemover::Create(config, sample_rate_hz)); - return Create(config, sample_rate_hz, std::move(render_buffer), + std::unique_ptr echo_remover(EchoRemover::Create( + config, sample_rate_hz, num_render_channels, num_capture_channels)); + return Create(config, sample_rate_hz, num_render_channels, + num_capture_channels, std::move(render_buffer), std::move(delay_controller), std::move(echo_remover)); } BlockProcessor* BlockProcessor::Create( const EchoCanceller3Config& config, int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover) { - return new BlockProcessorImpl( - config, sample_rate_hz, std::move(render_buffer), - std::move(delay_controller), std::move(echo_remover)); + return new BlockProcessorImpl(config, sample_rate_hz, num_render_channels, + num_capture_channels, std::move(render_buffer), + std::move(delay_controller), + std::move(echo_remover)); } } // namespace webrtc diff --git a/modules/audio_processing/aec3/block_processor.h b/modules/audio_processing/aec3/block_processor.h index 8b1bb908f3..3ae5a75a8a 100644 --- a/modules/audio_processing/aec3/block_processor.h +++ b/modules/audio_processing/aec3/block_processor.h @@ -28,15 +28,21 @@ namespace webrtc { class BlockProcessor { public: static BlockProcessor* Create(const EchoCanceller3Config& config, - int sample_rate_hz); + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels); // Only used for testing purposes. static BlockProcessor* Create( const EchoCanceller3Config& config, int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels, std::unique_ptr render_buffer); static BlockProcessor* Create( const EchoCanceller3Config& config, int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover); @@ -53,11 +59,11 @@ class BlockProcessor { virtual void ProcessCapture( bool echo_path_gain_change, bool capture_signal_saturation, - std::vector>* capture_block) = 0; + std::vector>>* capture_block) = 0; // Buffers a block of render data supplied by a FrameBlocker object. virtual void BufferRender( - const std::vector>& render_block) = 0; + const std::vector>>& render_block) = 0; // Reports whether echo leakage has been detected in the echo canceller // output. diff --git a/modules/audio_processing/aec3/block_processor_unittest.cc b/modules/audio_processing/aec3/block_processor_unittest.cc index bd085da4e7..9c315e19f0 100644 --- a/modules/audio_processing/aec3/block_processor_unittest.cc +++ b/modules/audio_processing/aec3/block_processor_unittest.cc @@ -36,11 +36,16 @@ using ::testing::StrictMock; // Verifies that the basic BlockProcessor functionality works and that the API // methods are callable. void RunBasicSetupAndApiCallTest(int sample_rate_hz, int num_iterations) { - std::unique_ptr block_processor( - BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); - std::vector> block(NumBandsForRate(sample_rate_hz), - std::vector(kBlockSize, 1000.f)); + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; + std::unique_ptr block_processor( + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz, + kNumRenderChannels, kNumCaptureChannels)); + std::vector>> block( + NumBandsForRate(sample_rate_hz), + std::vector>(kNumRenderChannels, + std::vector(kBlockSize, 1000.f))); for (int k = 0; k < num_iterations; ++k) { block_processor->BufferRender(block); block_processor->ProcessCapture(false, false, &block); @@ -50,43 +55,67 @@ void RunBasicSetupAndApiCallTest(int sample_rate_hz, int num_iterations) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) void RunRenderBlockSizeVerificationTest(int sample_rate_hz) { + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; + std::unique_ptr block_processor( - BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); - std::vector> block( - NumBandsForRate(sample_rate_hz), std::vector(kBlockSize - 1, 0.f)); + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz, + kNumRenderChannels, kNumCaptureChannels)); + std::vector>> block( + NumBandsForRate(sample_rate_hz), + std::vector>(kNumRenderChannels, + std::vector(kBlockSize - 1, 0.f))); EXPECT_DEATH(block_processor->BufferRender(block), ""); } void RunCaptureBlockSizeVerificationTest(int sample_rate_hz) { + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; + std::unique_ptr block_processor( - BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); - std::vector> block( - NumBandsForRate(sample_rate_hz), std::vector(kBlockSize - 1, 0.f)); + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz, + kNumRenderChannels, kNumCaptureChannels)); + std::vector>> block( + NumBandsForRate(sample_rate_hz), + std::vector>(kNumRenderChannels, + std::vector(kBlockSize - 1, 0.f))); EXPECT_DEATH(block_processor->ProcessCapture(false, false, &block), ""); } void RunRenderNumBandsVerificationTest(int sample_rate_hz) { + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; + const size_t wrong_num_bands = NumBandsForRate(sample_rate_hz) < 3 ? NumBandsForRate(sample_rate_hz) + 1 : 1; std::unique_ptr block_processor( - BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); - std::vector> block(wrong_num_bands, - std::vector(kBlockSize, 0.f)); + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz, + kNumRenderChannels, kNumCaptureChannels)); + std::vector>> block( + wrong_num_bands, + std::vector>(kNumRenderChannels, + std::vector(kBlockSize, 0.f))); EXPECT_DEATH(block_processor->BufferRender(block), ""); } void RunCaptureNumBandsVerificationTest(int sample_rate_hz) { + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; + const size_t wrong_num_bands = NumBandsForRate(sample_rate_hz) < 3 ? NumBandsForRate(sample_rate_hz) + 1 : 1; std::unique_ptr block_processor( - BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz)); - std::vector> block(wrong_num_bands, - std::vector(kBlockSize, 0.f)); + BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz, + kNumRenderChannels, kNumCaptureChannels)); + std::vector>> block( + wrong_num_bands, + std::vector>(kNumRenderChannels, + std::vector(kBlockSize, 0.f))); EXPECT_DEATH(block_processor->ProcessCapture(false, false, &block), ""); } @@ -104,17 +133,19 @@ std::string ProduceDebugText(int sample_rate_hz) { // the render delay buffer inside block processor. // TODO(peah): Activate the unittest once the required code has been landed. TEST(BlockProcessor, DISABLED_DelayControllerIntegration) { + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; constexpr size_t kNumBlocks = 310; constexpr size_t kDelayInSamples = 640; constexpr size_t kDelayHeadroom = 1; constexpr size_t kDelayInBlocks = kDelayInSamples / kBlockSize - kDelayHeadroom; Random random_generator(42U); - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr> render_delay_buffer_mock( - new StrictMock(rate)); + new StrictMock(rate, 1)); EXPECT_CALL(*render_delay_buffer_mock, Insert(_)) .Times(kNumBlocks) .WillRepeatedly(Return(RenderDelayBuffer::BufferingEvent::kNone)); @@ -125,16 +156,21 @@ TEST(BlockProcessor, DISABLED_DelayControllerIntegration) { .Times(kNumBlocks + 1) .WillRepeatedly(Return(0)); std::unique_ptr block_processor(BlockProcessor::Create( - EchoCanceller3Config(), rate, std::move(render_delay_buffer_mock))); + EchoCanceller3Config(), rate, kNumRenderChannels, kNumCaptureChannels, + std::move(render_delay_buffer_mock))); - std::vector> render_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); - std::vector> capture_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>(kNumRenderChannels, + std::vector(kBlockSize, 0.f))); + std::vector>> capture_block( + NumBandsForRate(rate), + std::vector>(kNumCaptureChannels, + std::vector(kBlockSize, 0.f))); DelayBuffer signal_delay_buffer(kDelayInSamples); for (size_t k = 0; k < kNumBlocks; ++k) { - RandomizeSampleVector(&random_generator, render_block[0]); - signal_delay_buffer.Delay(render_block[0], capture_block[0]); + RandomizeSampleVector(&random_generator, render_block[0][0]); + signal_delay_buffer.Delay(render_block[0][0], capture_block[0][0]); block_processor->BufferRender(render_block); block_processor->ProcessCapture(false, false, &capture_block); } @@ -144,12 +180,15 @@ TEST(BlockProcessor, DISABLED_DelayControllerIntegration) { // Verifies that BlockProcessor submodules are called in a proper manner. TEST(BlockProcessor, DISABLED_SubmoduleIntegration) { constexpr size_t kNumBlocks = 310; + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; + Random random_generator(42U); - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr> render_delay_buffer_mock( - new StrictMock(rate)); + new StrictMock(rate, 1)); std::unique_ptr< ::testing::StrictMock> render_delay_controller_mock( @@ -174,17 +213,22 @@ TEST(BlockProcessor, DISABLED_SubmoduleIntegration) { .Times(kNumBlocks); std::unique_ptr block_processor(BlockProcessor::Create( - EchoCanceller3Config(), rate, std::move(render_delay_buffer_mock), + EchoCanceller3Config(), rate, kNumRenderChannels, kNumCaptureChannels, + std::move(render_delay_buffer_mock), std::move(render_delay_controller_mock), std::move(echo_remover_mock))); - std::vector> render_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); - std::vector> capture_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>(kNumRenderChannels, + std::vector(kBlockSize, 0.f))); + std::vector>> capture_block( + NumBandsForRate(rate), + std::vector>(kNumCaptureChannels, + std::vector(kBlockSize, 0.f))); DelayBuffer signal_delay_buffer(640); for (size_t k = 0; k < kNumBlocks; ++k) { - RandomizeSampleVector(&random_generator, render_block[0]); - signal_delay_buffer.Delay(render_block[0], capture_block[0]); + RandomizeSampleVector(&random_generator, render_block[0][0]); + signal_delay_buffer.Delay(render_block[0][0], capture_block[0][0]); block_processor->BufferRender(render_block); block_processor->ProcessCapture(false, false, &capture_block); block_processor->UpdateEchoLeakageStatus(false); @@ -193,7 +237,7 @@ TEST(BlockProcessor, DISABLED_SubmoduleIntegration) { } TEST(BlockProcessor, BasicSetupAndApiCalls) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunBasicSetupAndApiCallTest(rate, 1); } @@ -207,21 +251,21 @@ TEST(BlockProcessor, TestLongerCall) { // TODO(gustaf): Re-enable the test once the issue with memory leaks during // DEATH tests on test bots has been fixed. TEST(BlockProcessor, DISABLED_VerifyRenderBlockSizeCheck) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunRenderBlockSizeVerificationTest(rate); } } TEST(BlockProcessor, VerifyCaptureBlockSizeCheck) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunCaptureBlockSizeVerificationTest(rate); } } TEST(BlockProcessor, VerifyRenderNumBandsCheck) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunRenderNumBandsVerificationTest(rate); } @@ -230,7 +274,7 @@ TEST(BlockProcessor, VerifyRenderNumBandsCheck) { // TODO(peah): Verify the check for correct number of bands in the capture // signal. TEST(BlockProcessor, VerifyCaptureNumBandsCheck) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunCaptureNumBandsVerificationTest(rate); } @@ -239,7 +283,7 @@ TEST(BlockProcessor, VerifyCaptureNumBandsCheck) { // Verifiers that the verification for null ProcessCapture input works. TEST(BlockProcessor, NullProcessCaptureParameter) { EXPECT_DEATH(std::unique_ptr( - BlockProcessor::Create(EchoCanceller3Config(), 8000)) + BlockProcessor::Create(EchoCanceller3Config(), 16000, 1, 1)) ->ProcessCapture(false, false, nullptr), ""); } @@ -249,7 +293,7 @@ TEST(BlockProcessor, NullProcessCaptureParameter) { // tests on test bots has been fixed. TEST(BlockProcessor, DISABLED_WrongSampleRate) { EXPECT_DEATH(std::unique_ptr( - BlockProcessor::Create(EchoCanceller3Config(), 8001)), + BlockProcessor::Create(EchoCanceller3Config(), 8001, 1, 1)), ""); } diff --git a/modules/audio_processing/aec3/decimator_unittest.cc b/modules/audio_processing/aec3/decimator_unittest.cc index cf8de84a2b..946089ab0b 100644 --- a/modules/audio_processing/aec3/decimator_unittest.cc +++ b/modules/audio_processing/aec3/decimator_unittest.cc @@ -90,7 +90,7 @@ void ProduceDecimatedSinusoidalOutputPower(int sample_rate_hz, TEST(Decimator, NoLeakageFromUpperFrequencies) { float input_power; float output_power; - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { for (auto down_sampling_factor : kDownSamplingFactors) { ProduceDebugText(rate); ProduceDecimatedSinusoidalOutputPower(rate, down_sampling_factor, diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc index e857a7e45a..4154e539b5 100644 --- a/modules/audio_processing/aec3/echo_audibility.cc +++ b/modules/audio_processing/aec3/echo_audibility.cc @@ -97,7 +97,7 @@ bool EchoAudibility::IsRenderTooLow(const MatrixBuffer& block_buffer) { } else { for (int idx = render_block_write_prev_; idx != render_block_write_current; idx = block_buffer.IncIndex(idx)) { - auto block = block_buffer.buffer[idx][0]; + auto block = block_buffer.buffer[idx][0][0]; auto r = std::minmax_element(block.cbegin(), block.cend()); float max_abs = std::max(std::fabs(*r.first), std::fabs(*r.second)); if (max_abs < 10) { diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index c2ad56b83d..cf953ae205 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -45,27 +45,36 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { return adjusted_cfg; } -void FillSubFrameView(AudioBuffer* frame, - size_t sub_frame_index, - std::vector>* sub_frame_view) { +void FillSubFrameView( + AudioBuffer* frame, + size_t sub_frame_index, + std::vector>>* sub_frame_view) { RTC_DCHECK_GE(1, sub_frame_index); RTC_DCHECK_LE(0, sub_frame_index); RTC_DCHECK_EQ(frame->num_bands(), sub_frame_view->size()); - for (size_t k = 0; k < sub_frame_view->size(); ++k) { - (*sub_frame_view)[k] = rtc::ArrayView( - &frame->split_bands(0)[k][sub_frame_index * kSubFrameLength], - kSubFrameLength); + RTC_DCHECK_EQ(frame->num_channels(), (*sub_frame_view)[0].size()); + for (size_t band = 0; band < sub_frame_view->size(); ++band) { + for (size_t channel = 0; channel < (*sub_frame_view)[0].size(); ++channel) { + (*sub_frame_view)[band][channel] = rtc::ArrayView( + &frame->split_bands(channel)[band][sub_frame_index * kSubFrameLength], + kSubFrameLength); + } } } -void FillSubFrameView(std::vector>* frame, - size_t sub_frame_index, - std::vector>* sub_frame_view) { +void FillSubFrameView( + std::vector>>* frame, + size_t sub_frame_index, + std::vector>>* sub_frame_view) { RTC_DCHECK_GE(1, sub_frame_index); RTC_DCHECK_EQ(frame->size(), sub_frame_view->size()); - for (size_t k = 0; k < frame->size(); ++k) { - (*sub_frame_view)[k] = rtc::ArrayView( - &(*frame)[k][sub_frame_index * kSubFrameLength], kSubFrameLength); + RTC_DCHECK_EQ((*frame)[0].size(), (*sub_frame_view)[0].size()); + for (size_t band = 0; band < frame->size(); ++band) { + for (size_t channel = 0; channel < (*frame)[band].size(); ++channel) { + (*sub_frame_view)[band][channel] = rtc::ArrayView( + &(*frame)[band][channel][sub_frame_index * kSubFrameLength], + kSubFrameLength); + } } } @@ -77,8 +86,8 @@ void ProcessCaptureFrameContent( FrameBlocker* capture_blocker, BlockFramer* output_framer, BlockProcessor* block_processor, - std::vector>* block, - std::vector>* sub_frame_view) { + std::vector>>* block, + std::vector>>* sub_frame_view) { FillSubFrameView(capture, sub_frame_index, sub_frame_view); capture_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block); block_processor->ProcessCapture(level_change, saturated_microphone_signal, @@ -92,7 +101,7 @@ void ProcessRemainingCaptureFrameContent( FrameBlocker* capture_blocker, BlockFramer* output_framer, BlockProcessor* block_processor, - std::vector>* block) { + std::vector>>* block) { if (!capture_blocker->IsBlockAvailable()) { return; } @@ -104,20 +113,21 @@ void ProcessRemainingCaptureFrameContent( } void BufferRenderFrameContent( - std::vector>* render_frame, + std::vector>>* render_frame, size_t sub_frame_index, FrameBlocker* render_blocker, BlockProcessor* block_processor, - std::vector>* block, - std::vector>* sub_frame_view) { + std::vector>>* block, + std::vector>>* sub_frame_view) { FillSubFrameView(render_frame, sub_frame_index, sub_frame_view); render_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block); block_processor->BufferRender(*block); } -void BufferRemainingRenderFrameContent(FrameBlocker* render_blocker, - BlockProcessor* block_processor, - std::vector>* block) { +void BufferRemainingRenderFrameContent( + FrameBlocker* render_blocker, + BlockProcessor* block_processor, + std::vector>>* block) { if (!render_blocker->IsBlockAvailable()) { return; } @@ -127,14 +137,19 @@ void BufferRemainingRenderFrameContent(FrameBlocker* render_blocker, void CopyBufferIntoFrame(const AudioBuffer& buffer, size_t num_bands, - size_t frame_length, - std::vector>* frame) { + size_t num_channels, + std::vector>>* frame) { RTC_DCHECK_EQ(num_bands, frame->size()); - RTC_DCHECK_EQ(frame_length, (*frame)[0].size()); - for (size_t k = 0; k < num_bands; ++k) { - rtc::ArrayView buffer_view(&buffer.split_bands_const(0)[k][0], - frame_length); - std::copy(buffer_view.begin(), buffer_view.end(), (*frame)[k].begin()); + RTC_DCHECK_EQ(num_channels, (*frame)[0].size()); + RTC_DCHECK_EQ(AudioBuffer::kSplitBandSize, (*frame)[0][0].size()); + for (size_t band = 0; band < num_bands; ++band) { + for (size_t channel = 0; channel < num_channels; ++channel) { + rtc::ArrayView buffer_view( + &buffer.split_bands_const(channel)[band][0], + AudioBuffer::kSplitBandSize); + std::copy(buffer_view.begin(), buffer_view.end(), + (*frame)[band][channel].begin()); + } } } @@ -143,40 +158,39 @@ void CopyBufferIntoFrame(const AudioBuffer& buffer, class EchoCanceller3::RenderWriter { public: RenderWriter(ApmDataDumper* data_dumper, - SwapQueue>, + SwapQueue>>, Aec3RenderQueueItemVerifier>* render_transfer_queue, - int sample_rate_hz, - int frame_length, - int num_bands); + size_t num_bands, + size_t num_channels); ~RenderWriter(); void Insert(const AudioBuffer& input); private: ApmDataDumper* data_dumper_; - const int sample_rate_hz_; - const size_t frame_length_; - const int num_bands_; + const size_t num_bands_; + const size_t num_channels_; HighPassFilter high_pass_filter_; - std::vector> render_queue_input_frame_; - SwapQueue>, Aec3RenderQueueItemVerifier>* - render_transfer_queue_; + std::vector>> render_queue_input_frame_; + SwapQueue>>, + Aec3RenderQueueItemVerifier>* render_transfer_queue_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderWriter); }; EchoCanceller3::RenderWriter::RenderWriter( ApmDataDumper* data_dumper, - SwapQueue>, Aec3RenderQueueItemVerifier>* - render_transfer_queue, - int sample_rate_hz, - int frame_length, - int num_bands) + SwapQueue>>, + Aec3RenderQueueItemVerifier>* render_transfer_queue, + size_t num_bands, + size_t num_channels) : data_dumper_(data_dumper), - sample_rate_hz_(sample_rate_hz), - frame_length_(frame_length), num_bands_(num_bands), - high_pass_filter_(1), - render_queue_input_frame_(num_bands_, - std::vector(frame_length_, 0.f)), + num_channels_(num_channels), + high_pass_filter_(num_channels), + render_queue_input_frame_( + num_bands_, + std::vector>( + num_channels_, + std::vector(AudioBuffer::kSplitBandSize, 0.f))), render_transfer_queue_(render_transfer_queue) { RTC_DCHECK(data_dumper); } @@ -185,21 +199,21 @@ EchoCanceller3::RenderWriter::~RenderWriter() = default; void EchoCanceller3::RenderWriter::Insert(const AudioBuffer& input) { RTC_DCHECK_EQ(1, input.num_channels()); - RTC_DCHECK_EQ(frame_length_, input.num_frames_per_band()); + RTC_DCHECK_EQ(AudioBuffer::kSplitBandSize, input.num_frames_per_band()); RTC_DCHECK_EQ(num_bands_, input.num_bands()); // TODO(bugs.webrtc.org/8759) Temporary work-around. - if (num_bands_ != static_cast(input.num_bands())) + if (num_bands_ != input.num_bands()) return; - data_dumper_->DumpWav("aec3_render_input", frame_length_, - &input.split_bands_const(0)[0][0], - LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpWav("aec3_render_input", AudioBuffer::kSplitBandSize, + &input.split_bands_const(0)[0][0], 16000, 1); - CopyBufferIntoFrame(input, num_bands_, frame_length_, + CopyBufferIntoFrame(input, num_bands_, num_channels_, &render_queue_input_frame_); - - high_pass_filter_.Process(render_queue_input_frame_[0]); + for (size_t channel = 0; channel < num_channels_; ++channel) { + high_pass_filter_.Process(render_queue_input_frame_[0][channel]); + } static_cast(render_transfer_queue_->Insert(&render_queue_input_frame_)); } @@ -207,43 +221,71 @@ void EchoCanceller3::RenderWriter::Insert(const AudioBuffer& input) { int EchoCanceller3::instance_count_ = 0; EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, - int sample_rate_hz) - : EchoCanceller3( - AdjustConfig(config), - sample_rate_hz, - std::unique_ptr( - BlockProcessor::Create(AdjustConfig(config), sample_rate_hz))) {} + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) + : EchoCanceller3(AdjustConfig(config), + sample_rate_hz, + num_render_channels, + num_capture_channels, + std::unique_ptr( + BlockProcessor::Create(AdjustConfig(config), + sample_rate_hz, + num_render_channels, + num_capture_channels))) {} EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels, std::unique_ptr block_processor) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), config_(config), sample_rate_hz_(sample_rate_hz), num_bands_(NumBandsForRate(sample_rate_hz_)), - frame_length_(rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)), - output_framer_(num_bands_), - capture_blocker_(num_bands_), - render_blocker_(num_bands_), + num_render_channels_(num_render_channels), + num_capture_channels_(num_capture_channels), + output_framer_(num_bands_, num_capture_channels_), + capture_blocker_(num_bands_, num_capture_channels_), + render_blocker_(num_bands_, num_render_channels_), render_transfer_queue_( kRenderTransferQueueSizeFrames, - std::vector>( + std::vector>>( num_bands_, - std::vector(frame_length_, 0.f)), - Aec3RenderQueueItemVerifier(num_bands_, frame_length_)), + std::vector>( + num_render_channels_, + std::vector(AudioBuffer::kSplitBandSize, 0.f))), + Aec3RenderQueueItemVerifier(num_bands_, + num_render_channels_, + AudioBuffer::kSplitBandSize)), block_processor_(std::move(block_processor)), - render_queue_output_frame_(num_bands_, - std::vector(frame_length_, 0.f)), - block_(num_bands_, std::vector(kBlockSize, 0.f)), - sub_frame_view_(num_bands_), + render_queue_output_frame_( + num_bands_, + std::vector>( + num_render_channels_, + std::vector(AudioBuffer::kSplitBandSize, 0.f))), + render_block_( + num_bands_, + std::vector>(num_render_channels_, + std::vector(kBlockSize, 0.f))), + capture_block_( + num_bands_, + std::vector>(num_capture_channels_, + std::vector(kBlockSize, 0.f))), + render_sub_frame_view_( + num_bands_, + std::vector>(num_render_channels_)), + capture_sub_frame_view_( + num_bands_, + std::vector>(num_capture_channels_)), block_delay_buffer_(num_bands_, - frame_length_, + AudioBuffer::kSplitBandSize, config_.delay.fixed_capture_delay_samples) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); - render_writer_.reset( - new RenderWriter(data_dumper_.get(), &render_transfer_queue_, - sample_rate_hz_, frame_length_, num_bands_)); + render_writer_.reset(new RenderWriter(data_dumper_.get(), + &render_transfer_queue_, num_bands_, + num_render_channels_)); RTC_DCHECK_EQ(num_bands_, std::max(sample_rate_hz_, 16000) / 16000); RTC_DCHECK_GE(kMaxNumBands, num_bands_); @@ -253,6 +295,7 @@ EchoCanceller3::~EchoCanceller3() = default; void EchoCanceller3::AnalyzeRender(const AudioBuffer& render) { RTC_DCHECK_RUNS_SERIALIZED(&render_race_checker_); + RTC_DCHECK_EQ(render.num_channels(), num_render_channels_); data_dumper_->DumpRaw("aec3_call_order", static_cast(EchoCanceller3ApiCall::kRender)); @@ -265,10 +308,10 @@ void EchoCanceller3::AnalyzeCapture(const AudioBuffer& capture) { capture.channels_const()[0], sample_rate_hz_, 1); saturated_microphone_signal_ = false; - for (size_t k = 0; k < capture.num_channels(); ++k) { + for (size_t channel = 0; channel < capture.num_channels(); ++channel) { saturated_microphone_signal_ |= DetectSaturation(rtc::ArrayView( - capture.channels_const()[k], capture.num_frames())); + capture.channels_const()[channel], capture.num_frames())); if (saturated_microphone_signal_) { break; } @@ -280,7 +323,8 @@ void EchoCanceller3::ProcessCapture(AudioBuffer* capture, bool level_change) { RTC_DCHECK(capture); RTC_DCHECK_EQ(1u, capture->num_channels()); RTC_DCHECK_EQ(num_bands_, capture->num_bands()); - RTC_DCHECK_EQ(frame_length_, capture->num_frames_per_band()); + RTC_DCHECK_EQ(AudioBuffer::kSplitBandSize, capture->num_frames_per_band()); + RTC_DCHECK_EQ(capture->num_channels(), num_capture_channels_); data_dumper_->DumpRaw("aec3_call_order", static_cast(EchoCanceller3ApiCall::kCapture)); @@ -293,32 +337,29 @@ void EchoCanceller3::ProcessCapture(AudioBuffer* capture, bool level_change) { block_delay_buffer_.DelaySignal(capture); } - rtc::ArrayView capture_lower_band = - rtc::ArrayView(&capture->split_bands(0)[0][0], frame_length_); + rtc::ArrayView capture_lower_band = rtc::ArrayView( + &capture->split_bands(0)[0][0], AudioBuffer::kSplitBandSize); - data_dumper_->DumpWav("aec3_capture_input", capture_lower_band, - LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpWav("aec3_capture_input", capture_lower_band, 16000, 1); EmptyRenderQueue(); - ProcessCaptureFrameContent( - capture, level_change, saturated_microphone_signal_, 0, &capture_blocker_, - &output_framer_, block_processor_.get(), &block_, &sub_frame_view_); + ProcessCaptureFrameContent(capture, level_change, + saturated_microphone_signal_, 0, &capture_blocker_, + &output_framer_, block_processor_.get(), + &capture_block_, &capture_sub_frame_view_); - if (sample_rate_hz_ != 8000) { - ProcessCaptureFrameContent( - capture, level_change, saturated_microphone_signal_, 1, - &capture_blocker_, &output_framer_, block_processor_.get(), &block_, - &sub_frame_view_); - } + ProcessCaptureFrameContent(capture, level_change, + saturated_microphone_signal_, 1, &capture_blocker_, + &output_framer_, block_processor_.get(), + &capture_block_, &capture_sub_frame_view_); ProcessRemainingCaptureFrameContent( level_change, saturated_microphone_signal_, &capture_blocker_, - &output_framer_, block_processor_.get(), &block_); + &output_framer_, block_processor_.get(), &capture_block_); - data_dumper_->DumpWav("aec3_capture_output", frame_length_, - &capture->split_bands(0)[0][0], - LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpWav("aec3_capture_output", AudioBuffer::kSplitBandSize, + &capture->split_bands(0)[0][0], 16000, 1); } EchoControl::Metrics EchoCanceller3::GetMetrics() const { @@ -342,16 +383,15 @@ void EchoCanceller3::EmptyRenderQueue() { api_call_metrics_.ReportRenderCall(); BufferRenderFrameContent(&render_queue_output_frame_, 0, &render_blocker_, - block_processor_.get(), &block_, &sub_frame_view_); + block_processor_.get(), &render_block_, + &render_sub_frame_view_); - if (sample_rate_hz_ != 8000) { - BufferRenderFrameContent(&render_queue_output_frame_, 1, &render_blocker_, - block_processor_.get(), &block_, - &sub_frame_view_); - } + BufferRenderFrameContent(&render_queue_output_frame_, 1, &render_blocker_, + block_processor_.get(), &render_block_, + &render_sub_frame_view_); BufferRemainingRenderFrameContent(&render_blocker_, block_processor_.get(), - &block_); + &render_block_); frame_to_buffer = render_transfer_queue_.Remove(&render_queue_output_frame_); diff --git a/modules/audio_processing/aec3/echo_canceller3.h b/modules/audio_processing/aec3/echo_canceller3.h index d7dea80136..5b59674337 100644 --- a/modules/audio_processing/aec3/echo_canceller3.h +++ b/modules/audio_processing/aec3/echo_canceller3.h @@ -27,7 +27,6 @@ #include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" -#include "rtc_base/constructor_magic.h" #include "rtc_base/race_checker.h" #include "rtc_base/swap_queue.h" #include "rtc_base/thread_annotations.h" @@ -38,23 +37,33 @@ namespace webrtc { // queue. class Aec3RenderQueueItemVerifier { public: - explicit Aec3RenderQueueItemVerifier(size_t num_bands, size_t frame_length) - : num_bands_(num_bands), frame_length_(frame_length) {} + Aec3RenderQueueItemVerifier(size_t num_bands, + size_t num_channels, + size_t frame_length) + : num_bands_(num_bands), + num_channels_(num_channels), + frame_length_(frame_length) {} - bool operator()(const std::vector>& v) const { + bool operator()(const std::vector>>& v) const { if (v.size() != num_bands_) { return false; } - for (const auto& v_k : v) { - if (v_k.size() != frame_length_) { + for (const auto& band : v) { + if (band.size() != num_channels_) { return false; } + for (const auto& channel : band) { + if (channel.size() != frame_length_) { + return false; + } + } } return true; } private: const size_t num_bands_; + const size_t num_channels_; const size_t frame_length_; }; @@ -73,12 +82,20 @@ class Aec3RenderQueueItemVerifier { class EchoCanceller3 : public EchoControl { public: // Normal c-tor to use. - EchoCanceller3(const EchoCanceller3Config& config, int sample_rate_hz); + EchoCanceller3(const EchoCanceller3Config& config, + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels); // Testing c-tor that is used only for testing purposes. EchoCanceller3(const EchoCanceller3Config& config, int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels, std::unique_ptr block_processor); ~EchoCanceller3() override; + EchoCanceller3(const EchoCanceller3&) = delete; + EchoCanceller3& operator=(const EchoCanceller3&) = delete; + // Analyzes and stores an internal copy of the split-band domain render // signal. void AnalyzeRender(AudioBuffer* render) override { AnalyzeRender(*render); } @@ -128,25 +145,30 @@ class EchoCanceller3 : public EchoControl { const EchoCanceller3Config config_; const int sample_rate_hz_; const int num_bands_; - const size_t frame_length_; + const size_t num_render_channels_; + const size_t num_capture_channels_; BlockFramer output_framer_ RTC_GUARDED_BY(capture_race_checker_); FrameBlocker capture_blocker_ RTC_GUARDED_BY(capture_race_checker_); FrameBlocker render_blocker_ RTC_GUARDED_BY(capture_race_checker_); - SwapQueue>, Aec3RenderQueueItemVerifier> + SwapQueue>>, + Aec3RenderQueueItemVerifier> render_transfer_queue_; std::unique_ptr block_processor_ RTC_GUARDED_BY(capture_race_checker_); - std::vector> render_queue_output_frame_ + std::vector>> render_queue_output_frame_ RTC_GUARDED_BY(capture_race_checker_); bool saturated_microphone_signal_ RTC_GUARDED_BY(capture_race_checker_) = false; - std::vector> block_ RTC_GUARDED_BY(capture_race_checker_); - std::vector> sub_frame_view_ + std::vector>> render_block_ + RTC_GUARDED_BY(capture_race_checker_); + std::vector>> capture_block_ + RTC_GUARDED_BY(capture_race_checker_); + std::vector>> render_sub_frame_view_ + RTC_GUARDED_BY(capture_race_checker_); + std::vector>> capture_sub_frame_view_ RTC_GUARDED_BY(capture_race_checker_); BlockDelayBuffer block_delay_buffer_ RTC_GUARDED_BY(capture_race_checker_); ApiCallJitterMetrics api_call_metrics_ RTC_GUARDED_BY(capture_race_checker_); - - RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(EchoCanceller3); }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/modules/audio_processing/aec3/echo_canceller3_unittest.cc index a29b779348..a2f3367394 100644 --- a/modules/audio_processing/aec3/echo_canceller3_unittest.cc +++ b/modules/audio_processing/aec3/echo_canceller3_unittest.cc @@ -109,12 +109,13 @@ class CaptureTransportVerificationProcessor : public BlockProcessor { explicit CaptureTransportVerificationProcessor(size_t num_bands) {} ~CaptureTransportVerificationProcessor() override = default; - void ProcessCapture(bool level_change, - bool saturated_microphone_signal, - std::vector>* capture_block) override { - } + void ProcessCapture( + bool level_change, + bool saturated_microphone_signal, + std::vector>>* capture_block) override {} - void BufferRender(const std::vector>& block) override {} + void BufferRender( + const std::vector>>& block) override {} void UpdateEchoLeakageStatus(bool leakage_detected) override {} @@ -133,16 +134,18 @@ class RenderTransportVerificationProcessor : public BlockProcessor { explicit RenderTransportVerificationProcessor(size_t num_bands) {} ~RenderTransportVerificationProcessor() override = default; - void ProcessCapture(bool level_change, - bool saturated_microphone_signal, - std::vector>* capture_block) override { - std::vector> render_block = + void ProcessCapture( + bool level_change, + bool saturated_microphone_signal, + std::vector>>* capture_block) override { + std::vector>> render_block = received_render_blocks_.front(); received_render_blocks_.pop_front(); capture_block->swap(render_block); } - void BufferRender(const std::vector>& block) override { + void BufferRender( + const std::vector>>& block) override { received_render_blocks_.push_back(block); } @@ -153,7 +156,8 @@ class RenderTransportVerificationProcessor : public BlockProcessor { void SetAudioBufferDelay(size_t delay_ms) override {} private: - std::deque>> received_render_blocks_; + std::deque>>> + received_render_blocks_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderTransportVerificationProcessor); }; @@ -162,7 +166,7 @@ class EchoCanceller3Tester { explicit EchoCanceller3Tester(int sample_rate_hz) : sample_rate_hz_(sample_rate_hz), num_bands_(NumBandsForRate(sample_rate_hz_)), - frame_length_(sample_rate_hz_ == 8000 ? 80 : 160), + frame_length_(160), fullband_frame_length_(rtc::CheckedDivExact(sample_rate_hz_, 100)), capture_buffer_(fullband_frame_length_ * 100, 1, @@ -182,7 +186,7 @@ class EchoCanceller3Tester { // output. void RunCaptureTransportVerificationTest() { EchoCanceller3 aec3( - EchoCanceller3Config(), sample_rate_hz_, + EchoCanceller3Config(), sample_rate_hz_, 1, 1, std::unique_ptr( new CaptureTransportVerificationProcessor(num_bands_))); @@ -207,7 +211,7 @@ class EchoCanceller3Tester { // block processor. void RunRenderTransportVerificationTest() { EchoCanceller3 aec3( - EchoCanceller3Config(), sample_rate_hz_, + EchoCanceller3Config(), sample_rate_hz_, 1, 1, std::unique_ptr( new RenderTransportVerificationProcessor(num_bands_))); @@ -251,37 +255,34 @@ class EchoCanceller3Tester { void RunEchoPathChangeVerificationTest( EchoPathChangeTestVariant echo_path_change_test_variant) { - const size_t num_full_blocks_per_frame = - rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100) / kBlockSize; - const size_t expected_num_block_to_process = - (kNumFramesToProcess * - rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)) / - kBlockSize; + constexpr size_t kNumFullBlocksPerFrame = 160 / kBlockSize; + constexpr size_t kExpectedNumBlocksToProcess = + (kNumFramesToProcess * 160) / kBlockSize; std::unique_ptr> block_processor_mock( new StrictMock()); EXPECT_CALL(*block_processor_mock, BufferRender(_)) - .Times(expected_num_block_to_process); + .Times(kExpectedNumBlocksToProcess); EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(_)).Times(0); switch (echo_path_change_test_variant) { case EchoPathChangeTestVariant::kNone: EXPECT_CALL(*block_processor_mock, ProcessCapture(false, _, _)) - .Times(expected_num_block_to_process); + .Times(kExpectedNumBlocksToProcess); break; case EchoPathChangeTestVariant::kOneSticky: EXPECT_CALL(*block_processor_mock, ProcessCapture(true, _, _)) - .Times(expected_num_block_to_process); + .Times(kExpectedNumBlocksToProcess); break; case EchoPathChangeTestVariant::kOneNonSticky: EXPECT_CALL(*block_processor_mock, ProcessCapture(true, _, _)) - .Times(num_full_blocks_per_frame); + .Times(kNumFullBlocksPerFrame); EXPECT_CALL(*block_processor_mock, ProcessCapture(false, _, _)) - .Times(expected_num_block_to_process - num_full_blocks_per_frame); + .Times(kExpectedNumBlocksToProcess - kNumFullBlocksPerFrame); break; } - EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, + EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, 1, 1, std::move(block_processor_mock)); for (size_t frame_index = 0; frame_index < kNumFramesToProcess; @@ -330,17 +331,15 @@ class EchoCanceller3Tester { void RunEchoLeakageVerificationTest( EchoLeakageTestVariant leakage_report_variant) { - const size_t expected_num_block_to_process = - (kNumFramesToProcess * - rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)) / - kBlockSize; + constexpr size_t kExpectedNumBlocksToProcess = + (kNumFramesToProcess * 160) / kBlockSize; std::unique_ptr> block_processor_mock( new StrictMock()); EXPECT_CALL(*block_processor_mock, BufferRender(_)) - .Times(expected_num_block_to_process); + .Times(kExpectedNumBlocksToProcess); EXPECT_CALL(*block_processor_mock, ProcessCapture(_, _, _)) - .Times(expected_num_block_to_process); + .Times(kExpectedNumBlocksToProcess); switch (leakage_report_variant) { case EchoLeakageTestVariant::kNone: @@ -363,7 +362,7 @@ class EchoCanceller3Tester { } break; } - EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, + EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, 1, 1, std::move(block_processor_mock)); for (size_t frame_index = 0; frame_index < kNumFramesToProcess; @@ -418,41 +417,38 @@ class EchoCanceller3Tester { void RunCaptureSaturationVerificationTest( SaturationTestVariant saturation_variant) { - const size_t num_full_blocks_per_frame = - rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100) / kBlockSize; - const size_t expected_num_block_to_process = - (kNumFramesToProcess * - rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)) / - kBlockSize; + const size_t kNumFullBlocksPerFrame = 160 / kBlockSize; + const size_t kExpectedNumBlocksToProcess = + (kNumFramesToProcess * 160) / kBlockSize; std::unique_ptr> block_processor_mock( new StrictMock()); EXPECT_CALL(*block_processor_mock, BufferRender(_)) - .Times(expected_num_block_to_process); + .Times(kExpectedNumBlocksToProcess); EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(_)).Times(0); switch (saturation_variant) { case SaturationTestVariant::kNone: EXPECT_CALL(*block_processor_mock, ProcessCapture(_, false, _)) - .Times(expected_num_block_to_process); + .Times(kExpectedNumBlocksToProcess); break; case SaturationTestVariant::kOneNegative: { ::testing::InSequence s; EXPECT_CALL(*block_processor_mock, ProcessCapture(_, true, _)) - .Times(num_full_blocks_per_frame); + .Times(kNumFullBlocksPerFrame); EXPECT_CALL(*block_processor_mock, ProcessCapture(_, false, _)) - .Times(expected_num_block_to_process - num_full_blocks_per_frame); + .Times(kExpectedNumBlocksToProcess - kNumFullBlocksPerFrame); } break; case SaturationTestVariant::kOnePositive: { ::testing::InSequence s; EXPECT_CALL(*block_processor_mock, ProcessCapture(_, true, _)) - .Times(num_full_blocks_per_frame); + .Times(kNumFullBlocksPerFrame); EXPECT_CALL(*block_processor_mock, ProcessCapture(_, false, _)) - .Times(expected_num_block_to_process - num_full_blocks_per_frame); + .Times(kExpectedNumBlocksToProcess - kNumFullBlocksPerFrame); } break; } - EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, + EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, 1, 1, std::move(block_processor_mock)); for (size_t frame_index = 0; frame_index < kNumFramesToProcess; ++frame_index) { @@ -492,7 +488,7 @@ class EchoCanceller3Tester { void RunRenderSwapQueueVerificationTest() { const EchoCanceller3Config config; EchoCanceller3 aec3( - config, sample_rate_hz_, + config, sample_rate_hz_, 1, 1, std::unique_ptr( new RenderTransportVerificationProcessor(num_bands_))); @@ -542,7 +538,7 @@ class EchoCanceller3Tester { // This test verifies that a buffer overrun in the render swapqueue is // properly reported. void RunRenderPipelineSwapQueueOverrunReturnValueTest() { - EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_); + EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, 1, 1); constexpr size_t kRenderTransferQueueSize = 30; for (size_t k = 0; k < 2; ++k) { @@ -567,7 +563,7 @@ class EchoCanceller3Tester { // Set aec3_sample_rate_hz to be different from sample_rate_hz_ in such a // way that the number of bands for the rates are different. const int aec3_sample_rate_hz = sample_rate_hz_ == 48000 ? 32000 : 48000; - EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz); + EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz, 1, 1); PopulateInputFrame(frame_length_, 0, &render_buffer_.channels_f()[0][0], 0); EXPECT_DEATH(aec3.AnalyzeRender(&render_buffer_), ""); @@ -580,43 +576,12 @@ class EchoCanceller3Tester { // Set aec3_sample_rate_hz to be different from sample_rate_hz_ in such a // way that the number of bands for the rates are different. const int aec3_sample_rate_hz = sample_rate_hz_ == 48000 ? 32000 : 48000; - EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz); + EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz, 1, 1); PopulateInputFrame(frame_length_, num_bands_, 0, &capture_buffer_.split_bands_f(0)[0], 100); EXPECT_DEATH(aec3.ProcessCapture(&capture_buffer_, false), ""); } - // Verifies the that the check for the frame length in the AnalyzeRender input - // is correct by adjusting the sample rates of EchoCanceller3 and the input - // AudioBuffer to have a different frame lengths. - void RunAnalyzeRenderFrameLengthCheckVerification() { - // Set aec3_sample_rate_hz to be different from sample_rate_hz_ in such a - // way that the band frame lengths are different. - const int aec3_sample_rate_hz = sample_rate_hz_ == 8000 ? 16000 : 8000; - EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz); - - OptionalBandSplit(); - PopulateInputFrame(frame_length_, 0, &render_buffer_.channels_f()[0][0], 0); - - EXPECT_DEATH(aec3.AnalyzeRender(&render_buffer_), ""); - } - - // Verifies the that the check for the frame length in the AnalyzeRender input - // is correct by adjusting the sample rates of EchoCanceller3 and the input - // AudioBuffer to have a different frame lengths. - void RunProcessCaptureFrameLengthCheckVerification() { - // Set aec3_sample_rate_hz to be different from sample_rate_hz_ in such a - // way that the band frame lengths are different. - const int aec3_sample_rate_hz = sample_rate_hz_ == 8000 ? 16000 : 8000; - EchoCanceller3 aec3(EchoCanceller3Config(), aec3_sample_rate_hz); - - OptionalBandSplit(); - PopulateInputFrame(frame_length_, num_bands_, 0, - &capture_buffer_.split_bands_f(0)[0], 100); - - EXPECT_DEATH(aec3.ProcessCapture(&capture_buffer_, false), ""); - } - #endif private: @@ -653,28 +618,25 @@ std::string ProduceDebugText(int sample_rate_hz, int variant) { } // namespace TEST(EchoCanceller3Buffering, CaptureBitexactness) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); EchoCanceller3Tester(rate).RunCaptureTransportVerificationTest(); } } TEST(EchoCanceller3Buffering, RenderBitexactness) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); EchoCanceller3Tester(rate).RunRenderTransportVerificationTest(); } } TEST(EchoCanceller3Buffering, RenderSwapQueue) { - for (auto rate : {8000, 16000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - EchoCanceller3Tester(rate).RunRenderSwapQueueVerificationTest(); - } + EchoCanceller3Tester(16000).RunRenderSwapQueueVerificationTest(); } TEST(EchoCanceller3Buffering, RenderSwapQueueOverrunReturnValue) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); EchoCanceller3Tester(rate) .RunRenderPipelineSwapQueueOverrunReturnValueTest(); @@ -685,7 +647,7 @@ TEST(EchoCanceller3Messaging, CaptureSaturation) { auto variants = {EchoCanceller3Tester::SaturationTestVariant::kNone, EchoCanceller3Tester::SaturationTestVariant::kOneNegative, EchoCanceller3Tester::SaturationTestVariant::kOnePositive}; - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { for (auto variant : variants) { SCOPED_TRACE(ProduceDebugText(rate, static_cast(variant))); EchoCanceller3Tester(rate).RunCaptureSaturationVerificationTest(variant); @@ -698,7 +660,7 @@ TEST(EchoCanceller3Messaging, EchoPathChange) { EchoCanceller3Tester::EchoPathChangeTestVariant::kNone, EchoCanceller3Tester::EchoPathChangeTestVariant::kOneSticky, EchoCanceller3Tester::EchoPathChangeTestVariant::kOneNonSticky}; - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { for (auto variant : variants) { SCOPED_TRACE(ProduceDebugText(rate, static_cast(variant))); EchoCanceller3Tester(rate).RunEchoPathChangeVerificationTest(variant); @@ -712,7 +674,7 @@ TEST(EchoCanceller3Messaging, EchoLeakage) { EchoCanceller3Tester::EchoLeakageTestVariant::kFalseSticky, EchoCanceller3Tester::EchoLeakageTestVariant::kTrueSticky, EchoCanceller3Tester::EchoLeakageTestVariant::kTrueNonSticky}; - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { for (auto variant : variants) { SCOPED_TRACE(ProduceDebugText(rate, static_cast(variant))); EchoCanceller3Tester(rate).RunEchoLeakageVerificationTest(variant); @@ -723,33 +685,16 @@ TEST(EchoCanceller3Messaging, EchoLeakage) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) TEST(EchoCanceller3InputCheck, WrongCaptureNumBandsCheckVerification) { - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); EchoCanceller3Tester(rate).RunProcessCaptureNumBandsCheckVerification(); } } -// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH -// tests on test bots has been fixed. -TEST(EchoCanceller3InputCheck, - DISABLED_WrongRenderFrameLengthCheckVerification) { - for (auto rate : {8000, 16000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - EchoCanceller3Tester(rate).RunAnalyzeRenderFrameLengthCheckVerification(); - } -} - -TEST(EchoCanceller3InputCheck, WrongCaptureFrameLengthCheckVerification) { - for (auto rate : {8000, 16000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - EchoCanceller3Tester(rate).RunProcessCaptureFrameLengthCheckVerification(); - } -} - // Verifiers that the verification for null input to the capture processing api // call works. TEST(EchoCanceller3InputCheck, NullCaptureProcessingParameter) { - EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 16000) + EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 16000, 1, 1) .ProcessCapture(nullptr, false), ""); } @@ -759,7 +704,7 @@ TEST(EchoCanceller3InputCheck, NullCaptureProcessingParameter) { // tests on test bots has been fixed. TEST(EchoCanceller3InputCheck, DISABLED_WrongSampleRate) { ApmDataDumper data_dumper(0); - EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 8001), ""); + EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 8001, 1, 1), ""); } #endif diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc b/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc index ddf6bc58de..9a1bf4442f 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc +++ b/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc @@ -36,12 +36,17 @@ std::string ProduceDebugText(size_t delay, size_t down_sampling_factor) { // Verifies that the basic API calls work. TEST(EchoPathDelayEstimator, BasicApiCalls) { + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); ApmDataDumper data_dumper(0); EchoCanceller3Config config; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); EchoPathDelayEstimator estimator(&data_dumper, config); - std::vector> render(3, std::vector(kBlockSize)); + std::vector>> render( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize))); std::vector capture(kBlockSize); for (size_t k = 0; k < 100; ++k) { render_delay_buffer->Insert(render); @@ -53,8 +58,14 @@ TEST(EchoPathDelayEstimator, BasicApiCalls) { // Verifies that the delay estimator produces correct delay for artificially // delayed signals. TEST(EchoPathDelayEstimator, DelayEstimation) { + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + Random random_generator(42U); - std::vector> render(3, std::vector(kBlockSize)); + std::vector>> render( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize))); std::vector capture(kBlockSize); ApmDataDumper data_dumper(0); constexpr size_t kDownSamplingFactors[] = {2, 4, 8}; @@ -65,14 +76,14 @@ TEST(EchoPathDelayEstimator, DelayEstimation) { for (size_t delay_samples : {30, 64, 150, 200, 800, 4000}) { SCOPED_TRACE(ProduceDebugText(delay_samples, down_sampling_factor)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); DelayBuffer signal_delay_buffer(delay_samples); EchoPathDelayEstimator estimator(&data_dumper, config); absl::optional estimated_delay_samples; for (size_t k = 0; k < (500 + (delay_samples) / kBlockSize); ++k) { - RandomizeSampleVector(&random_generator, render[0]); - signal_delay_buffer.Delay(render[0], capture); + RandomizeSampleVector(&random_generator, render[0][0]); + signal_delay_buffer.Delay(render[0][0], capture); render_delay_buffer->Insert(render); if (k == 0) { @@ -106,20 +117,26 @@ TEST(EchoPathDelayEstimator, DelayEstimation) { // Verifies that the delay estimator does not produce delay estimates for render // signals of low level. TEST(EchoPathDelayEstimator, NoDelayEstimatesForLowLevelRenderSignals) { + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); Random random_generator(42U); EchoCanceller3Config config; - std::vector> render(3, std::vector(kBlockSize)); + std::vector>> render( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize))); std::vector capture(kBlockSize); ApmDataDumper data_dumper(0); EchoPathDelayEstimator estimator(&data_dumper, config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 48000)); + RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz, + kNumChannels)); for (size_t k = 0; k < 100; ++k) { - RandomizeSampleVector(&random_generator, render[0]); - for (auto& render_k : render[0]) { + RandomizeSampleVector(&random_generator, render[0][0]); + for (auto& render_k : render[0][0]) { render_k *= 100.f / 32767.f; } - std::copy(render[0].begin(), render[0].end(), capture.begin()); + std::copy(render[0][0].begin(), render[0][0].end(), capture.begin()); render_delay_buffer->Insert(render); render_delay_buffer->PrepareCaptureProcessing(); EXPECT_FALSE(estimator.EstimateDelay( @@ -137,7 +154,7 @@ TEST(EchoPathDelayEstimator, DISABLED_WrongRenderBlockSize) { EchoCanceller3Config config; EchoPathDelayEstimator estimator(&data_dumper, config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, 48000, 1)); std::vector capture(kBlockSize); EXPECT_DEATH(estimator.EstimateDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture), @@ -152,7 +169,7 @@ TEST(EchoPathDelayEstimator, WrongCaptureBlockSize) { EchoCanceller3Config config; EchoPathDelayEstimator estimator(&data_dumper, config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, 48000, 1)); std::vector capture(std::vector(kBlockSize - 1)); EXPECT_DEATH(estimator.EstimateDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture), diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index c7e7f7cd5e..60538d644e 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -84,7 +84,10 @@ void WindowedPaddedFft(const Aec3Fft& fft, // Class for removing the echo from the capture signal. class EchoRemoverImpl final : public EchoRemover { public: - EchoRemoverImpl(const EchoCanceller3Config& config, int sample_rate_hz); + EchoRemoverImpl(const EchoCanceller3Config& config, + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels); ~EchoRemoverImpl() override; void GetMetrics(EchoControl::Metrics* metrics) const override; @@ -92,11 +95,12 @@ class EchoRemoverImpl final : public EchoRemover { // Removes the echo from a block of samples from the capture signal. The // supplied render signal is assumed to be pre-aligned with the capture // signal. - void ProcessCapture(EchoPathVariability echo_path_variability, - bool capture_signal_saturation, - const absl::optional& external_delay, - RenderBuffer* render_buffer, - std::vector>* capture) override; + void ProcessCapture( + EchoPathVariability echo_path_variability, + bool capture_signal_saturation, + const absl::optional& external_delay, + RenderBuffer* render_buffer, + std::vector>>* capture) override; // Updates the status on whether echo leakage is detected in the output of the // echo remover. @@ -117,6 +121,8 @@ class EchoRemoverImpl final : public EchoRemover { std::unique_ptr data_dumper_; const Aec3Optimization optimization_; const int sample_rate_hz_; + const size_t num_render_channels_; + const size_t num_capture_channels_; const bool use_shadow_filter_output_; Subtractor subtractor_; SuppressionGain suppression_gain_; @@ -141,13 +147,17 @@ class EchoRemoverImpl final : public EchoRemover { int EchoRemoverImpl::instance_count_ = 0; EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, - int sample_rate_hz) + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) : config_(config), fft_(), data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), optimization_(DetectOptimization()), sample_rate_hz_(sample_rate_hz), + num_render_channels_(num_render_channels), + num_capture_channels_(num_capture_channels), use_shadow_filter_output_( config_.filter.enable_shadow_filter_output_usage), subtractor_(config, data_dumper_.get(), optimization_), @@ -161,6 +171,8 @@ EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, x_old_.fill(0.f); y_old_.fill(0.f); e_old_.fill(0.f); + (void)num_render_channels_; + (void)num_capture_channels_; } EchoRemoverImpl::~EchoRemoverImpl() = default; @@ -177,23 +189,26 @@ void EchoRemoverImpl::ProcessCapture( bool capture_signal_saturation, const absl::optional& external_delay, RenderBuffer* render_buffer, - std::vector>* capture) { + std::vector>>* capture) { ++block_counter_; - const std::vector>& x = render_buffer->Block(0); - std::vector>* y = capture; + const std::vector>>& x = + render_buffer->Block(0); + std::vector>>* y = capture; RTC_DCHECK(render_buffer); RTC_DCHECK(y); RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_)); RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_)); - RTC_DCHECK_EQ(x[0].size(), kBlockSize); - RTC_DCHECK_EQ((*y)[0].size(), kBlockSize); - const std::vector& x0 = x[0]; - std::vector& y0 = (*y)[0]; + RTC_DCHECK_EQ(x[0].size(), num_render_channels_); + RTC_DCHECK_EQ((*y)[0].size(), num_capture_channels_); + RTC_DCHECK_EQ(x[0][0].size(), kBlockSize); + RTC_DCHECK_EQ((*y)[0][0].size(), kBlockSize); + const std::vector& x0 = x[0][0]; + std::vector& y0 = (*y)[0][0]; data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, &y0[0], - LowestBandRate(sample_rate_hz_), 1); + 16000, 1); data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, &x0[0], - LowestBandRate(sample_rate_hz_), 1); + 16000, 1); data_dumper_->DumpRaw("aec3_echo_remover_capture_input", y0); data_dumper_->DumpRaw("aec3_echo_remover_render_input", x0); @@ -264,8 +279,7 @@ void EchoRemoverImpl::ProcessCapture( subtractor_output, y0); // Choose the linear output. - data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0], - LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0], 16000, 1); if (aec_state_.UseLinearFilterOutput()) { if (!linear_filter_output_last_selected_) { SignalTransition(y0, e, y0); @@ -280,8 +294,7 @@ void EchoRemoverImpl::ProcessCapture( linear_filter_output_last_selected_ = aec_state_.UseLinearFilterOutput(); const auto& Y_fft = aec_state_.UseLinearFilterOutput() ? E : Y; - data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0], - LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0], 16000, 1); // Estimate the residual echo power. residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2, @@ -317,16 +330,14 @@ void EchoRemoverImpl::ProcessCapture( // Debug outputs for the purpose of development and analysis. data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize, - &subtractor_output.s_main[0], - LowestBandRate(sample_rate_hz_), 1); + &subtractor_output.s_main[0], 16000, 1); data_dumper_->DumpRaw("aec3_output", y0); data_dumper_->DumpRaw("aec3_narrow_render", render_signal_analyzer_.NarrowPeakBand() ? 1 : 0); data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum()); data_dumper_->DumpRaw("aec3_suppressor_gain", G); - data_dumper_->DumpWav("aec3_output", - rtc::ArrayView(&y0[0], kBlockSize), - LowestBandRate(sample_rate_hz_), 1); + data_dumper_->DumpWav( + "aec3_output", rtc::ArrayView(&y0[0], kBlockSize), 16000, 1); data_dumper_->DumpRaw("aec3_using_subtractor_output", aec_state_.UseLinearFilterOutput() ? 1 : 0); data_dumper_->DumpRaw("aec3_E2", E2); @@ -390,8 +401,11 @@ void EchoRemoverImpl::FormLinearFilterOutput( } // namespace EchoRemover* EchoRemover::Create(const EchoCanceller3Config& config, - int sample_rate_hz) { - return new EchoRemoverImpl(config, sample_rate_hz); + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) { + return new EchoRemoverImpl(config, sample_rate_hz, num_render_channels, + num_capture_channels); } } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_remover.h b/modules/audio_processing/aec3/echo_remover.h index 357f67d51e..6098a68f14 100644 --- a/modules/audio_processing/aec3/echo_remover.h +++ b/modules/audio_processing/aec3/echo_remover.h @@ -26,7 +26,9 @@ namespace webrtc { class EchoRemover { public: static EchoRemover* Create(const EchoCanceller3Config& config, - int sample_rate_hz); + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels); virtual ~EchoRemover() = default; // Get current metrics. @@ -40,7 +42,7 @@ class EchoRemover { bool capture_signal_saturation, const absl::optional& external_delay, RenderBuffer* render_buffer, - std::vector>* capture) = 0; + std::vector>>* capture) = 0; // Updates the status on whether echo leakage is detected in the output of the // echo remover. diff --git a/modules/audio_processing/aec3/echo_remover_unittest.cc b/modules/audio_processing/aec3/echo_remover_unittest.cc index abe43ae3a6..15d091357a 100644 --- a/modules/audio_processing/aec3/echo_remover_unittest.cc +++ b/modules/audio_processing/aec3/echo_remover_unittest.cc @@ -44,29 +44,40 @@ std::string ProduceDebugText(int sample_rate_hz, int delay) { // Verifies the basic API call sequence TEST(EchoRemover, BasicApiCalls) { absl::optional delay_estimate; - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - std::unique_ptr remover( - EchoRemover::Create(EchoCanceller3Config(), rate)); - std::unique_ptr render_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), rate)); + for (auto rate : {16000, 32000, 48000}) { + for (size_t num_render_channels : {1, 2, 8}) { + for (size_t num_capture_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr remover( + EchoRemover::Create(EchoCanceller3Config(), rate, + num_render_channels, num_capture_channels)); + std::unique_ptr render_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), rate, + num_render_channels)); - std::vector> render(NumBandsForRate(rate), - std::vector(kBlockSize, 0.f)); - std::vector> capture( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); - for (size_t k = 0; k < 100; ++k) { - EchoPathVariability echo_path_variability( - k % 3 == 0 ? true : false, - k % 5 == 0 ? EchoPathVariability::DelayAdjustment::kNewDetectedDelay - : EchoPathVariability::DelayAdjustment::kNone, - false); - render_buffer->Insert(render); - render_buffer->PrepareCaptureProcessing(); + std::vector>> render( + NumBandsForRate(rate), + std::vector>( + num_render_channels, std::vector(kBlockSize, 0.f))); + std::vector>> capture( + NumBandsForRate(rate), + std::vector>( + num_capture_channels, std::vector(kBlockSize, 0.f))); + for (size_t k = 0; k < 100; ++k) { + EchoPathVariability echo_path_variability( + k % 3 == 0 ? true : false, + k % 5 == 0 + ? EchoPathVariability::DelayAdjustment::kNewDetectedDelay + : EchoPathVariability::DelayAdjustment::kNone, + false); + render_buffer->Insert(render); + render_buffer->PrepareCaptureProcessing(); - remover->ProcessCapture(echo_path_variability, k % 2 == 0 ? true : false, - delay_estimate, render_buffer->GetRenderBuffer(), - &capture); + remover->ProcessCapture(echo_path_variability, + k % 2 == 0 ? true : false, delay_estimate, + render_buffer->GetRenderBuffer(), &capture); + } + } } } } @@ -78,21 +89,22 @@ TEST(EchoRemover, BasicApiCalls) { // tests on test bots has been fixed. TEST(EchoRemover, DISABLED_WrongSampleRate) { EXPECT_DEATH(std::unique_ptr( - EchoRemover::Create(EchoCanceller3Config(), 8001)), + EchoRemover::Create(EchoCanceller3Config(), 8001, 1, 1)), ""); } // Verifies the check for the capture block size. TEST(EchoRemover, WrongCaptureBlockSize) { absl::optional delay_estimate; - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr remover( - EchoRemover::Create(EchoCanceller3Config(), rate)); + EchoRemover::Create(EchoCanceller3Config(), rate, 1, 1)); std::unique_ptr render_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), rate)); - std::vector> capture( - NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); + RenderDelayBuffer::Create(EchoCanceller3Config(), rate, 1)); + std::vector>> capture( + NumBandsForRate(rate), std::vector>( + 1, std::vector(kBlockSize - 1, 0.f))); EchoPathVariability echo_path_variability( false, EchoPathVariability::DelayAdjustment::kNone, false); EXPECT_DEATH( @@ -110,12 +122,13 @@ TEST(EchoRemover, DISABLED_WrongCaptureNumBands) { for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr remover( - EchoRemover::Create(EchoCanceller3Config(), rate)); + EchoRemover::Create(EchoCanceller3Config(), rate, 1, 1)); std::unique_ptr render_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), rate)); - std::vector> capture( + RenderDelayBuffer::Create(EchoCanceller3Config(), rate, 1)); + std::vector>> capture( NumBandsForRate(rate == 48000 ? 16000 : rate + 16000), - std::vector(kBlockSize, 0.f)); + std::vector>(1, + std::vector(kBlockSize, 0.f))); EchoPathVariability echo_path_variability( false, EchoPathVariability::DelayAdjustment::kNone, false); EXPECT_DEATH( @@ -129,9 +142,9 @@ TEST(EchoRemover, DISABLED_WrongCaptureNumBands) { TEST(EchoRemover, NullCapture) { absl::optional delay_estimate; std::unique_ptr remover( - EchoRemover::Create(EchoCanceller3Config(), 8000)); + EchoRemover::Create(EchoCanceller3Config(), 16000, 1, 1)); std::unique_ptr render_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 8000)); + RenderDelayBuffer::Create(EchoCanceller3Config(), 16000, 1)); EchoPathVariability echo_path_variability( false, EchoPathVariability::DelayAdjustment::kNone, false); EXPECT_DEATH( @@ -148,61 +161,76 @@ TEST(EchoRemover, BasicEchoRemoval) { constexpr int kNumBlocksToProcess = 500; Random random_generator(42U); absl::optional delay_estimate; - for (auto rate : {8000, 16000, 32000, 48000}) { - std::vector> x(NumBandsForRate(rate), - std::vector(kBlockSize, 0.f)); - std::vector> y(NumBandsForRate(rate), - std::vector(kBlockSize, 0.f)); - EchoPathVariability echo_path_variability( - false, EchoPathVariability::DelayAdjustment::kNone, false); - for (size_t delay_samples : {0, 64, 150, 200, 301}) { - SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); - EchoCanceller3Config config; - std::unique_ptr remover(EchoRemover::Create(config, rate)); - std::unique_ptr render_buffer( - RenderDelayBuffer::Create(config, rate)); - render_buffer->AlignFromDelay(delay_samples / kBlockSize); + for (size_t num_channels : {1, 2, 4}) { + for (auto rate : {16000, 32000, 48000}) { + std::vector>> x( + NumBandsForRate(rate), + std::vector>(num_channels, + std::vector(kBlockSize, 0.f))); + std::vector>> y( + NumBandsForRate(rate), + std::vector>(num_channels, + std::vector(kBlockSize, 0.f))); + EchoPathVariability echo_path_variability( + false, EchoPathVariability::DelayAdjustment::kNone, false); + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); + EchoCanceller3Config config; + std::unique_ptr remover( + EchoRemover::Create(config, rate, num_channels, num_channels)); + std::unique_ptr render_buffer( + RenderDelayBuffer::Create(config, rate, num_channels)); + render_buffer->AlignFromDelay(delay_samples / kBlockSize); - std::vector>> delay_buffers(x.size()); - for (size_t j = 0; j < x.size(); ++j) { - delay_buffers[j].reset(new DelayBuffer(delay_samples)); + std::vector>>> + delay_buffers(x.size()); + for (size_t band = 0; band < delay_buffers.size(); ++band) { + delay_buffers[band].resize(x[0].size()); + } + + for (size_t band = 0; band < x.size(); ++band) { + for (size_t channel = 0; channel < x[0].size(); ++channel) { + delay_buffers[band][channel].reset( + new DelayBuffer(delay_samples)); + } + } + + float input_energy = 0.f; + float output_energy = 0.f; + for (int k = 0; k < kNumBlocksToProcess; ++k) { + const bool silence = k < 100 || (k % 100 >= 10); + + for (size_t band = 0; band < x.size(); ++band) { + for (size_t channel = 0; channel < x[0].size(); ++channel) { + if (silence) { + std::fill(x[band][channel].begin(), x[band][channel].end(), + 0.f); + } else { + RandomizeSampleVector(&random_generator, x[band][channel]); + } + delay_buffers[band][channel]->Delay(x[band][channel], + y[band][channel]); + } + } + + if (k > kNumBlocksToProcess / 2) { + input_energy = std::inner_product(y[0][0].begin(), y[0][0].end(), + y[0][0].begin(), input_energy); + } + + render_buffer->Insert(x); + render_buffer->PrepareCaptureProcessing(); + + remover->ProcessCapture(echo_path_variability, false, delay_estimate, + render_buffer->GetRenderBuffer(), &y); + + if (k > kNumBlocksToProcess / 2) { + output_energy = std::inner_product(y[0][0].begin(), y[0][0].end(), + y[0][0].begin(), output_energy); + } + } + EXPECT_GT(input_energy, 10.f * output_energy); } - - float input_energy = 0.f; - float output_energy = 0.f; - for (int k = 0; k < kNumBlocksToProcess; ++k) { - const bool silence = k < 100 || (k % 100 >= 10); - - for (size_t j = 0; j < x.size(); ++j) { - if (silence) { - std::fill(x[j].begin(), x[j].end(), 0.f); - } else { - RandomizeSampleVector(&random_generator, x[j]); - } - delay_buffers[j]->Delay(x[j], y[j]); - } - - if (k > kNumBlocksToProcess / 2) { - for (size_t j = 0; j < x.size(); ++j) { - input_energy = std::inner_product(y[j].begin(), y[j].end(), - y[j].begin(), input_energy); - } - } - - render_buffer->Insert(x); - render_buffer->PrepareCaptureProcessing(); - - remover->ProcessCapture(echo_path_variability, false, delay_estimate, - render_buffer->GetRenderBuffer(), &y); - - if (k > kNumBlocksToProcess / 2) { - for (size_t j = 0; j < x.size(); ++j) { - output_energy = std::inner_product(y[j].begin(), y[j].end(), - y[j].begin(), output_energy); - } - } - } - EXPECT_GT(input_energy, 10.f * output_energy); } } } diff --git a/modules/audio_processing/aec3/erle_estimator_unittest.cc b/modules/audio_processing/aec3/erle_estimator_unittest.cc index 31c550c37d..18ba25a53b 100644 --- a/modules/audio_processing/aec3/erle_estimator_unittest.cc +++ b/modules/audio_processing/aec3/erle_estimator_unittest.cc @@ -46,7 +46,7 @@ void VerifyErle(rtc::ArrayView erle, EXPECT_NEAR(reference_lf, erle_time_domain, 0.5); } -void FormFarendTimeFrame(rtc::ArrayView x) { +void FormFarendTimeFrame(std::vector>>* x) { const std::array frame = { 7459.88, 17209.6, 17383, 20768.9, 16816.7, 18386.3, 4492.83, 9675.85, 6665.52, 14808.6, 9342.3, 7483.28, 19261.7, 4145.98, 1622.18, 13475.2, @@ -56,8 +56,12 @@ void FormFarendTimeFrame(rtc::ArrayView x) { 11405, 15031.4, 14541.6, 19765.5, 18346.3, 19350.2, 3157.47, 18095.8, 1743.68, 21328.2, 19727.5, 7295.16, 10332.4, 11055.5, 20107.4, 14708.4, 12416.2, 16434, 2454.69, 9840.8, 6867.23, 1615.75, 6059.9, 8394.19}; - RTC_DCHECK_GE(x.size(), frame.size()); - std::copy(frame.begin(), frame.end(), x.begin()); + for (size_t band = 0; band < x->size(); ++band) { + for (size_t channel = 0; channel < (*x)[band].size(); ++channel) { + RTC_DCHECK_GE((*x)[band][channel].size(), frame.size()); + std::copy(frame.begin(), frame.end(), (*x)[band][channel].begin()); + } + } } void FormFarendFrame(const RenderBuffer& render_buffer, @@ -75,14 +79,18 @@ void FormFarendFrame(const RenderBuffer& render_buffer, } // namespace -void FormNearendFrame(rtc::ArrayView x, +void FormNearendFrame(std::vector>>* x, std::array* X2, std::array* E2, std::array* Y2) { - x[0] = 0.f; - X2->fill(0.f); - Y2->fill(500.f * 1000.f * 1000.f); - E2->fill((*Y2)[0]); + for (size_t band = 0; band < x->size(); ++band) { + for (size_t channel = 0; channel < (*x)[band].size(); ++channel) { + std::fill((*x)[band][channel].begin(), (*x)[band][channel].end(), 0.f); + X2->fill(0.f); + Y2->fill(500.f * 1000.f * 1000.f); + E2->fill((*Y2)[0]); + } + } } void GetFilterFreq(std::vector>& @@ -104,18 +112,24 @@ TEST(ErleEstimator, VerifyErleIncreaseAndHold) { std::array X2; std::array E2; std::array Y2; + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + EchoCanceller3Config config; - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::vector> filter_frequency_response( config.filter.main.length_blocks); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); GetFilterFreq(filter_frequency_response, config.delay.delay_headroom_samples); ErleEstimator estimator(0, config); - FormFarendTimeFrame(x[0]); + FormFarendTimeFrame(&x); render_delay_buffer->Insert(x); render_delay_buffer->PrepareCaptureProcessing(); // Verifies that the ERLE estimate is properly increased to higher values. @@ -130,7 +144,7 @@ TEST(ErleEstimator, VerifyErleIncreaseAndHold) { VerifyErle(estimator.Erle(), std::pow(2.f, estimator.FullbandErleLog2()), config.erle.max_l, config.erle.max_h); - FormNearendFrame(x[0], &X2, &E2, &Y2); + FormNearendFrame(&x, &X2, &E2, &Y2); // Verifies that the ERLE is not immediately decreased during nearend // activity. for (size_t k = 0; k < 50; ++k) { @@ -144,22 +158,27 @@ TEST(ErleEstimator, VerifyErleIncreaseAndHold) { } TEST(ErleEstimator, VerifyErleTrackingOnOnsets) { + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); std::array X2; std::array E2; std::array Y2; EchoCanceller3Config config; - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::vector> filter_frequency_response( config.filter.main.length_blocks); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); GetFilterFreq(filter_frequency_response, config.delay.delay_headroom_samples); ErleEstimator estimator(0, config); - FormFarendTimeFrame(x[0]); + FormFarendTimeFrame(&x); render_delay_buffer->Insert(x); render_delay_buffer->PrepareCaptureProcessing(); @@ -180,7 +199,7 @@ TEST(ErleEstimator, VerifyErleTrackingOnOnsets) { estimator.Update(*render_delay_buffer->GetRenderBuffer(), filter_frequency_response, X2, Y2, E2, true, true); } - FormNearendFrame(x[0], &X2, &E2, &Y2); + FormNearendFrame(&x, &X2, &E2, &Y2); for (size_t k = 0; k < 300; ++k) { render_delay_buffer->Insert(x); render_delay_buffer->PrepareCaptureProcessing(); @@ -189,7 +208,7 @@ TEST(ErleEstimator, VerifyErleTrackingOnOnsets) { } } VerifyErleBands(estimator.ErleOnsets(), config.erle.min, config.erle.min); - FormNearendFrame(x[0], &X2, &E2, &Y2); + FormNearendFrame(&x, &X2, &E2, &Y2); for (size_t k = 0; k < 1000; k++) { estimator.Update(*render_delay_buffer->GetRenderBuffer(), filter_frequency_response, X2, Y2, E2, true, true); diff --git a/modules/audio_processing/aec3/filter_analyzer.cc b/modules/audio_processing/aec3/filter_analyzer.cc index 06bd4b7934..138c188970 100644 --- a/modules/audio_processing/aec3/filter_analyzer.cc +++ b/modules/audio_processing/aec3/filter_analyzer.cc @@ -96,8 +96,8 @@ void FilterAnalyzer::AnalyzeRegion( filter_length_blocks_ = filter_time_domain.size() * (1.f / kBlockSize); consistent_estimate_ = consistent_filter_detector_.Detect( - h_highpass_, region_, render_buffer.Block(-delay_blocks_)[0], peak_index_, - delay_blocks_); + h_highpass_, region_, render_buffer.Block(-delay_blocks_)[0][0], + peak_index_, delay_blocks_); } void FilterAnalyzer::UpdateFilterGain( diff --git a/modules/audio_processing/aec3/frame_blocker.cc b/modules/audio_processing/aec3/frame_blocker.cc index ca122e5ebb..63aaf098c5 100644 --- a/modules/audio_processing/aec3/frame_blocker.cc +++ b/modules/audio_processing/aec3/frame_blocker.cc @@ -15,55 +15,73 @@ namespace webrtc { -FrameBlocker::FrameBlocker(size_t num_bands) - : num_bands_(num_bands), buffer_(num_bands_) { - for (auto& b : buffer_) { - b.reserve(kBlockSize); - RTC_DCHECK(b.empty()); +FrameBlocker::FrameBlocker(size_t num_bands, size_t num_channels) + : num_bands_(num_bands), + num_channels_(num_channels), + buffer_(num_bands_, std::vector>(num_channels)) { + RTC_DCHECK_LT(0, num_bands); + RTC_DCHECK_LT(0, num_channels); + for (auto& band : buffer_) { + for (auto& channel : band) { + channel.reserve(kBlockSize); + RTC_DCHECK(channel.empty()); + } } } FrameBlocker::~FrameBlocker() = default; void FrameBlocker::InsertSubFrameAndExtractBlock( - const std::vector>& sub_frame, - std::vector>* block) { + const std::vector>>& sub_frame, + std::vector>>* block) { RTC_DCHECK(block); RTC_DCHECK_EQ(num_bands_, block->size()); RTC_DCHECK_EQ(num_bands_, sub_frame.size()); - for (size_t i = 0; i < num_bands_; ++i) { - RTC_DCHECK_GE(kBlockSize - 16, buffer_[i].size()); - RTC_DCHECK_EQ(kBlockSize, (*block)[i].size()); - RTC_DCHECK_EQ(kSubFrameLength, sub_frame[i].size()); - const int samples_to_block = kBlockSize - buffer_[i].size(); - (*block)[i].clear(); - (*block)[i].insert((*block)[i].begin(), buffer_[i].begin(), - buffer_[i].end()); - (*block)[i].insert((*block)[i].begin() + buffer_[i].size(), - sub_frame[i].begin(), - sub_frame[i].begin() + samples_to_block); - buffer_[i].clear(); - buffer_[i].insert(buffer_[i].begin(), - sub_frame[i].begin() + samples_to_block, - sub_frame[i].end()); + for (size_t band = 0; band < num_bands_; ++band) { + RTC_DCHECK_EQ(num_channels_, (*block)[band].size()); + RTC_DCHECK_EQ(num_channels_, sub_frame[band].size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + RTC_DCHECK_GE(kBlockSize - 16, buffer_[band][channel].size()); + RTC_DCHECK_EQ(kBlockSize, (*block)[band][channel].size()); + RTC_DCHECK_EQ(kSubFrameLength, sub_frame[band][channel].size()); + const int samples_to_block = kBlockSize - buffer_[band][channel].size(); + (*block)[band][channel].clear(); + (*block)[band][channel].insert((*block)[band][channel].begin(), + buffer_[band][channel].begin(), + buffer_[band][channel].end()); + (*block)[band][channel].insert( + (*block)[band][channel].begin() + buffer_[band][channel].size(), + sub_frame[band][channel].begin(), + sub_frame[band][channel].begin() + samples_to_block); + buffer_[band][channel].clear(); + buffer_[band][channel].insert( + buffer_[band][channel].begin(), + sub_frame[band][channel].begin() + samples_to_block, + sub_frame[band][channel].end()); + } } } bool FrameBlocker::IsBlockAvailable() const { - return kBlockSize == buffer_[0].size(); + return kBlockSize == buffer_[0][0].size(); } -void FrameBlocker::ExtractBlock(std::vector>* block) { +void FrameBlocker::ExtractBlock( + std::vector>>* block) { RTC_DCHECK(block); RTC_DCHECK_EQ(num_bands_, block->size()); RTC_DCHECK(IsBlockAvailable()); - for (size_t i = 0; i < num_bands_; ++i) { - RTC_DCHECK_EQ(kBlockSize, buffer_[i].size()); - RTC_DCHECK_EQ(kBlockSize, (*block)[i].size()); - (*block)[i].clear(); - (*block)[i].insert((*block)[i].begin(), buffer_[i].begin(), - buffer_[i].end()); - buffer_[i].clear(); + for (size_t band = 0; band < num_bands_; ++band) { + RTC_DCHECK_EQ(num_channels_, (*block)[band].size()); + for (size_t channel = 0; channel < num_channels_; ++channel) { + RTC_DCHECK_EQ(kBlockSize, buffer_[band][channel].size()); + RTC_DCHECK_EQ(kBlockSize, (*block)[band][channel].size()); + (*block)[band][channel].clear(); + (*block)[band][channel].insert((*block)[band][channel].begin(), + buffer_[band][channel].begin(), + buffer_[band][channel].end()); + buffer_[band][channel].clear(); + } } } diff --git a/modules/audio_processing/aec3/frame_blocker.h b/modules/audio_processing/aec3/frame_blocker.h index 759f431721..ebd6f776f1 100644 --- a/modules/audio_processing/aec3/frame_blocker.h +++ b/modules/audio_processing/aec3/frame_blocker.h @@ -17,32 +17,33 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { -// Class for producing 64 sample multiband blocks from frames consisting of 1 or -// 2 subframes of 80 samples. +// Class for producing 64 sample multiband blocks from frames consisting of 2 +// subframes of 80 samples. class FrameBlocker { public: - explicit FrameBlocker(size_t num_bands); + FrameBlocker(size_t num_bands, size_t num_channels); ~FrameBlocker(); + FrameBlocker(const FrameBlocker&) = delete; + FrameBlocker& operator=(const FrameBlocker&) = delete; + // Inserts one 80 sample multiband subframe from the multiband frame and // extracts one 64 sample multiband block. void InsertSubFrameAndExtractBlock( - const std::vector>& sub_frame, - std::vector>* block); + const std::vector>>& sub_frame, + std::vector>>* block); // Reports whether a multiband block of 64 samples is available for // extraction. bool IsBlockAvailable() const; // Extracts a multiband block of 64 samples. - void ExtractBlock(std::vector>* block); + void ExtractBlock(std::vector>>* block); private: const size_t num_bands_; - std::vector> buffer_; - - RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBlocker); + const size_t num_channels_; + std::vector>> buffer_; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/frame_blocker_unittest.cc b/modules/audio_processing/aec3/frame_blocker_unittest.cc index 3ec74cc093..e907608d95 100644 --- a/modules/audio_processing/aec3/frame_blocker_unittest.cc +++ b/modules/audio_processing/aec3/frame_blocker_unittest.cc @@ -24,45 +24,62 @@ namespace { float ComputeSampleValue(size_t chunk_counter, size_t chunk_size, size_t band, + size_t channel, size_t sample_index, int offset) { float value = - static_cast(chunk_counter * chunk_size + sample_index) + offset; + static_cast(chunk_counter * chunk_size + sample_index + channel) + + offset; return value > 0 ? 5000 * band + value : 0; } void FillSubFrame(size_t sub_frame_counter, int offset, - std::vector>* sub_frame) { - for (size_t k = 0; k < sub_frame->size(); ++k) { - for (size_t i = 0; i < (*sub_frame)[0].size(); ++i) { - (*sub_frame)[k][i] = - ComputeSampleValue(sub_frame_counter, kSubFrameLength, k, i, offset); + std::vector>>* sub_frame) { + for (size_t band = 0; band < sub_frame->size(); ++band) { + for (size_t channel = 0; channel < (*sub_frame)[band].size(); ++channel) { + for (size_t sample = 0; sample < (*sub_frame)[band][channel].size(); + ++sample) { + (*sub_frame)[band][channel][sample] = ComputeSampleValue( + sub_frame_counter, kSubFrameLength, band, channel, sample, offset); + } } } } -void FillSubFrameView(size_t sub_frame_counter, - int offset, - std::vector>* sub_frame, - std::vector>* sub_frame_view) { +void FillSubFrameView( + size_t sub_frame_counter, + int offset, + std::vector>>* sub_frame, + std::vector>>* sub_frame_view) { FillSubFrame(sub_frame_counter, offset, sub_frame); - for (size_t k = 0; k < sub_frame_view->size(); ++k) { - (*sub_frame_view)[k] = - rtc::ArrayView(&(*sub_frame)[k][0], (*sub_frame)[k].size()); + for (size_t band = 0; band < sub_frame_view->size(); ++band) { + for (size_t channel = 0; channel < (*sub_frame_view)[band].size(); + ++channel) { + (*sub_frame_view)[band][channel] = rtc::ArrayView( + &(*sub_frame)[band][channel][0], (*sub_frame)[band][channel].size()); + } } } -bool VerifySubFrame(size_t sub_frame_counter, - int offset, - const std::vector>& sub_frame_view) { - std::vector> reference_sub_frame( - sub_frame_view.size(), std::vector(sub_frame_view[0].size(), 0.f)); +bool VerifySubFrame( + size_t sub_frame_counter, + int offset, + const std::vector>>& sub_frame_view) { + std::vector>> reference_sub_frame( + sub_frame_view.size(), + std::vector>( + sub_frame_view[0].size(), + std::vector(sub_frame_view[0][0].size(), 0.f))); FillSubFrame(sub_frame_counter, offset, &reference_sub_frame); - for (size_t k = 0; k < sub_frame_view.size(); ++k) { - for (size_t i = 0; i < sub_frame_view[k].size(); ++i) { - if (reference_sub_frame[k][i] != sub_frame_view[k][i]) { - return false; + for (size_t band = 0; band < sub_frame_view.size(); ++band) { + for (size_t channel = 0; channel < sub_frame_view[band].size(); ++channel) { + for (size_t sample = 0; sample < sub_frame_view[band][channel].size(); + ++sample) { + if (reference_sub_frame[band][channel][sample] != + sub_frame_view[band][channel][sample]) { + return false; + } } } } @@ -71,13 +88,15 @@ bool VerifySubFrame(size_t sub_frame_counter, bool VerifyBlock(size_t block_counter, int offset, - const std::vector>& block) { - for (size_t k = 0; k < block.size(); ++k) { - for (size_t i = 0; i < block[k].size(); ++i) { - const float reference_value = - ComputeSampleValue(block_counter, kBlockSize, k, i, offset); - if (reference_value != block[k][i]) { - return false; + const std::vector>>& block) { + for (size_t band = 0; band < block.size(); ++band) { + for (size_t channel = 0; channel < block[band].size(); ++channel) { + for (size_t sample = 0; sample < block[band][channel].size(); ++sample) { + const float reference_value = ComputeSampleValue( + block_counter, kBlockSize, band, channel, sample, offset); + if (reference_value != block[band][channel][sample]) { + return false; + } } } } @@ -85,16 +104,19 @@ bool VerifyBlock(size_t block_counter, } // Verifies that the FrameBlocker properly forms blocks out of the frames. -void RunBlockerTest(int sample_rate_hz) { +void RunBlockerTest(int sample_rate_hz, size_t num_channels) { constexpr size_t kNumSubFramesToProcess = 20; const size_t num_bands = NumBandsForRate(sample_rate_hz); - std::vector> block(num_bands, - std::vector(kBlockSize, 0.f)); - std::vector> input_sub_frame( - num_bands, std::vector(kSubFrameLength, 0.f)); - std::vector> input_sub_frame_view(num_bands); - FrameBlocker blocker(num_bands); + std::vector>> block( + num_bands, std::vector>( + num_channels, std::vector(kBlockSize, 0.f))); + std::vector>> input_sub_frame( + num_bands, std::vector>( + num_channels, std::vector(kSubFrameLength, 0.f))); + std::vector>> input_sub_frame_view( + num_bands, std::vector>(num_channels)); + FrameBlocker blocker(num_bands, num_channels); size_t block_counter = 0; for (size_t sub_frame_index = 0; sub_frame_index < kNumSubFramesToProcess; @@ -119,20 +141,25 @@ void RunBlockerTest(int sample_rate_hz) { // Verifies that the FrameBlocker and BlockFramer work well together and produce // the expected output. -void RunBlockerAndFramerTest(int sample_rate_hz) { +void RunBlockerAndFramerTest(int sample_rate_hz, size_t num_channels) { const size_t kNumSubFramesToProcess = 20; const size_t num_bands = NumBandsForRate(sample_rate_hz); - std::vector> block(num_bands, - std::vector(kBlockSize, 0.f)); - std::vector> input_sub_frame( - num_bands, std::vector(kSubFrameLength, 0.f)); - std::vector> output_sub_frame( - num_bands, std::vector(kSubFrameLength, 0.f)); - std::vector> output_sub_frame_view(num_bands); - std::vector> input_sub_frame_view(num_bands); - FrameBlocker blocker(num_bands); - BlockFramer framer(num_bands); + std::vector>> block( + num_bands, std::vector>( + num_channels, std::vector(kBlockSize, 0.f))); + std::vector>> input_sub_frame( + num_bands, std::vector>( + num_channels, std::vector(kSubFrameLength, 0.f))); + std::vector>> output_sub_frame( + num_bands, std::vector>( + num_channels, std::vector(kSubFrameLength, 0.f))); + std::vector>> output_sub_frame_view( + num_bands, std::vector>(num_channels)); + std::vector>> input_sub_frame_view( + num_bands, std::vector>(num_channels)); + FrameBlocker blocker(num_bands, num_channels); + BlockFramer framer(num_bands, num_channels); for (size_t sub_frame_index = 0; sub_frame_index < kNumSubFramesToProcess; ++sub_frame_index) { @@ -153,28 +180,39 @@ void RunBlockerAndFramerTest(int sample_rate_hz) { blocker.ExtractBlock(&block); framer.InsertBlock(block); } - EXPECT_TRUE(VerifySubFrame(sub_frame_index, -64, output_sub_frame_view)); + if (sub_frame_index > 1) { + EXPECT_TRUE(VerifySubFrame(sub_frame_index, -64, output_sub_frame_view)); + } } } #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the FrameBlocker crashes if the InsertSubFrameAndExtractBlock // method is called for inputs with the wrong number of bands or band lengths. -void RunWronglySizedInsertAndExtractParametersTest(int sample_rate_hz, - size_t num_block_bands, - size_t block_length, - size_t num_sub_frame_bands, - size_t sub_frame_length) { +void RunWronglySizedInsertAndExtractParametersTest( + int sample_rate_hz, + size_t correct_num_channels, + size_t num_block_bands, + size_t num_block_channels, + size_t block_length, + size_t num_sub_frame_bands, + size_t num_sub_frame_channels, + size_t sub_frame_length) { const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); - std::vector> block(num_block_bands, - std::vector(block_length, 0.f)); - std::vector> input_sub_frame( - num_sub_frame_bands, std::vector(sub_frame_length, 0.f)); - std::vector> input_sub_frame_view( - input_sub_frame.size()); + std::vector>> block( + num_block_bands, + std::vector>(num_block_channels, + std::vector(block_length, 0.f))); + std::vector>> input_sub_frame( + num_sub_frame_bands, + std::vector>( + num_sub_frame_channels, std::vector(sub_frame_length, 0.f))); + std::vector>> input_sub_frame_view( + input_sub_frame.size(), + std::vector>(num_sub_frame_channels)); FillSubFrameView(0, 0, &input_sub_frame, &input_sub_frame_view); - FrameBlocker blocker(correct_num_bands); + FrameBlocker blocker(correct_num_bands, correct_num_channels); EXPECT_DEATH( blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &block), ""); } @@ -182,20 +220,29 @@ void RunWronglySizedInsertAndExtractParametersTest(int sample_rate_hz, // Verifies that the FrameBlocker crashes if the ExtractBlock method is called // for inputs with the wrong number of bands or band lengths. void RunWronglySizedExtractParameterTest(int sample_rate_hz, + size_t correct_num_channels, size_t num_block_bands, + size_t num_block_channels, size_t block_length) { const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); - std::vector> correct_block( - correct_num_bands, std::vector(kBlockSize, 0.f)); - std::vector> wrong_block( - num_block_bands, std::vector(block_length, 0.f)); - std::vector> input_sub_frame( - correct_num_bands, std::vector(kSubFrameLength, 0.f)); - std::vector> input_sub_frame_view( - input_sub_frame.size()); + std::vector>> correct_block( + correct_num_bands, + std::vector>(correct_num_channels, + std::vector(kBlockSize, 0.f))); + std::vector>> wrong_block( + num_block_bands, + std::vector>(num_block_channels, + std::vector(block_length, 0.f))); + std::vector>> input_sub_frame( + correct_num_bands, + std::vector>( + correct_num_channels, std::vector(kSubFrameLength, 0.f))); + std::vector>> input_sub_frame_view( + input_sub_frame.size(), + std::vector>(correct_num_channels)); FillSubFrameView(0, 0, &input_sub_frame, &input_sub_frame_view); - FrameBlocker blocker(correct_num_bands); + FrameBlocker blocker(correct_num_bands, correct_num_channels); blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &correct_block); blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &correct_block); blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &correct_block); @@ -208,17 +255,20 @@ void RunWronglySizedExtractParameterTest(int sample_rate_hz, // after a wrong number of previous InsertSubFrameAndExtractBlock method calls // have been made. void RunWrongExtractOrderTest(int sample_rate_hz, + size_t num_channels, size_t num_preceeding_api_calls) { - const size_t correct_num_bands = NumBandsForRate(sample_rate_hz); + const size_t num_bands = NumBandsForRate(sample_rate_hz); - std::vector> block(correct_num_bands, - std::vector(kBlockSize, 0.f)); - std::vector> input_sub_frame( - correct_num_bands, std::vector(kSubFrameLength, 0.f)); - std::vector> input_sub_frame_view( - input_sub_frame.size()); + std::vector>> block( + num_bands, std::vector>( + num_channels, std::vector(kBlockSize, 0.f))); + std::vector>> input_sub_frame( + num_bands, std::vector>( + num_channels, std::vector(kSubFrameLength, 0.f))); + std::vector>> input_sub_frame_view( + input_sub_frame.size(), std::vector>(num_channels)); FillSubFrameView(0, 0, &input_sub_frame, &input_sub_frame_view); - FrameBlocker blocker(correct_num_bands); + FrameBlocker blocker(num_bands, num_channels); for (size_t k = 0; k < num_preceeding_api_calls; ++k) { blocker.InsertSubFrameAndExtractBlock(input_sub_frame_view, &block); } @@ -227,9 +277,10 @@ void RunWrongExtractOrderTest(int sample_rate_hz, } #endif -std::string ProduceDebugText(int sample_rate_hz) { +std::string ProduceDebugText(int sample_rate_hz, size_t num_channels) { rtc::StringBuilder ss; ss << "Sample rate: " << sample_rate_hz; + ss << ", number of channels: " << num_channels; return ss.Release(); } @@ -237,104 +288,183 @@ std::string ProduceDebugText(int sample_rate_hz) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) TEST(FrameBlocker, WrongNumberOfBandsInBlockForInsertSubFrameAndExtractBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - const size_t wrong_num_bands = (correct_num_bands % 3) + 1; - RunWronglySizedInsertAndExtractParametersTest( - rate, wrong_num_bands, kBlockSize, correct_num_bands, kSubFrameLength); + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, wrong_num_bands, correct_num_channels, + kBlockSize, correct_num_bands, correct_num_channels, kSubFrameLength); + } + } +} + +TEST(FrameBlocker, + WrongNumberOfChannelsInBlockForInsertSubFrameAndExtractBlock) { + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_channels = correct_num_channels + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, wrong_num_channels, + kBlockSize, correct_num_bands, correct_num_channels, kSubFrameLength); + } } } TEST(FrameBlocker, WrongNumberOfBandsInSubFrameForInsertSubFrameAndExtractBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - const size_t wrong_num_bands = (correct_num_bands % 3) + 1; - RunWronglySizedInsertAndExtractParametersTest( - rate, correct_num_bands, kBlockSize, wrong_num_bands, kSubFrameLength); + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, correct_num_channels, + kBlockSize, wrong_num_bands, correct_num_channels, kSubFrameLength); + } + } +} + +TEST(FrameBlocker, + WrongNumberOfChannelsInSubFrameForInsertSubFrameAndExtractBlock) { + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_channels = correct_num_channels + 1; + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, wrong_num_channels, + kBlockSize, correct_num_bands, wrong_num_channels, kSubFrameLength); + } } } TEST(FrameBlocker, WrongNumberOfSamplesInBlockForInsertSubFrameAndExtractBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - RunWronglySizedInsertAndExtractParametersTest( - rate, correct_num_bands, kBlockSize - 1, correct_num_bands, - kSubFrameLength); + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, correct_num_channels, + kBlockSize - 1, correct_num_bands, correct_num_channels, + kSubFrameLength); + } } } TEST(FrameBlocker, WrongNumberOfSamplesInSubFrameForInsertSubFrameAndExtractBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - RunWronglySizedInsertAndExtractParametersTest(rate, correct_num_bands, - kBlockSize, correct_num_bands, - kSubFrameLength - 1); - } -} - -TEST(FrameBlocker, WrongNumberOfBandsInBlockForExtractBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - const size_t wrong_num_bands = (correct_num_bands % 3) + 1; - RunWronglySizedExtractParameterTest(rate, wrong_num_bands, kBlockSize); - } -} - -TEST(FrameBlocker, WrongNumberOfSamplesInBlockForExtractBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - const size_t correct_num_bands = NumBandsForRate(rate); - RunWronglySizedExtractParameterTest(rate, correct_num_bands, - kBlockSize - 1); - } -} - -TEST(FrameBlocker, WrongNumberOfPreceedingApiCallsForExtractBlock) { - for (auto rate : {8000, 16000, 32000, 48000}) { - for (size_t num_calls = 0; num_calls < 4; ++num_calls) { - rtc::StringBuilder ss; - ss << "Sample rate: " << rate; - ss << ", Num preceeding InsertSubFrameAndExtractBlock calls: " - << num_calls; - - SCOPED_TRACE(ss.str()); - RunWrongExtractOrderTest(rate, num_calls); + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedInsertAndExtractParametersTest( + rate, correct_num_channels, correct_num_bands, correct_num_channels, + kBlockSize, correct_num_bands, correct_num_channels, + kSubFrameLength - 1); } } } +TEST(FrameBlocker, WrongNumberOfBandsInBlockForExtractBlock) { + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_bands = (correct_num_bands % 3) + 1; + RunWronglySizedExtractParameterTest(rate, correct_num_channels, + wrong_num_bands, correct_num_channels, + kBlockSize); + } + } +} + +TEST(FrameBlocker, WrongNumberOfChannelsInBlockForExtractBlock) { + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + const size_t wrong_num_channels = correct_num_channels + 1; + RunWronglySizedExtractParameterTest(rate, correct_num_channels, + correct_num_bands, wrong_num_channels, + kBlockSize); + } + } +} + +TEST(FrameBlocker, WrongNumberOfSamplesInBlockForExtractBlock) { + for (auto rate : {16000, 32000, 48000}) { + for (size_t correct_num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); + const size_t correct_num_bands = NumBandsForRate(rate); + RunWronglySizedExtractParameterTest(rate, correct_num_channels, + correct_num_bands, + correct_num_channels, kBlockSize - 1); + } + } +} + +TEST(FrameBlocker, WrongNumberOfPreceedingApiCallsForExtractBlock) { + for (auto rate : {16000, 32000, 48000}) { + for (size_t num_channels : {1, 2, 4, 8}) { + for (size_t num_calls = 0; num_calls < 4; ++num_calls) { + rtc::StringBuilder ss; + ss << "Sample rate: " << rate; + ss << "Num channels: " << num_channels; + ss << ", Num preceeding InsertSubFrameAndExtractBlock calls: " + << num_calls; + + SCOPED_TRACE(ss.str()); + RunWrongExtractOrderTest(rate, num_channels, num_calls); + } + } + } +} + +// Verifies that the verification for 0 number of channels works. +TEST(FrameBlocker, ZeroNumberOfChannelsParameter) { + EXPECT_DEATH(FrameBlocker(16000, 0), ""); +} + +// Verifies that the verification for 0 number of bands works. +TEST(FrameBlocker, ZeroNumberOfBandsParameter) { + EXPECT_DEATH(FrameBlocker(0, 1), ""); +} + // Verifiers that the verification for null sub_frame pointer works. TEST(FrameBlocker, NullBlockParameter) { - std::vector> sub_frame( - 1, std::vector(kSubFrameLength, 0.f)); - std::vector> sub_frame_view(sub_frame.size()); + std::vector>> sub_frame( + 1, std::vector>( + 1, std::vector(kSubFrameLength, 0.f))); + std::vector>> sub_frame_view( + sub_frame.size()); FillSubFrameView(0, 0, &sub_frame, &sub_frame_view); EXPECT_DEATH( - FrameBlocker(1).InsertSubFrameAndExtractBlock(sub_frame_view, nullptr), + FrameBlocker(1, 1).InsertSubFrameAndExtractBlock(sub_frame_view, nullptr), ""); } #endif TEST(FrameBlocker, BlockBitexactness) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - RunBlockerTest(rate); + for (auto rate : {16000, 32000, 48000}) { + for (size_t num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, num_channels)); + RunBlockerTest(rate, num_channels); + } } } TEST(FrameBlocker, BlockerAndFramer) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - RunBlockerAndFramerTest(rate); + for (auto rate : {16000, 32000, 48000}) { + for (size_t num_channels : {1, 2, 4, 8}) { + SCOPED_TRACE(ProduceDebugText(rate, num_channels)); + RunBlockerAndFramerTest(rate, num_channels); + } } } diff --git a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc index 34412b88d0..648762a9c8 100644 --- a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc +++ b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc @@ -42,6 +42,10 @@ void RunFilterUpdateTest(int num_blocks_to_process, std::array* y_last_block, FftData* G_last_block) { ApmDataDumper data_dumper(42); + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + EchoCanceller3Config config; config.filter.main.length_blocks = filter_length_blocks; config.filter.shadow.length_blocks = filter_length_blocks; @@ -61,11 +65,13 @@ void RunFilterUpdateTest(int num_blocks_to_process, MainFilterUpdateGain main_gain(config.filter.main, config.filter.config_change_duration_blocks); Random random_generator(42U); - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::vector y(kBlockSize, 0.f); config.delay.default_delay = 1; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); AecState aec_state(config); RenderSignalAnalyzer render_signal_analyzer(config); absl::optional delay_estimate; @@ -101,11 +107,19 @@ void RunFilterUpdateTest(int num_blocks_to_process, // Create the render signal. if (use_silent_render_in_second_half && k > num_blocks_to_process / 2) { - std::fill(x[0].begin(), x[0].end(), 0.f); + for (size_t band = 0; band < x.size(); ++band) { + for (size_t channel = 0; channel < x[band].size(); ++channel) { + std::fill(x[band][channel].begin(), x[band][channel].end(), 0.f); + } + } } else { - RandomizeSampleVector(&random_generator, x[0]); + for (size_t band = 0; band < x.size(); ++band) { + for (size_t channel = 0; channel < x[band].size(); ++channel) { + RandomizeSampleVector(&random_generator, x[band][channel]); + } + } } - delay_buffer.Delay(x[0], y); + delay_buffer.Delay(x[0][0], y); render_delay_buffer->Insert(x); if (k == 0) { diff --git a/modules/audio_processing/aec3/matched_filter.cc b/modules/audio_processing/aec3/matched_filter.cc index 757219d52c..5a62b7cf8a 100644 --- a/modules/audio_processing/aec3/matched_filter.cc +++ b/modules/audio_processing/aec3/matched_filter.cc @@ -442,15 +442,15 @@ void MatchedFilter::LogFilterProperties(int sample_rate_hz, size_t shift, size_t downsampling_factor) const { size_t alignment_shift = 0; - const int fs_by_1000 = LowestBandRate(sample_rate_hz) / 1000; + constexpr int kFsBy1000 = 16; for (size_t k = 0; k < filters_.size(); ++k) { int start = static_cast(alignment_shift * downsampling_factor); int end = static_cast((alignment_shift + filters_[k].size()) * downsampling_factor); RTC_LOG(LS_INFO) << "Filter " << k << ": start: " - << (start - static_cast(shift)) / fs_by_1000 + << (start - static_cast(shift)) / kFsBy1000 << " ms, end: " - << (end - static_cast(shift)) / fs_by_1000 << " ms."; + << (end - static_cast(shift)) / kFsBy1000 << " ms."; alignment_shift += filter_intra_lag_shift_; } } diff --git a/modules/audio_processing/aec3/matched_filter_unittest.cc b/modules/audio_processing/aec3/matched_filter_unittest.cc index c204af4366..8f2c5c2ae3 100644 --- a/modules/audio_processing/aec3/matched_filter_unittest.cc +++ b/modules/audio_processing/aec3/matched_filter_unittest.cc @@ -140,11 +140,16 @@ TEST(MatchedFilter, TestSse2Optimizations) { // delayed signals. TEST(MatchedFilter, LagEstimation) { Random random_generator(42U); + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + for (auto down_sampling_factor : kDownSamplingFactors) { const size_t sub_block_size = kBlockSize / down_sampling_factor; - std::vector> render(3, - std::vector(kBlockSize, 0.f)); + std::vector>> render( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::array capture; capture.fill(0.f); ApmDataDumper data_dumper(0); @@ -163,12 +168,16 @@ TEST(MatchedFilter, LagEstimation) { config.delay.delay_candidate_detection_threshold); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); // Analyze the correlation between render and capture. for (size_t k = 0; k < (600 + delay_samples / sub_block_size); ++k) { - RandomizeSampleVector(&random_generator, render[0]); - signal_delay_buffer.Delay(render[0], capture); + for (size_t band = 0; band < kNumBands; ++band) { + for (size_t channel = 0; channel < kNumChannels; ++channel) { + RandomizeSampleVector(&random_generator, render[band][channel]); + } + } + signal_delay_buffer.Delay(render[0][0], capture); render_delay_buffer->Insert(render); if (k == 0) { @@ -245,6 +254,9 @@ TEST(MatchedFilter, LagEstimation) { // Verifies that the matched filter does not produce reliable and accurate // estimates for uncorrelated render and capture signals. TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) { + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); Random random_generator(42U); for (auto down_sampling_factor : kDownSamplingFactors) { EchoCanceller3Config config; @@ -252,14 +264,15 @@ TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) { config.delay.num_filters = kNumMatchedFilters; const size_t sub_block_size = kBlockSize / down_sampling_factor; - std::vector> render(3, - std::vector(kBlockSize, 0.f)); + std::vector>> render( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::array capture_data; rtc::ArrayView capture(capture_data.data(), sub_block_size); std::fill(capture.begin(), capture.end(), 0.f); ApmDataDumper data_dumper(0); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); MatchedFilter filter(&data_dumper, DetectOptimization(), sub_block_size, kWindowSizeSubBlocks, kNumMatchedFilters, kAlignmentShiftSubBlocks, 150, @@ -268,7 +281,7 @@ TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) { // Analyze the correlation between render and capture. for (size_t k = 0; k < 100; ++k) { - RandomizeSampleVector(&random_generator, render[0]); + RandomizeSampleVector(&random_generator, render[0][0]); RandomizeSampleVector(&random_generator, capture); render_delay_buffer->Insert(render); filter.Update(render_delay_buffer->GetDownsampledRenderBuffer(), capture); @@ -289,11 +302,16 @@ TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) { // render signals of low level. TEST(MatchedFilter, LagNotUpdatedForLowLevelRender) { Random random_generator(42U); + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + for (auto down_sampling_factor : kDownSamplingFactors) { const size_t sub_block_size = kBlockSize / down_sampling_factor; - std::vector> render(3, - std::vector(kBlockSize, 0.f)); + std::vector>> render( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::array capture; capture.fill(0.f); ApmDataDumper data_dumper(0); @@ -304,16 +322,17 @@ TEST(MatchedFilter, LagNotUpdatedForLowLevelRender) { config.delay.delay_estimate_smoothing, config.delay.delay_candidate_detection_threshold); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 48000)); + RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz, + kNumChannels)); Decimator capture_decimator(down_sampling_factor); // Analyze the correlation between render and capture. for (size_t k = 0; k < 100; ++k) { - RandomizeSampleVector(&random_generator, render[0]); - for (auto& render_k : render[0]) { + RandomizeSampleVector(&random_generator, render[0][0]); + for (auto& render_k : render[0][0]) { render_k *= 149.f / 32767.f; } - std::copy(render[0].begin(), render[0].end(), capture.begin()); + std::copy(render[0][0].begin(), render[0][0].end(), capture.begin()); std::array downsampled_capture_data; rtc::ArrayView downsampled_capture(downsampled_capture_data.data(), sub_block_size); diff --git a/modules/audio_processing/aec3/matrix_buffer.cc b/modules/audio_processing/aec3/matrix_buffer.cc index bd6daea95c..2fd71b417f 100644 --- a/modules/audio_processing/aec3/matrix_buffer.cc +++ b/modules/audio_processing/aec3/matrix_buffer.cc @@ -14,14 +14,22 @@ namespace webrtc { -MatrixBuffer::MatrixBuffer(size_t size, size_t height, size_t width) +MatrixBuffer::MatrixBuffer(size_t size, + size_t num_bands, + size_t num_channels, + size_t frame_length) : size(static_cast(size)), buffer(size, - std::vector>(height, - std::vector(width, 0.f))) { - for (auto& c : buffer) { - for (auto& b : c) { - std::fill(b.begin(), b.end(), 0.f); + std::vector>>( + num_bands, + std::vector>( + num_channels, + std::vector(frame_length, 0.f)))) { + for (auto& block : buffer) { + for (auto& band : block) { + for (auto& channel : band) { + std::fill(channel.begin(), channel.end(), 0.f); + } } } } diff --git a/modules/audio_processing/aec3/matrix_buffer.h b/modules/audio_processing/aec3/matrix_buffer.h index 8fb96d21c1..97736a3096 100644 --- a/modules/audio_processing/aec3/matrix_buffer.h +++ b/modules/audio_processing/aec3/matrix_buffer.h @@ -21,8 +21,12 @@ namespace webrtc { // Struct for bundling a circular buffer of two dimensional vector objects // together with the read and write indices. +// TODO(peah): Change name of this class to be more specific to what it does. struct MatrixBuffer { - MatrixBuffer(size_t size, size_t height, size_t width); + MatrixBuffer(size_t size, + size_t num_bands, + size_t num_channels, + size_t frame_length); ~MatrixBuffer(); int IncIndex(int index) const { @@ -49,7 +53,7 @@ struct MatrixBuffer { void DecReadIndex() { read = DecIndex(read); } const int size; - std::vector>> buffer; + std::vector>>> buffer; int write = 0; int read = 0; }; diff --git a/modules/audio_processing/aec3/mock/mock_block_processor.h b/modules/audio_processing/aec3/mock/mock_block_processor.h index 85b88f7fc7..cb93714513 100644 --- a/modules/audio_processing/aec3/mock/mock_block_processor.h +++ b/modules/audio_processing/aec3/mock/mock_block_processor.h @@ -24,12 +24,13 @@ class MockBlockProcessor : public BlockProcessor { MockBlockProcessor(); virtual ~MockBlockProcessor(); - MOCK_METHOD3(ProcessCapture, - void(bool level_change, - bool saturated_microphone_signal, - std::vector>* capture_block)); + MOCK_METHOD3( + ProcessCapture, + void(bool level_change, + bool saturated_microphone_signal, + std::vector>>* capture_block)); MOCK_METHOD1(BufferRender, - void(const std::vector>& block)); + void(const std::vector>>& block)); MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics)); MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); diff --git a/modules/audio_processing/aec3/mock/mock_echo_remover.h b/modules/audio_processing/aec3/mock/mock_echo_remover.h index 5faea26513..f8dd348975 100644 --- a/modules/audio_processing/aec3/mock/mock_echo_remover.h +++ b/modules/audio_processing/aec3/mock/mock_echo_remover.h @@ -32,7 +32,7 @@ class MockEchoRemover : public EchoRemover { bool capture_signal_saturation, const absl::optional& delay_estimate, RenderBuffer* render_buffer, - std::vector>* capture)); + std::vector>>* capture)); MOCK_CONST_METHOD0(Delay, absl::optional()); MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics)); diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc index 75262354c7..de87000128 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc +++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc @@ -13,9 +13,11 @@ namespace webrtc { namespace test { -MockRenderDelayBuffer::MockRenderDelayBuffer(int sample_rate_hz) +MockRenderDelayBuffer::MockRenderDelayBuffer(int sample_rate_hz, + size_t num_channels) : block_buffer_(GetRenderDelayBufferSize(4, 4, 12), NumBandsForRate(sample_rate_hz), + num_channels, kBlockSize), spectrum_buffer_(block_buffer_.buffer.size(), kFftLengthBy2Plus1), fft_buffer_(block_buffer_.buffer.size()), diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h index 0dd1b9126c..1ad07278cd 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h @@ -24,13 +24,13 @@ namespace test { class MockRenderDelayBuffer : public RenderDelayBuffer { public: - explicit MockRenderDelayBuffer(int sample_rate_hz); + MockRenderDelayBuffer(int sample_rate_hz, size_t num_channels); virtual ~MockRenderDelayBuffer(); MOCK_METHOD0(Reset, void()); MOCK_METHOD1(Insert, RenderDelayBuffer::BufferingEvent( - const std::vector>& block)); + const std::vector>>& block)); MOCK_METHOD0(PrepareCaptureProcessing, RenderDelayBuffer::BufferingEvent()); MOCK_METHOD1(AlignFromDelay, bool(size_t delay)); MOCK_METHOD0(AlignFromExternalDelay, void()); diff --git a/modules/audio_processing/aec3/render_buffer.h b/modules/audio_processing/aec3/render_buffer.h index 762eab802c..8759760ada 100644 --- a/modules/audio_processing/aec3/render_buffer.h +++ b/modules/audio_processing/aec3/render_buffer.h @@ -36,7 +36,8 @@ class RenderBuffer { ~RenderBuffer(); // Get a block. - const std::vector>& Block(int buffer_offset_blocks) const { + const std::vector>>& Block( + int buffer_offset_blocks) const { int position = block_buffer_->OffsetIndex(block_buffer_->read, buffer_offset_blocks); return block_buffer_->buffer[position]; diff --git a/modules/audio_processing/aec3/render_buffer_unittest.cc b/modules/audio_processing/aec3/render_buffer_unittest.cc index fadd600b87..4437178305 100644 --- a/modules/audio_processing/aec3/render_buffer_unittest.cc +++ b/modules/audio_processing/aec3/render_buffer_unittest.cc @@ -22,7 +22,7 @@ namespace webrtc { // Verifies the check for non-null fft buffer. TEST(RenderBuffer, NullExternalFftBuffer) { - MatrixBuffer block_buffer(10, 3, kBlockSize); + MatrixBuffer block_buffer(10, 3, 1, kBlockSize); VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1); EXPECT_DEATH(RenderBuffer(&block_buffer, &spectrum_buffer, nullptr), ""); } @@ -30,7 +30,7 @@ TEST(RenderBuffer, NullExternalFftBuffer) { // Verifies the check for non-null spectrum buffer. TEST(RenderBuffer, NullExternalSpectrumBuffer) { FftBuffer fft_buffer(10); - MatrixBuffer block_buffer(10, 3, kBlockSize); + MatrixBuffer block_buffer(10, 3, 1, kBlockSize); EXPECT_DEATH(RenderBuffer(&block_buffer, nullptr, &fft_buffer), ""); } diff --git a/modules/audio_processing/aec3/render_delay_buffer.cc b/modules/audio_processing/aec3/render_delay_buffer.cc index 11fe4507cf..379f5a1eb0 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.cc +++ b/modules/audio_processing/aec3/render_delay_buffer.cc @@ -39,12 +39,15 @@ namespace { class RenderDelayBufferImpl final : public RenderDelayBuffer { public: - RenderDelayBufferImpl(const EchoCanceller3Config& config, int sample_rate_hz); + RenderDelayBufferImpl(const EchoCanceller3Config& config, + int sample_rate_hz, + size_t num_render_channels); RenderDelayBufferImpl() = delete; ~RenderDelayBufferImpl() override; void Reset() override; - BufferingEvent Insert(const std::vector>& block) override; + BufferingEvent Insert( + const std::vector>>& block) override; BufferingEvent PrepareCaptureProcessing() override; bool AlignFromDelay(size_t delay) override; void AlignFromExternalDelay() override; @@ -90,12 +93,11 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { bool external_audio_buffer_delay_verified_after_reset_ = false; size_t min_latency_blocks_ = 0; size_t excess_render_detection_counter_ = 0; - int sample_rate_hz_; int MapDelayToTotalDelay(size_t delay) const; int ComputeDelay() const; void ApplyTotalDelay(int delay); - void InsertBlock(const std::vector>& block, + void InsertBlock(const std::vector>>& block, int previous_write); bool DetectActiveRender(rtc::ArrayView x) const; bool DetectExcessRenderBlocks(); @@ -109,7 +111,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { int RenderDelayBufferImpl::instance_count_ = 0; RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, - int sample_rate_hz) + int sample_rate_hz, + size_t num_render_channels) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), optimization_(DetectOptimization()), @@ -122,6 +125,7 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, config.delay.num_filters, config.filter.main.length_blocks), NumBandsForRate(sample_rate_hz), + num_render_channels, kBlockSize), spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1), ffts_(blocks_.buffer.size()), @@ -132,9 +136,7 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, render_decimator_(down_sampling_factor_), fft_(), render_ds_(sub_block_size_, 0.f), - buffer_headroom_(config.filter.main.length_blocks), - sample_rate_hz_(sample_rate_hz) { - RTC_DCHECK_GE(sample_rate_hz, 8000); + buffer_headroom_(config.filter.main.length_blocks) { RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size()); RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size()); @@ -184,7 +186,7 @@ void RenderDelayBufferImpl::Reset() { // Inserts a new block into the render buffers. RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert( - const std::vector>& block) { + const std::vector>>& block) { ++render_call_counter_; if (delay_) { if (!last_call_was_render_) { @@ -212,7 +214,7 @@ RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert( // Detect and update render activity. if (!render_activity_) { - render_activity_counter_ += DetectActiveRender(block[0]) ? 1 : 0; + render_activity_counter_ += DetectActiveRender(block[0][0]) ? 1 : 0; render_activity_ = render_activity_counter_ >= 20; } @@ -315,8 +317,7 @@ void RenderDelayBufferImpl::SetAudioBufferDelay(size_t delay_ms) { } // Convert delay from milliseconds to blocks (rounded down). - external_audio_buffer_delay_ = - delay_ms >> ((sample_rate_hz_ == 8000) ? 1 : 2); + external_audio_buffer_delay_ = delay_ms >> 2; } bool RenderDelayBufferImpl::HasReceivedBufferDelay() { @@ -359,7 +360,7 @@ void RenderDelayBufferImpl::AlignFromExternalDelay() { // Inserts a block into the render buffers. void RenderDelayBufferImpl::InsertBlock( - const std::vector>& block, + const std::vector>>& block, int previous_write) { auto& b = blocks_; auto& lr = low_rate_; @@ -372,13 +373,14 @@ void RenderDelayBufferImpl::InsertBlock( std::copy(block[k].begin(), block[k].end(), b.buffer[b.write][k].begin()); } - data_dumper_->DumpWav("aec3_render_decimator_input", block[0].size(), - block[0].data(), 16000, 1); - render_decimator_.Decimate(block[0], ds); + data_dumper_->DumpWav("aec3_render_decimator_input", block[0][0].size(), + block[0][0].data(), 16000, 1); + render_decimator_.Decimate(block[0][0], ds); data_dumper_->DumpWav("aec3_render_decimator_output", ds.size(), ds.data(), 16000 / down_sampling_factor_, 1); std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write); - fft_.PaddedFft(block[0], b.buffer[previous_write][0], &f.buffer[f.write]); + fft_.PaddedFft(block[0][0], b.buffer[previous_write][0][0], + &f.buffer[f.write]); f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write]); } @@ -457,8 +459,9 @@ bool RenderDelayBufferImpl::RenderUnderrun() { } // namespace RenderDelayBuffer* RenderDelayBuffer::Create(const EchoCanceller3Config& config, - int sample_rate_hz) { - return new RenderDelayBufferImpl(config, sample_rate_hz); + int sample_rate_hz, + size_t num_render_channels) { + return new RenderDelayBufferImpl(config, sample_rate_hz, num_render_channels); } } // namespace webrtc diff --git a/modules/audio_processing/aec3/render_delay_buffer.h b/modules/audio_processing/aec3/render_delay_buffer.h index 562d2c110d..e53f6d273d 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.h +++ b/modules/audio_processing/aec3/render_delay_buffer.h @@ -33,7 +33,8 @@ class RenderDelayBuffer { }; static RenderDelayBuffer* Create(const EchoCanceller3Config& config, - int sample_rate_hz); + int sample_rate_hz, + size_t num_render_channels); virtual ~RenderDelayBuffer() = default; // Resets the buffer alignment. @@ -41,7 +42,7 @@ class RenderDelayBuffer { // Inserts a block into the buffer. virtual BufferingEvent Insert( - const std::vector>& block) = 0; + const std::vector>>& block) = 0; // Updates the buffers one step based on the specified buffer delay. Returns // an enum indicating whether there was a special event that occurred. diff --git a/modules/audio_processing/aec3/render_delay_buffer_unittest.cc b/modules/audio_processing/aec3/render_delay_buffer_unittest.cc index 143980ca42..35e81319cf 100644 --- a/modules/audio_processing/aec3/render_delay_buffer_unittest.cc +++ b/modules/audio_processing/aec3/render_delay_buffer_unittest.cc @@ -35,36 +35,43 @@ std::string ProduceDebugText(int sample_rate_hz) { // Verifies that the buffer overflow is correctly reported. TEST(RenderDelayBuffer, BufferOverflow) { const EchoCanceller3Config config; - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - std::unique_ptr delay_buffer( - RenderDelayBuffer::Create(config, rate)); - std::vector> block_to_insert( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); - for (size_t k = 0; k < 10; ++k) { - EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kNone, - delay_buffer->Insert(block_to_insert)); - } - bool overrun_occurred = false; - for (size_t k = 0; k < 1000; ++k) { - RenderDelayBuffer::BufferingEvent event = - delay_buffer->Insert(block_to_insert); - overrun_occurred = - overrun_occurred || - RenderDelayBuffer::BufferingEvent::kRenderOverrun == event; - } + for (auto num_channels : {1, 2, 8}) { + for (auto rate : {16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(config, rate, num_channels)); + std::vector>> block_to_insert( + NumBandsForRate(rate), + std::vector>(num_channels, + std::vector(kBlockSize, 0.f))); + for (size_t k = 0; k < 10; ++k) { + EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kNone, + delay_buffer->Insert(block_to_insert)); + } + bool overrun_occurred = false; + for (size_t k = 0; k < 1000; ++k) { + RenderDelayBuffer::BufferingEvent event = + delay_buffer->Insert(block_to_insert); + overrun_occurred = + overrun_occurred || + RenderDelayBuffer::BufferingEvent::kRenderOverrun == event; + } - EXPECT_TRUE(overrun_occurred); + EXPECT_TRUE(overrun_occurred); + } } } // Verifies that the check for available block works. TEST(RenderDelayBuffer, AvailableBlock) { - constexpr size_t kNumBands = 1; - std::unique_ptr delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 16000)); - std::vector> input_block( - kNumBands, std::vector(kBlockSize, 1.f)); + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + EchoCanceller3Config(), kSampleRateHz, kNumChannels)); + std::vector>> input_block( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 1.f))); EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kNone, delay_buffer->Insert(input_block)); delay_buffer->PrepareCaptureProcessing(); @@ -74,7 +81,7 @@ TEST(RenderDelayBuffer, AvailableBlock) { TEST(RenderDelayBuffer, AlignFromDelay) { EchoCanceller3Config config; std::unique_ptr delay_buffer( - RenderDelayBuffer::Create(config, 16000)); + RenderDelayBuffer::Create(config, 16000, 1)); ASSERT_TRUE(delay_buffer->Delay()); delay_buffer->Reset(); size_t initial_internal_delay = 0; @@ -92,32 +99,55 @@ TEST(RenderDelayBuffer, AlignFromDelay) { // tests on test bots has been fixed. TEST(RenderDelayBuffer, DISABLED_WrongDelay) { std::unique_ptr delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 48000)); + RenderDelayBuffer::Create(EchoCanceller3Config(), 48000, 1)); EXPECT_DEATH(delay_buffer->AlignFromDelay(21), ""); } // Verifies the check for the number of bands in the inserted blocks. TEST(RenderDelayBuffer, WrongNumberOfBands) { for (auto rate : {16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - std::unique_ptr delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), rate)); - std::vector> block_to_insert( - NumBandsForRate(rate < 48000 ? rate + 16000 : 16000), - std::vector(kBlockSize, 0.f)); - EXPECT_DEATH(delay_buffer->Insert(block_to_insert), ""); + for (size_t num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + EchoCanceller3Config(), rate, num_channels)); + std::vector>> block_to_insert( + NumBandsForRate(rate < 48000 ? rate + 16000 : 16000), + std::vector>(num_channels, + std::vector(kBlockSize, 0.f))); + EXPECT_DEATH(delay_buffer->Insert(block_to_insert), ""); + } + } +} + +// Verifies the check for the number of channels in the inserted blocks. +TEST(RenderDelayBuffer, WrongNumberOfChannels) { + for (auto rate : {16000, 32000, 48000}) { + for (size_t num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + EchoCanceller3Config(), rate, num_channels)); + std::vector>> block_to_insert( + NumBandsForRate(rate), + std::vector>(num_channels + 1, + std::vector(kBlockSize, 0.f))); + EXPECT_DEATH(delay_buffer->Insert(block_to_insert), ""); + } } } // Verifies the check of the length of the inserted blocks. TEST(RenderDelayBuffer, WrongBlockLength) { - for (auto rate : {8000, 16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - std::unique_ptr delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 48000)); - std::vector> block_to_insert( - NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); - EXPECT_DEATH(delay_buffer->Insert(block_to_insert), ""); + for (auto rate : {16000, 32000, 48000}) { + for (size_t num_channels : {1, 2, 8}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + EchoCanceller3Config(), rate, num_channels)); + std::vector>> block_to_insert( + NumBandsForRate(rate), + std::vector>( + num_channels, std::vector(kBlockSize - 1, 0.f))); + EXPECT_DEATH(delay_buffer->Insert(block_to_insert), ""); + } } } diff --git a/modules/audio_processing/aec3/render_delay_controller_unittest.cc b/modules/audio_processing/aec3/render_delay_controller_unittest.cc index ff3fb7bc1e..995ecc9849 100644 --- a/modules/audio_processing/aec3/render_delay_controller_unittest.cc +++ b/modules/audio_processing/aec3/render_delay_controller_unittest.cc @@ -53,10 +53,10 @@ TEST(RenderDelayController, NoRenderSignal) { for (auto down_sampling_factor : kDownSamplingFactors) { config.delay.down_sampling_factor = down_sampling_factor; config.delay.num_filters = num_matched_filters; - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr delay_buffer( - RenderDelayBuffer::Create(config, rate)); + RenderDelayBuffer::Create(config, rate, 1)); std::unique_ptr delay_controller( RenderDelayController::Create(config, rate)); for (size_t k = 0; k < 100; ++k) { @@ -72,6 +72,7 @@ TEST(RenderDelayController, NoRenderSignal) { // Verifies the basic API call sequence. TEST(RenderDelayController, BasicApiCalls) { + constexpr size_t kNumChannels = 1; std::vector capture_block(kBlockSize, 0.f); absl::optional delay_blocks; for (size_t num_matched_filters = 4; num_matched_filters == 10; @@ -80,11 +81,13 @@ TEST(RenderDelayController, BasicApiCalls) { EchoCanceller3Config config; config.delay.down_sampling_factor = down_sampling_factor; config.delay.num_filters = num_matched_filters; - for (auto rate : {8000, 16000, 32000, 48000}) { - std::vector> render_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (auto rate : {16000, 32000, 48000}) { + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate)); + RenderDelayBuffer::Create(config, rate, kNumChannels)); std::unique_ptr delay_controller( RenderDelayController::Create(EchoCanceller3Config(), rate)); for (size_t k = 0; k < 10; ++k) { @@ -114,35 +117,45 @@ TEST(RenderDelayController, Alignment) { config.delay.down_sampling_factor = down_sampling_factor; config.delay.num_filters = num_matched_filters; - for (auto rate : {8000, 16000, 32000, 48000}) { - std::vector> render_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (size_t num_render_channels : {1, 2}) { + for (auto rate : {16000, 32000, 48000}) { + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>( + num_render_channels, std::vector(kBlockSize, 0.f))); - for (size_t delay_samples : {15, 50, 150, 200, 800, 4000}) { - absl::optional delay_blocks; - SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); - std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate)); - std::unique_ptr delay_controller( - RenderDelayController::Create(config, rate)); - DelayBuffer signal_delay_buffer(delay_samples); - for (size_t k = 0; k < (400 + delay_samples / kBlockSize); ++k) { - RandomizeSampleVector(&random_generator, render_block[0]); - signal_delay_buffer.Delay(render_block[0], capture_block); - render_delay_buffer->Insert(render_block); - render_delay_buffer->PrepareCaptureProcessing(); - delay_blocks = delay_controller->GetDelay( - render_delay_buffer->GetDownsampledRenderBuffer(), - render_delay_buffer->Delay(), capture_block); + for (size_t delay_samples : {15, 50, 150, 200, 800, 4000}) { + absl::optional delay_blocks; + SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, rate, num_render_channels)); + std::unique_ptr delay_controller( + RenderDelayController::Create(config, rate)); + DelayBuffer signal_delay_buffer(delay_samples); + for (size_t k = 0; k < (400 + delay_samples / kBlockSize); ++k) { + for (size_t band = 0; band < render_block.size(); ++band) { + for (size_t channel = 0; channel < render_block[band].size(); + ++channel) { + RandomizeSampleVector(&random_generator, + render_block[band][channel]); + } + } + signal_delay_buffer.Delay(render_block[0][0], capture_block); + render_delay_buffer->Insert(render_block); + render_delay_buffer->PrepareCaptureProcessing(); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), + render_delay_buffer->Delay(), capture_block); + } + ASSERT_TRUE(!!delay_blocks); + + constexpr int kDelayHeadroomBlocks = 1; + size_t expected_delay_blocks = + std::max(0, static_cast(delay_samples / kBlockSize) - + kDelayHeadroomBlocks); + + EXPECT_EQ(expected_delay_blocks, delay_blocks->delay); } - ASSERT_TRUE(!!delay_blocks); - - constexpr int kDelayHeadroomBlocks = 1; - size_t expected_delay_blocks = - std::max(0, static_cast(delay_samples / kBlockSize) - - kDelayHeadroomBlocks); - - EXPECT_EQ(expected_delay_blocks, delay_blocks->delay); } } } @@ -153,35 +166,41 @@ TEST(RenderDelayController, Alignment) { // delays. TEST(RenderDelayController, NonCausalAlignment) { Random random_generator(42U); + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; for (size_t num_matched_filters = 4; num_matched_filters == 10; num_matched_filters++) { for (auto down_sampling_factor : kDownSamplingFactors) { EchoCanceller3Config config; config.delay.down_sampling_factor = down_sampling_factor; config.delay.num_filters = num_matched_filters; - for (auto rate : {8000, 16000, 32000, 48000}) { - std::vector> render_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); - std::vector> capture_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (auto rate : {16000, 32000, 48000}) { + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>( + kNumRenderChannels, std::vector(kBlockSize, 0.f))); + std::vector>> capture_block( + NumBandsForRate(rate), + std::vector>( + kNumCaptureChannels, std::vector(kBlockSize, 0.f))); for (int delay_samples : {-15, -50, -150, -200}) { absl::optional delay_blocks; SCOPED_TRACE(ProduceDebugText(rate, -delay_samples)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate)); + RenderDelayBuffer::Create(config, rate, kNumRenderChannels)); std::unique_ptr delay_controller( RenderDelayController::Create(EchoCanceller3Config(), rate)); DelayBuffer signal_delay_buffer(-delay_samples); for (int k = 0; k < (400 - delay_samples / static_cast(kBlockSize)); ++k) { - RandomizeSampleVector(&random_generator, capture_block[0]); - signal_delay_buffer.Delay(capture_block[0], render_block[0]); + RandomizeSampleVector(&random_generator, capture_block[0][0]); + signal_delay_buffer.Delay(capture_block[0][0], render_block[0][0]); render_delay_buffer->Insert(render_block); render_delay_buffer->PrepareCaptureProcessing(); delay_blocks = delay_controller->GetDelay( render_delay_buffer->GetDownsampledRenderBuffer(), - render_delay_buffer->Delay(), capture_block[0]); + render_delay_buffer->Delay(), capture_block[0][0]); } ASSERT_FALSE(delay_blocks); @@ -195,6 +214,7 @@ TEST(RenderDelayController, NonCausalAlignment) { // simple timeshifts between the signals when there is jitter in the API calls. TEST(RenderDelayController, AlignmentWithJitter) { Random random_generator(42U); + constexpr size_t kNumRenderChannels = 1; std::vector capture_block(kBlockSize, 0.f); for (size_t num_matched_filters = 4; num_matched_filters == 10; num_matched_filters++) { @@ -202,14 +222,16 @@ TEST(RenderDelayController, AlignmentWithJitter) { EchoCanceller3Config config; config.delay.down_sampling_factor = down_sampling_factor; config.delay.num_filters = num_matched_filters; - for (auto rate : {8000, 16000, 32000, 48000}) { - std::vector> render_block( - NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (auto rate : {16000, 32000, 48000}) { + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>( + kNumRenderChannels, std::vector(kBlockSize, 0.f))); for (size_t delay_samples : {15, 50, 300, 800}) { absl::optional delay_blocks; SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate)); + RenderDelayBuffer::Create(config, rate, kNumRenderChannels)); std::unique_ptr delay_controller( RenderDelayController::Create(config, rate)); DelayBuffer signal_delay_buffer(delay_samples); @@ -220,8 +242,8 @@ TEST(RenderDelayController, AlignmentWithJitter) { ++j) { std::vector> capture_block_buffer; for (size_t k = 0; k < (kMaxTestJitterBlocks - 1); ++k) { - RandomizeSampleVector(&random_generator, render_block[0]); - signal_delay_buffer.Delay(render_block[0], capture_block); + RandomizeSampleVector(&random_generator, render_block[0][0]); + signal_delay_buffer.Delay(render_block[0][0], capture_block); capture_block_buffer.push_back(capture_block); render_delay_buffer->Insert(render_block); } @@ -259,10 +281,10 @@ TEST(RenderDelayController, InitialHeadroom) { EchoCanceller3Config config; config.delay.down_sampling_factor = down_sampling_factor; config.delay.num_filters = num_matched_filters; - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate)); + RenderDelayBuffer::Create(config, rate, 1)); std::unique_ptr delay_controller( RenderDelayController::Create(config, rate)); @@ -277,10 +299,10 @@ TEST(RenderDelayController, InitialHeadroom) { TEST(RenderDelayController, WrongCaptureSize) { std::vector block(kBlockSize - 1, 0.f); EchoCanceller3Config config; - for (auto rate : {8000, 16000, 32000, 48000}) { + for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate)); + RenderDelayBuffer::Create(config, rate, 1)); EXPECT_DEATH( std::unique_ptr( RenderDelayController::Create(EchoCanceller3Config(), rate)) @@ -298,7 +320,7 @@ TEST(RenderDelayController, DISABLED_WrongSampleRate) { SCOPED_TRACE(ProduceDebugText(rate)); EchoCanceller3Config config; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate)); + RenderDelayBuffer::Create(config, rate, 1)); EXPECT_DEATH( std::unique_ptr( RenderDelayController::Create(EchoCanceller3Config(), rate)), diff --git a/modules/audio_processing/aec3/render_signal_analyzer.cc b/modules/audio_processing/aec3/render_signal_analyzer.cc index e3e41a77a8..88bacaf79a 100644 --- a/modules/audio_processing/aec3/render_signal_analyzer.cc +++ b/modules/audio_processing/aec3/render_signal_analyzer.cc @@ -66,13 +66,15 @@ void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer, } // Assess the render signal strength. - const std::vector>& x_latest = render_buffer.Block(0); - auto result0 = std::minmax_element(x_latest[0].begin(), x_latest[0].end()); + const std::vector>>& x_latest = + render_buffer.Block(0); + auto result0 = + std::minmax_element(x_latest[0][0].begin(), x_latest[0][0].end()); float max_abs = std::max(fabs(*result0.first), fabs(*result0.second)); if (x_latest.size() > 1) { const auto result1 = - std::minmax_element(x_latest[1].begin(), x_latest[1].end()); + std::minmax_element(x_latest[1][0].begin(), x_latest[1][0].end()); max_abs = std::max(max_abs, static_cast(std::max(fabs(*result1.first), fabs(*result1.second)))); diff --git a/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc b/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc index 53a41b1768..27a31f0be7 100644 --- a/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc +++ b/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc @@ -33,14 +33,23 @@ constexpr float kPi = 3.141592f; void ProduceSinusoid(int sample_rate_hz, float sinusoidal_frequency_hz, size_t* sample_counter, - rtc::ArrayView x) { + std::vector>>* x) { // Produce a sinusoid of the specified frequency. for (size_t k = *sample_counter, j = 0; k < (*sample_counter + kBlockSize); ++k, ++j) { - x[j] = 32767.f * - std::sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz); + for (size_t channel = 0; channel < (*x)[0].size(); ++channel) { + (*x)[0][channel][j] = + 32767.f * + std::sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz); + } } *sample_counter = *sample_counter + kBlockSize; + + for (size_t band = 1; band < x->size(); ++band) { + for (size_t channel = 0; channel < (*x)[band].size(); ++channel) { + std::fill((*x)[band][channel].begin(), (*x)[band][channel].end(), 0.f); + } + } } } // namespace @@ -58,15 +67,17 @@ TEST(RenderSignalAnalyzer, NullMaskOutput) { TEST(RenderSignalAnalyzer, NoFalseDetectionOfNarrowBands) { RenderSignalAnalyzer analyzer(EchoCanceller3Config{}); Random random_generator(42U); - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + 3, + std::vector>(1, std::vector(kBlockSize, 0.f))); std::array x_old; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(EchoCanceller3Config(), 48000)); + RenderDelayBuffer::Create(EchoCanceller3Config(), 48000, 1)); std::array mask; x_old.fill(0.f); for (size_t k = 0; k < 100; ++k) { - RandomizeSampleVector(&random_generator, x[0]); + RandomizeSampleVector(&random_generator, x[0][0]); render_delay_buffer->Insert(x); if (k == 0) { @@ -89,12 +100,17 @@ TEST(RenderSignalAnalyzer, NoFalseDetectionOfNarrowBands) { TEST(RenderSignalAnalyzer, NarrowBandDetection) { RenderSignalAnalyzer analyzer(EchoCanceller3Config{}); Random random_generator(42U); - std::vector> x(3, std::vector(kBlockSize, 0.f)); + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + std::vector>> x( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::array x_old; Aec3Fft fft; EchoCanceller3Config config; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); std::array mask; x_old.fill(0.f); @@ -104,7 +120,7 @@ TEST(RenderSignalAnalyzer, NarrowBandDetection) { size_t sample_counter = 0; for (size_t k = 0; k < 100; ++k) { ProduceSinusoid(16000, 16000 / 2 * kSinusFrequencyBin / kFftLengthBy2, - &sample_counter, x[0]); + &sample_counter, &x); render_delay_buffer->Insert(x); if (k == 0) { diff --git a/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc b/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc index d277d42f3e..863f8f8ae7 100644 --- a/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc +++ b/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc @@ -27,7 +27,7 @@ TEST(ResidualEchoEstimator, NullResidualEchoPowerOutput) { EchoCanceller3Config config; AecState aec_state(config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, 48000, 1)); std::vector> H2; std::array S2_linear; std::array Y2; @@ -42,12 +42,16 @@ TEST(ResidualEchoEstimator, NullResidualEchoPowerOutput) { // TODO(peah): This test is broken in the sense that it not at all tests what it // seems to test. Enable the test once that is adressed. TEST(ResidualEchoEstimator, DISABLED_BasicTest) { + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + EchoCanceller3Config config; config.ep_strength.default_len = 0.f; ResidualEchoEstimator estimator(config); AecState aec_state(config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); std::array E2_main; std::array E2_shadow; @@ -57,7 +61,9 @@ TEST(ResidualEchoEstimator, DISABLED_BasicTest) { std::array R2; EchoPathVariability echo_path_variability( false, EchoPathVariability::DelayAdjustment::kNone, false); - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::vector> H2(10); Random random_generator(42U); SubtractorOutput output; @@ -86,8 +92,8 @@ TEST(ResidualEchoEstimator, DISABLED_BasicTest) { Y2.fill(kLevel); for (int k = 0; k < 1993; ++k) { - RandomizeSampleVector(&random_generator, x[0]); - std::for_each(x[0].begin(), x[0].end(), [](float& a) { a /= 30.f; }); + RandomizeSampleVector(&random_generator, x[0][0]); + std::for_each(x[0][0].begin(), x[0][0].end(), [](float& a) { a /= 30.f; }); render_delay_buffer->Insert(x); if (k == 0) { render_delay_buffer->Reset(); diff --git a/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc index 7372e5eb28..b49b00d966 100644 --- a/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc +++ b/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc @@ -32,6 +32,7 @@ namespace { // gain functionality. void RunFilterUpdateTest(int num_blocks_to_process, size_t delay_samples, + size_t num_render_channels, int filter_length_blocks, const std::vector& blocks_with_saturation, std::array* e_last_block, @@ -50,17 +51,19 @@ void RunFilterUpdateTest(int num_blocks_to_process, DetectOptimization(), &data_dumper); Aec3Fft fft; + constexpr int kSampleRateHz = 48000; config.delay.default_delay = 1; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, num_render_channels)); - std::array x_old; - x_old.fill(0.f); ShadowFilterUpdateGain shadow_gain( config.filter.shadow, config.filter.config_change_duration_blocks); Random random_generator(42U); - std::vector> x(3, std::vector(kBlockSize, 0.f)); - std::vector y(kBlockSize, 0.f); + std::vector>> x( + NumBandsForRate(kSampleRateHz), + std::vector>(num_render_channels, + std::vector(kBlockSize, 0.f))); + std::array y; AecState aec_state(config); RenderSignalAnalyzer render_signal_analyzer(config); std::array s; @@ -79,8 +82,12 @@ void RunFilterUpdateTest(int num_blocks_to_process, k) != blocks_with_saturation.end(); // Create the render signal. - RandomizeSampleVector(&random_generator, x[0]); - delay_buffer.Delay(x[0], y); + for (size_t band = 0; band < x.size(); ++band) { + for (size_t channel = 0; channel < x[band].size(); ++channel) { + RandomizeSampleVector(&random_generator, x[band][channel]); + } + } + delay_buffer.Delay(x[0][0], y); render_delay_buffer->Insert(x); if (k == 0) { @@ -151,25 +158,30 @@ TEST(ShadowFilterUpdateGain, NullDataOutputGain) { TEST(ShadowFilterUpdateGain, GainCausesFilterToConverge) { std::vector blocks_with_echo_path_changes; std::vector blocks_with_saturation; - for (size_t filter_length_blocks : {12, 20, 30}) { - for (size_t delay_samples : {0, 64, 150, 200, 301}) { - SCOPED_TRACE(ProduceDebugText(delay_samples, filter_length_blocks)); - std::array e; - std::array y; - FftData G; + for (size_t num_render_channels : {1, 2, 8}) { + for (size_t filter_length_blocks : {12, 20, 30}) { + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + SCOPED_TRACE(ProduceDebugText(delay_samples, filter_length_blocks)); - RunFilterUpdateTest(1000, delay_samples, filter_length_blocks, - blocks_with_saturation, &e, &y, &G); + std::array e; + std::array y; + FftData G; - // Verify that the main filter is able to perform well. - // Use different criteria to take overmodelling into account. - if (filter_length_blocks == 12) { - EXPECT_LT(1000 * std::inner_product(e.begin(), e.end(), e.begin(), 0.f), - std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); - } else { - EXPECT_LT(std::inner_product(e.begin(), e.end(), e.begin(), 0.f), - std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); + RunFilterUpdateTest(1000, delay_samples, num_render_channels, + filter_length_blocks, blocks_with_saturation, &e, + &y, &G); + + // Verify that the main filter is able to perform well. + // Use different criteria to take overmodelling into account. + if (filter_length_blocks == 12) { + EXPECT_LT( + 1000 * std::inner_product(e.begin(), e.end(), e.begin(), 0.f), + std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); + } else { + EXPECT_LT(std::inner_product(e.begin(), e.end(), e.begin(), 0.f), + std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); + } } } } @@ -178,36 +190,38 @@ TEST(ShadowFilterUpdateGain, GainCausesFilterToConverge) { // Verifies that the magnitude of the gain on average decreases for a // persistently exciting signal. TEST(ShadowFilterUpdateGain, DecreasingGain) { - for (size_t filter_length_blocks : {12, 20, 30}) { - SCOPED_TRACE(ProduceDebugText(filter_length_blocks)); - std::vector blocks_with_echo_path_changes; - std::vector blocks_with_saturation; + for (size_t num_render_channels : {1, 2, 8}) { + for (size_t filter_length_blocks : {12, 20, 30}) { + SCOPED_TRACE(ProduceDebugText(filter_length_blocks)); + std::vector blocks_with_echo_path_changes; + std::vector blocks_with_saturation; - std::array e; - std::array y; - FftData G_a; - FftData G_b; - FftData G_c; - std::array G_a_power; - std::array G_b_power; - std::array G_c_power; + std::array e; + std::array y; + FftData G_a; + FftData G_b; + FftData G_c; + std::array G_a_power; + std::array G_b_power; + std::array G_c_power; - RunFilterUpdateTest(100, 65, filter_length_blocks, blocks_with_saturation, - &e, &y, &G_a); - RunFilterUpdateTest(200, 65, filter_length_blocks, blocks_with_saturation, - &e, &y, &G_b); - RunFilterUpdateTest(300, 65, filter_length_blocks, blocks_with_saturation, - &e, &y, &G_c); + RunFilterUpdateTest(100, 65, num_render_channels, filter_length_blocks, + blocks_with_saturation, &e, &y, &G_a); + RunFilterUpdateTest(200, 65, num_render_channels, filter_length_blocks, + blocks_with_saturation, &e, &y, &G_b); + RunFilterUpdateTest(300, 65, num_render_channels, filter_length_blocks, + blocks_with_saturation, &e, &y, &G_c); - G_a.Spectrum(Aec3Optimization::kNone, G_a_power); - G_b.Spectrum(Aec3Optimization::kNone, G_b_power); - G_c.Spectrum(Aec3Optimization::kNone, G_c_power); + G_a.Spectrum(Aec3Optimization::kNone, G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, G_b_power); + G_c.Spectrum(Aec3Optimization::kNone, G_c_power); - EXPECT_GT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), - std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); + EXPECT_GT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), + std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); - EXPECT_GT(std::accumulate(G_b_power.begin(), G_b_power.end(), 0.), - std::accumulate(G_c_power.begin(), G_c_power.end(), 0.)); + EXPECT_GT(std::accumulate(G_b_power.begin(), G_b_power.end(), 0.), + std::accumulate(G_c_power.begin(), G_c_power.end(), 0.)); + } } } @@ -218,21 +232,23 @@ TEST(ShadowFilterUpdateGain, SaturationBehavior) { for (int k = 99; k < 200; ++k) { blocks_with_saturation.push_back(k); } - for (size_t filter_length_blocks : {12, 20, 30}) { - SCOPED_TRACE(ProduceDebugText(filter_length_blocks)); + for (size_t num_render_channels : {1, 2, 8}) { + for (size_t filter_length_blocks : {12, 20, 30}) { + SCOPED_TRACE(ProduceDebugText(filter_length_blocks)); - std::array e; - std::array y; - FftData G_a; - FftData G_a_ref; - G_a_ref.re.fill(0.f); - G_a_ref.im.fill(0.f); + std::array e; + std::array y; + FftData G_a; + FftData G_a_ref; + G_a_ref.re.fill(0.f); + G_a_ref.im.fill(0.f); - RunFilterUpdateTest(100, 65, filter_length_blocks, blocks_with_saturation, - &e, &y, &G_a); + RunFilterUpdateTest(100, 65, num_render_channels, filter_length_blocks, + blocks_with_saturation, &e, &y, &G_a); - EXPECT_EQ(G_a_ref.re, G_a.re); - EXPECT_EQ(G_a_ref.im, G_a.im); + EXPECT_EQ(G_a_ref.re, G_a.re); + EXPECT_EQ(G_a_ref.im, G_a.im); + } } } diff --git a/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc b/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc index f27c9051d0..b8c83f767b 100644 --- a/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc +++ b/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc @@ -24,7 +24,7 @@ namespace webrtc { namespace { -void GetActiveFrame(rtc::ArrayView x) { +void GetActiveFrame(std::vector>>* x) { const std::array frame = { 7459.88, 17209.6, 17383, 20768.9, 16816.7, 18386.3, 4492.83, 9675.85, 6665.52, 14808.6, 9342.3, 7483.28, 19261.7, 4145.98, 1622.18, 13475.2, @@ -34,8 +34,12 @@ void GetActiveFrame(rtc::ArrayView x) { 11405, 15031.4, 14541.6, 19765.5, 18346.3, 19350.2, 3157.47, 18095.8, 1743.68, 21328.2, 19727.5, 7295.16, 10332.4, 11055.5, 20107.4, 14708.4, 12416.2, 16434, 2454.69, 9840.8, 6867.23, 1615.75, 6059.9, 8394.19}; - RTC_DCHECK_GE(x.size(), frame.size()); - std::copy(frame.begin(), frame.end(), x.begin()); + for (size_t band = 0; band < x->size(); ++band) { + for (size_t channel = 0; channel < (*x)[band].size(); ++channel) { + RTC_DCHECK_GE((*x)[band][channel].size(), frame.size()); + std::copy(frame.begin(), frame.end(), (*x)[band][channel].begin()); + } + } } class TestInputs { @@ -58,13 +62,15 @@ class TestInputs { std::array Y2_; std::array E2_; std::vector> H2_; - std::vector> x_; + std::vector>> x_; }; TestInputs::TestInputs(const EchoCanceller3Config& cfg) - : render_delay_buffer_(RenderDelayBuffer::Create(cfg, 16000)), + : render_delay_buffer_(RenderDelayBuffer::Create(cfg, 16000, 1)), H2_(cfg.filter.main.length_blocks), - x_(1, std::vector(kBlockSize, 0.f)) { + x_(1, + std::vector>(1, + std::vector(kBlockSize, 0.f))) { render_delay_buffer_->AlignFromDelay(4); render_buffer_ = render_delay_buffer_->GetRenderBuffer(); for (auto& H : H2_) { @@ -77,9 +83,9 @@ TestInputs::~TestInputs() = default; void TestInputs::Update() { if (n_ % 2 == 0) { - std::fill(x_[0].begin(), x_[0].end(), 0.f); + std::fill(x_[0][0].begin(), x_[0][0].end(), 0.f); } else { - GetActiveFrame(x_[0]); + GetActiveFrame(&x_); } render_delay_buffer_->Insert(x_); diff --git a/modules/audio_processing/aec3/subtractor_unittest.cc b/modules/audio_processing/aec3/subtractor_unittest.cc index bcf3b272f2..f29b44625d 100644 --- a/modules/audio_processing/aec3/subtractor_unittest.cc +++ b/modules/audio_processing/aec3/subtractor_unittest.cc @@ -31,19 +31,24 @@ float RunSubtractorTest(int num_blocks_to_process, bool uncorrelated_inputs, const std::vector& blocks_with_echo_path_changes) { ApmDataDumper data_dumper(42); + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); EchoCanceller3Config config; config.filter.main.length_blocks = main_filter_length_blocks; config.filter.shadow.length_blocks = shadow_filter_length_blocks; Subtractor subtractor(config, &data_dumper, DetectOptimization()); absl::optional delay_estimate; - std::vector> x(3, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); std::vector y(kBlockSize, 0.f); std::array x_old; SubtractorOutput output; config.delay.default_delay = 1; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); RenderSignalAnalyzer render_signal_analyzer(config); Random random_generator(42U); Aec3Fft fft; @@ -58,11 +63,11 @@ float RunSubtractorTest(int num_blocks_to_process, DelayBuffer delay_buffer(delay_samples); for (int k = 0; k < num_blocks_to_process; ++k) { - RandomizeSampleVector(&random_generator, x[0]); + RandomizeSampleVector(&random_generator, x[0][0]); if (uncorrelated_inputs) { RandomizeSampleVector(&random_generator, y); } else { - delay_buffer.Delay(x[0], y); + delay_buffer.Delay(x[0][0], y); } render_delay_buffer->Insert(x); if (k == 0) { @@ -126,7 +131,7 @@ TEST(Subtractor, DISABLED_NullOutput) { EchoCanceller3Config config; Subtractor subtractor(config, &data_dumper, DetectOptimization()); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, 48000, 1)); RenderSignalAnalyzer render_signal_analyzer(config); std::vector y(kBlockSize, 0.f); @@ -142,7 +147,7 @@ TEST(Subtractor, WrongCaptureSize) { EchoCanceller3Config config; Subtractor subtractor(config, &data_dumper, DetectOptimization()); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, 48000, 1)); RenderSignalAnalyzer render_signal_analyzer(config); std::vector y(kBlockSize - 1, 0.f); SubtractorOutput output; diff --git a/modules/audio_processing/aec3/suppression_filter.cc b/modules/audio_processing/aec3/suppression_filter.cc index 6fe296c219..6679a87789 100644 --- a/modules/audio_processing/aec3/suppression_filter.cc +++ b/modules/audio_processing/aec3/suppression_filter.cc @@ -79,7 +79,7 @@ void SuppressionFilter::ApplyGain( const std::array& suppression_gain, float high_bands_gain, const FftData& E_lowest_band, - std::vector>* e) { + std::vector>>* e) { RTC_DCHECK(e); RTC_DCHECK_EQ(e->size(), NumBandsForRate(sample_rate_hz_)); FftData E; @@ -111,14 +111,14 @@ void SuppressionFilter::ApplyGain( fft_.Ifft(E, &e_extended); std::transform(e_output_old_[0].begin(), e_output_old_[0].end(), - std::begin(kSqrtHanning) + kFftLengthBy2, (*e)[0].begin(), + std::begin(kSqrtHanning) + kFftLengthBy2, (*e)[0][0].begin(), [&](float a, float b) { return kIfftNormalization * a * b; }); std::transform(e_extended.begin(), e_extended.begin() + kFftLengthBy2, std::begin(kSqrtHanning), e_extended.begin(), [&](float a, float b) { return kIfftNormalization * a * b; }); - std::transform((*e)[0].begin(), (*e)[0].end(), e_extended.begin(), - (*e)[0].begin(), std::plus()); - std::for_each((*e)[0].begin(), (*e)[0].end(), [](float& x_k) { + std::transform((*e)[0][0].begin(), (*e)[0][0].end(), e_extended.begin(), + (*e)[0][0].begin(), std::plus()); + std::for_each((*e)[0][0].begin(), (*e)[0][0].end(), [](float& x_k) { x_k = rtc::SafeClamp(x_k, -32768.f, 32767.f); }); std::copy(e_extended.begin() + kFftLengthBy2, e_extended.begin() + kFftLength, @@ -140,8 +140,9 @@ void SuppressionFilter::ApplyGain( 0.4f * std::sqrt(1.f - high_bands_gain * high_bands_gain); std::transform( - (*e)[1].begin(), (*e)[1].end(), time_domain_high_band_noise.begin(), - (*e)[1].begin(), [&](float a, float b) { + (*e)[1][0].begin(), (*e)[1][0].end(), + time_domain_high_band_noise.begin(), (*e)[1][0].begin(), + [&](float a, float b) { return std::max( std::min(b * high_bands_noise_scaling + high_bands_gain * a, 32767.0f), @@ -150,16 +151,16 @@ void SuppressionFilter::ApplyGain( if (e->size() > 2) { RTC_DCHECK_EQ(3, e->size()); - std::for_each((*e)[2].begin(), (*e)[2].end(), [&](float& a) { + std::for_each((*e)[2][0].begin(), (*e)[2][0].end(), [&](float& a) { a = rtc::SafeClamp(a * high_bands_gain, -32768.f, 32767.f); }); } std::array tmp; for (size_t k = 1; k < e->size(); ++k) { - std::copy((*e)[k].begin(), (*e)[k].end(), tmp.begin()); + std::copy((*e)[k][0].begin(), (*e)[k][0].end(), tmp.begin()); std::copy(e_output_old_[k].begin(), e_output_old_[k].end(), - (*e)[k].begin()); + (*e)[k][0].begin()); std::copy(tmp.begin(), tmp.end(), e_output_old_[k].begin()); } } diff --git a/modules/audio_processing/aec3/suppression_filter.h b/modules/audio_processing/aec3/suppression_filter.h index 63569b101f..03b13c873b 100644 --- a/modules/audio_processing/aec3/suppression_filter.h +++ b/modules/audio_processing/aec3/suppression_filter.h @@ -31,7 +31,7 @@ class SuppressionFilter { const std::array& suppression_gain, float high_bands_gain, const FftData& E_lowest_band, - std::vector>* e); + std::vector>>* e); private: const Aec3Optimization optimization_; diff --git a/modules/audio_processing/aec3/suppression_filter_unittest.cc b/modules/audio_processing/aec3/suppression_filter_unittest.cc index 80d96ece0f..1e05a02cb8 100644 --- a/modules/audio_processing/aec3/suppression_filter_unittest.cc +++ b/modules/audio_processing/aec3/suppression_filter_unittest.cc @@ -26,14 +26,23 @@ constexpr float kPi = 3.141592f; void ProduceSinusoid(int sample_rate_hz, float sinusoidal_frequency_hz, size_t* sample_counter, - rtc::ArrayView x) { + std::vector>>* x) { // Produce a sinusoid of the specified frequency. for (size_t k = *sample_counter, j = 0; k < (*sample_counter + kBlockSize); ++k, ++j) { - x[j] = 32767.f * - std::sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz); + for (size_t channel = 0; channel < (*x)[0].size(); ++channel) { + (*x)[0][channel][j] = + 32767.f * + std::sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz); + } } *sample_counter = *sample_counter + kBlockSize; + + for (size_t band = 1; band < x->size(); ++band) { + for (size_t channel = 0; channel < (*x)[band].size(); ++channel) { + std::fill((*x)[band][channel].begin(), (*x)[band][channel].end(), 0.f); + } + } } } // namespace @@ -75,29 +84,41 @@ TEST(SuppressionFilter, ComfortNoiseInUnityGain) { cn_high_bands.re.fill(1.f); cn_high_bands.im.fill(1.f); - std::vector> e(3, std::vector(kBlockSize, 0.f)); - std::vector> e_ref = e; + std::vector>> e( + 3, + std::vector>(1, std::vector(kBlockSize, 0.f))); + std::vector>> e_ref = e; FftData E; - fft.PaddedFft(e[0], e_old_, Aec3Fft::Window::kSqrtHanning, &E); - std::copy(e[0].begin(), e[0].end(), e_old_.begin()); + fft.PaddedFft(e[0][0], e_old_, Aec3Fft::Window::kSqrtHanning, &E); + std::copy(e[0][0].begin(), e[0][0].end(), e_old_.begin()); filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e); - for (size_t k = 0; k < e.size(); ++k) { - EXPECT_EQ(e_ref[k], e[k]); + for (size_t band = 0; band < e.size(); ++band) { + for (size_t channel = 0; channel < e[band].size(); ++channel) { + for (size_t sample = 0; sample < e[band][channel].size(); ++sample) { + EXPECT_EQ(e_ref[band][channel][sample], e[band][channel][sample]); + } + } } } // Verifies that the suppressor is able to suppress a signal. TEST(SuppressionFilter, SignalSuppression) { - SuppressionFilter filter(Aec3Optimization::kNone, 48000); + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + constexpr size_t kNumChannels = 1; + + SuppressionFilter filter(Aec3Optimization::kNone, kSampleRateHz); FftData cn; FftData cn_high_bands; std::array e_old_; Aec3Fft fft; std::array gain; - std::vector> e(3, std::vector(kBlockSize, 0.f)); + std::vector>> e( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); e_old_.fill(0.f); gain.fill(1.f); @@ -113,18 +134,17 @@ TEST(SuppressionFilter, SignalSuppression) { float e0_input = 0.f; float e0_output = 0.f; for (size_t k = 0; k < 100; ++k) { - ProduceSinusoid(16000, 16000 * 40 / kFftLengthBy2 / 2, &sample_counter, - e[0]); - e0_input = - std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_input); + ProduceSinusoid(16000, 16000 * 40 / kFftLengthBy2 / 2, &sample_counter, &e); + e0_input = std::inner_product(e[0][0].begin(), e[0][0].end(), + e[0][0].begin(), e0_input); FftData E; - fft.PaddedFft(e[0], e_old_, Aec3Fft::Window::kSqrtHanning, &E); - std::copy(e[0].begin(), e[0].end(), e_old_.begin()); + fft.PaddedFft(e[0][0], e_old_, Aec3Fft::Window::kSqrtHanning, &E); + std::copy(e[0][0].begin(), e[0][0].end(), e_old_.begin()); filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e); - e0_output = - std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_output); + e0_output = std::inner_product(e[0][0].begin(), e[0][0].end(), + e[0][0].begin(), e0_output); } EXPECT_LT(e0_output, e0_input / 1000.f); @@ -133,13 +153,19 @@ TEST(SuppressionFilter, SignalSuppression) { // Verifies that the suppressor is able to pass through a desired signal while // applying suppressing for some frequencies. TEST(SuppressionFilter, SignalTransparency) { - SuppressionFilter filter(Aec3Optimization::kNone, 48000); + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + + SuppressionFilter filter(Aec3Optimization::kNone, kSampleRateHz); FftData cn; std::array e_old_; Aec3Fft fft; FftData cn_high_bands; std::array gain; - std::vector> e(3, std::vector(kBlockSize, 0.f)); + std::vector>> e( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); e_old_.fill(0.f); gain.fill(1.f); std::for_each(gain.begin() + 30, gain.end(), [](float& a) { a = 0.f; }); @@ -154,18 +180,17 @@ TEST(SuppressionFilter, SignalTransparency) { float e0_input = 0.f; float e0_output = 0.f; for (size_t k = 0; k < 100; ++k) { - ProduceSinusoid(16000, 16000 * 10 / kFftLengthBy2 / 2, &sample_counter, - e[0]); - e0_input = - std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_input); + ProduceSinusoid(16000, 16000 * 10 / kFftLengthBy2 / 2, &sample_counter, &e); + e0_input = std::inner_product(e[0][0].begin(), e[0][0].end(), + e[0][0].begin(), e0_input); FftData E; - fft.PaddedFft(e[0], e_old_, Aec3Fft::Window::kSqrtHanning, &E); - std::copy(e[0].begin(), e[0].end(), e_old_.begin()); + fft.PaddedFft(e[0][0], e_old_, Aec3Fft::Window::kSqrtHanning, &E); + std::copy(e[0][0].begin(), e[0][0].end(), e_old_.begin()); filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e); - e0_output = - std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_output); + e0_output = std::inner_product(e[0][0].begin(), e[0][0].end(), + e[0][0].begin(), e0_output); } EXPECT_LT(0.9f * e0_input, e0_output); @@ -173,13 +198,19 @@ TEST(SuppressionFilter, SignalTransparency) { // Verifies that the suppressor delay. TEST(SuppressionFilter, Delay) { - SuppressionFilter filter(Aec3Optimization::kNone, 48000); + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 48000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); + + SuppressionFilter filter(Aec3Optimization::kNone, kSampleRateHz); FftData cn; FftData cn_high_bands; std::array e_old_; Aec3Fft fft; std::array gain; - std::vector> e(3, std::vector(kBlockSize, 0.f)); + std::vector>> e( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); gain.fill(1.f); @@ -189,21 +220,26 @@ TEST(SuppressionFilter, Delay) { cn_high_bands.im.fill(0.f); for (size_t k = 0; k < 100; ++k) { - for (size_t j = 0; j < 3; ++j) { - for (size_t i = 0; i < kBlockSize; ++i) { - e[j][i] = k * kBlockSize + i; + for (size_t band = 0; band < kNumBands; ++band) { + for (size_t channel = 0; channel < kNumChannels; ++channel) { + for (size_t sample = 0; sample < kBlockSize; ++sample) { + e[band][channel][sample] = k * kBlockSize + sample + channel; + } } } FftData E; - fft.PaddedFft(e[0], e_old_, Aec3Fft::Window::kSqrtHanning, &E); - std::copy(e[0].begin(), e[0].end(), e_old_.begin()); + fft.PaddedFft(e[0][0], e_old_, Aec3Fft::Window::kSqrtHanning, &E); + std::copy(e[0][0].begin(), e[0][0].end(), e_old_.begin()); filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e); if (k > 2) { - for (size_t j = 0; j < 2; ++j) { - for (size_t i = 0; i < kBlockSize; ++i) { - EXPECT_NEAR(k * kBlockSize + i - kBlockSize, e[j][i], 0.01); + for (size_t band = 0; band < kNumBands; ++band) { + for (size_t channel = 0; channel < kNumChannels; ++channel) { + for (size_t sample = 0; sample < kBlockSize; ++sample) { + EXPECT_NEAR(k * kBlockSize + sample - kBlockSize + channel, + e[band][channel][sample], 0.01); + } } } } diff --git a/modules/audio_processing/aec3/suppression_gain.cc b/modules/audio_processing/aec3/suppression_gain.cc index 4831b7163f..89ebe0f4f3 100644 --- a/modules/audio_processing/aec3/suppression_gain.cc +++ b/modules/audio_processing/aec3/suppression_gain.cc @@ -108,7 +108,7 @@ float SuppressionGain::UpperBandsGain( const std::array& comfort_noise_spectrum, const absl::optional& narrow_peak_band, bool saturated_echo, - const std::vector>& render, + const std::vector>>& render, const std::array& low_band_gain) const { RTC_DCHECK_LT(0, render.size()); if (render.size() == 1) { @@ -131,12 +131,12 @@ float SuppressionGain::UpperBandsGain( // Compute the upper and lower band energies. const auto sum_of_squares = [](float a, float b) { return a + b * b; }; - const float low_band_energy = - std::accumulate(render[0].begin(), render[0].end(), 0.f, sum_of_squares); + const float low_band_energy = std::accumulate( + render[0][0].begin(), render[0][0].end(), 0.f, sum_of_squares); float high_band_energy = 0.f; for (size_t k = 1; k < render.size(); ++k) { - const float energy = std::accumulate(render[k].begin(), render[k].end(), - 0.f, sum_of_squares); + const float energy = std::accumulate( + render[k][0].begin(), render[k][0].end(), 0.f, sum_of_squares); high_band_energy = std::max(high_band_energy, energy); } @@ -317,7 +317,7 @@ void SuppressionGain::GetGain( const std::array& comfort_noise_spectrum, const RenderSignalAnalyzer& render_signal_analyzer, const AecState& aec_state, - const std::vector>& render, + const std::vector>>& render, float* high_bands_gain, std::array* low_band_gain) { RTC_DCHECK(high_bands_gain); @@ -366,10 +366,10 @@ void SuppressionGain::SetInitialState(bool state) { // Detects when the render signal can be considered to have low power and // consist of stationary noise. bool SuppressionGain::LowNoiseRenderDetector::Detect( - const std::vector>& render) { + const std::vector>>& render) { float x2_sum = 0.f; float x2_max = 0.f; - for (auto x_k : render[0]) { + for (auto x_k : render[0][0]) { const float x2 = x_k * x_k; x2_sum += x2; x2_max = std::max(x2_max, x2); diff --git a/modules/audio_processing/aec3/suppression_gain.h b/modules/audio_processing/aec3/suppression_gain.h index 2b34dbe46b..a583ef01a3 100644 --- a/modules/audio_processing/aec3/suppression_gain.h +++ b/modules/audio_processing/aec3/suppression_gain.h @@ -41,7 +41,7 @@ class SuppressionGain { const std::array& comfort_noise_spectrum, const RenderSignalAnalyzer& render_signal_analyzer, const AecState& aec_state, - const std::vector>& render, + const std::vector>>& render, float* high_bands_gain, std::array* low_band_gain); @@ -55,7 +55,7 @@ class SuppressionGain { const std::array& comfort_noise_spectrum, const absl::optional& narrow_peak_band, bool saturated_echo, - const std::vector>& render, + const std::vector>>& render, const std::array& low_band_gain) const; void GainToNoAudibleEcho( @@ -84,7 +84,7 @@ class SuppressionGain { class LowNoiseRenderDetector { public: - bool Detect(const std::vector>& render); + bool Detect(const std::vector>>& render); private: float average_power_ = 32768.f * 32768.f; diff --git a/modules/audio_processing/aec3/suppression_gain_unittest.cc b/modules/audio_processing/aec3/suppression_gain_unittest.cc index 7d305f89ea..331b903ebe 100644 --- a/modules/audio_processing/aec3/suppression_gain_unittest.cc +++ b/modules/audio_processing/aec3/suppression_gain_unittest.cc @@ -47,8 +47,9 @@ TEST(SuppressionGain, NullOutputGains) { SuppressionGain(EchoCanceller3Config{}, DetectOptimization(), 16000) .GetGain(E2, S2, R2, N2, RenderSignalAnalyzer((EchoCanceller3Config{})), aec_state, - std::vector>( - 3, std::vector(kBlockSize, 0.f)), + std::vector>>( + 3, std::vector>( + 1, std::vector(kBlockSize, 0.f))), &high_bands_gain, nullptr), ""); } @@ -57,8 +58,11 @@ TEST(SuppressionGain, NullOutputGains) { // Does a sanity check that the gains are correctly computed. TEST(SuppressionGain, BasicGainComputation) { + constexpr size_t kNumChannels = 1; + constexpr int kSampleRateHz = 16000; + constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); SuppressionGain suppression_gain(EchoCanceller3Config(), DetectOptimization(), - 16000); + kSampleRateHz); RenderSignalAnalyzer analyzer(EchoCanceller3Config{}); float high_bands_gain; std::array E2; @@ -69,13 +73,15 @@ TEST(SuppressionGain, BasicGainComputation) { std::array g; SubtractorOutput output; std::array y; - std::vector> x(1, std::vector(kBlockSize, 0.f)); + std::vector>> x( + kNumBands, std::vector>( + kNumChannels, std::vector(kBlockSize, 0.f))); EchoCanceller3Config config; AecState aec_state(config); ApmDataDumper data_dumper(42); Subtractor subtractor(config, &data_dumper, DetectOptimization()); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, 48000)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); absl::optional delay_estimate; // Ensure that a strong noise is detected to mask any echoes. diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index bc61b523b9..d639fd57f6 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -1849,7 +1849,8 @@ void AudioProcessingImpl::InitializeEchoController() { echo_control_factory_->Create(proc_sample_rate_hz()); } else { private_submodules_->echo_controller = absl::make_unique( - EchoCanceller3Config(), proc_sample_rate_hz()); + EchoCanceller3Config(), proc_sample_rate_hz(), + /*num_render_channels=*/1, /*num_capture_channels=*/1); } capture_nonlocked_.echo_controller_enabled = true; diff --git a/modules/audio_processing/audio_processing_impl_unittest.cc b/modules/audio_processing/audio_processing_impl_unittest.cc index 72bd673531..68d17ae00f 100644 --- a/modules/audio_processing/audio_processing_impl_unittest.cc +++ b/modules/audio_processing/audio_processing_impl_unittest.cc @@ -60,6 +60,12 @@ class MockEchoControlFactory : public EchoControlFactory { return mock; } + std::unique_ptr Create(int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) override { + return Create(sample_rate_hz); + } + private: std::unique_ptr next_mock_; }; diff --git a/modules/audio_processing/audio_processing_unittest.cc b/modules/audio_processing/audio_processing_unittest.cc index 9c30ab088b..14ca3299bd 100644 --- a/modules/audio_processing/audio_processing_unittest.cc +++ b/modules/audio_processing/audio_processing_unittest.cc @@ -2513,6 +2513,12 @@ class MyEchoControlFactory : public EchoControlFactory { EXPECT_CALL(*ec, ProcessCapture(::testing::_, ::testing::_)).Times(2); return std::unique_ptr(ec); } + + std::unique_ptr Create(int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) { + return Create(sample_rate_hz); + } }; TEST(ApmConfiguration, EchoControlInjection) {