diff --git a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc index 022e860d45..e99ff2adb7 100644 --- a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc +++ b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc @@ -329,150 +329,155 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) { constexpr int kSampleRateHz = 48000; constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); constexpr size_t kNumBlocksToProcessPerRenderChannel = 1000; - constexpr size_t kNumCaptureChannels = 1; - for (size_t num_render_channels : {1, 2, 3, 6, 8}) { - ApmDataDumper data_dumper(42); - EchoCanceller3Config config; + for (size_t num_capture_channels : {1, 2, 4}) { + for (size_t num_render_channels : {1, 2, 3, 6, 8}) { + ApmDataDumper data_dumper(42); + EchoCanceller3Config config; - if (num_render_channels == 33) { - config.filter.main = {13, 0.00005f, 0.0005f, 0.0001f, 2.f, 20075344.f}; - config.filter.shadow = {13, 0.1f, 20075344.f}; - config.filter.main_initial = {12, 0.005f, 0.5f, 0.001f, 2.f, 20075344.f}; - config.filter.shadow_initial = {12, 0.7f, 20075344.f}; - } - - AdaptiveFirFilter filter( - config.filter.main.length_blocks, config.filter.main.length_blocks, - config.filter.config_change_duration_blocks, num_render_channels, - DetectOptimization(), &data_dumper); - std::vector>> H2( - kNumCaptureChannels, std::vector>( - filter.max_filter_size_partitions(), - std::array())); - std::vector> h( - kNumCaptureChannels, - std::vector( - GetTimeDomainLength(filter.max_filter_size_partitions()), 0.f)); - Aec3Fft fft; - config.delay.default_delay = 1; - std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, kSampleRateHz, num_render_channels)); - ShadowFilterUpdateGain gain(config.filter.shadow, - config.filter.config_change_duration_blocks); - Random random_generator(42U); - std::vector>> x( - kNumBands, - std::vector>(num_render_channels, - std::vector(kBlockSize, 0.f))); - std::vector n(kBlockSize, 0.f); - std::vector y(kBlockSize, 0.f); - AecState aec_state(EchoCanceller3Config{}, kNumCaptureChannels); - RenderSignalAnalyzer render_signal_analyzer(config); - absl::optional delay_estimate; - std::vector e(kBlockSize, 0.f); - std::array s_scratch; - std::vector output(kNumCaptureChannels); - FftData S; - FftData G; - FftData E; - std::vector> Y2(kNumCaptureChannels); - std::vector> E2_main( - kNumCaptureChannels); - std::array E2_shadow; - // [B,A] = butter(2,100/8000,'high') - constexpr CascadedBiQuadFilter::BiQuadCoefficients - kHighPassFilterCoefficients = {{0.97261f, -1.94523f, 0.97261f}, - {-1.94448f, 0.94598f}}; - for (auto& Y2_ch : Y2) { - Y2_ch.fill(0.f); - } - for (auto& E2_main_ch : E2_main) { - E2_main_ch.fill(0.f); - } - E2_shadow.fill(0.f); - for (auto& subtractor_output : output) { - subtractor_output.Reset(); - } - - constexpr float kScale = 1.0f / kFftLengthBy2; - - for (size_t delay_samples : {0, 64, 150, 200, 301}) { - std::vector> delay_buffer( - num_render_channels, DelayBuffer(delay_samples)); - std::vector> x_hp_filter( - num_render_channels); - for (size_t ch = 0; ch < num_render_channels; ++ch) { - x_hp_filter[ch] = std::make_unique( - kHighPassFilterCoefficients, 1); + if (num_render_channels == 33) { + config.filter.main = {13, 0.00005f, 0.0005f, 0.0001f, 2.f, 20075344.f}; + config.filter.shadow = {13, 0.1f, 20075344.f}; + config.filter.main_initial = {12, 0.005f, 0.5f, + 0.001f, 2.f, 20075344.f}; + config.filter.shadow_initial = {12, 0.7f, 20075344.f}; } - CascadedBiQuadFilter y_hp_filter(kHighPassFilterCoefficients, 1); - SCOPED_TRACE(ProduceDebugText(num_render_channels, delay_samples)); - const size_t num_blocks_to_process = - kNumBlocksToProcessPerRenderChannel * num_render_channels; - for (size_t j = 0; j < num_blocks_to_process; ++j) { - std::fill(y.begin(), y.end(), 0.f); + AdaptiveFirFilter filter( + config.filter.main.length_blocks, config.filter.main.length_blocks, + config.filter.config_change_duration_blocks, num_render_channels, + DetectOptimization(), &data_dumper); + std::vector>> H2( + num_capture_channels, + std::vector>( + filter.max_filter_size_partitions(), + std::array())); + std::vector> h( + num_capture_channels, + std::vector( + GetTimeDomainLength(filter.max_filter_size_partitions()), 0.f)); + Aec3Fft fft; + config.delay.default_delay = 1; + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, kSampleRateHz, + num_render_channels)); + ShadowFilterUpdateGain gain(config.filter.shadow, + config.filter.config_change_duration_blocks); + Random random_generator(42U); + std::vector>> x( + kNumBands, + std::vector>(num_render_channels, + std::vector(kBlockSize, 0.f))); + std::vector n(kBlockSize, 0.f); + std::vector y(kBlockSize, 0.f); + AecState aec_state(EchoCanceller3Config{}, num_capture_channels); + RenderSignalAnalyzer render_signal_analyzer(config); + absl::optional delay_estimate; + std::vector e(kBlockSize, 0.f); + std::array s_scratch; + std::vector output(num_capture_channels); + FftData S; + FftData G; + FftData E; + std::vector> Y2( + num_capture_channels); + std::vector> E2_main( + num_capture_channels); + std::array E2_shadow; + // [B,A] = butter(2,100/8000,'high') + constexpr CascadedBiQuadFilter::BiQuadCoefficients + kHighPassFilterCoefficients = {{0.97261f, -1.94523f, 0.97261f}, + {-1.94448f, 0.94598f}}; + for (auto& Y2_ch : Y2) { + Y2_ch.fill(0.f); + } + for (auto& E2_main_ch : E2_main) { + E2_main_ch.fill(0.f); + } + E2_shadow.fill(0.f); + for (auto& subtractor_output : output) { + subtractor_output.Reset(); + } + + constexpr float kScale = 1.0f / kFftLengthBy2; + + for (size_t delay_samples : {0, 64, 150, 200, 301}) { + std::vector> delay_buffer( + num_render_channels, DelayBuffer(delay_samples)); + std::vector> x_hp_filter( + num_render_channels); for (size_t ch = 0; ch < num_render_channels; ++ch) { - RandomizeSampleVector(&random_generator, x[0][ch]); - std::array y_channel; - delay_buffer[ch].Delay(x[0][ch], y_channel); + x_hp_filter[ch] = std::make_unique( + kHighPassFilterCoefficients, 1); + } + CascadedBiQuadFilter y_hp_filter(kHighPassFilterCoefficients, 1); + + SCOPED_TRACE(ProduceDebugText(num_render_channels, delay_samples)); + const size_t num_blocks_to_process = + kNumBlocksToProcessPerRenderChannel * num_render_channels; + for (size_t j = 0; j < num_blocks_to_process; ++j) { + std::fill(y.begin(), y.end(), 0.f); + for (size_t ch = 0; ch < num_render_channels; ++ch) { + RandomizeSampleVector(&random_generator, x[0][ch]); + std::array y_channel; + delay_buffer[ch].Delay(x[0][ch], y_channel); + for (size_t k = 0; k < y.size(); ++k) { + y[k] += y_channel[k] / num_render_channels; + } + } + + RandomizeSampleVector(&random_generator, n); + const float noise_scaling = 1.f / 100.f / num_render_channels; for (size_t k = 0; k < y.size(); ++k) { - y[k] += y_channel[k] / num_render_channels; + y[k] += n[k] * noise_scaling; } - } - RandomizeSampleVector(&random_generator, n); - const float noise_scaling = 1.f / 100.f / num_render_channels; - for (size_t k = 0; k < y.size(); ++k) { - y[k] += n[k] * noise_scaling; - } - - for (size_t ch = 0; ch < num_render_channels; ++ch) { - x_hp_filter[ch]->Process(x[0][ch]); - } - y_hp_filter.Process(y); - - render_delay_buffer->Insert(x); - if (j == 0) { - render_delay_buffer->Reset(); - } - render_delay_buffer->PrepareCaptureProcessing(); - auto* const render_buffer = render_delay_buffer->GetRenderBuffer(); - - render_signal_analyzer.Update(*render_buffer, - aec_state.MinDirectPathFilterDelay()); - - filter.Filter(*render_buffer, &S); - fft.Ifft(S, &s_scratch); - std::transform(y.begin(), y.end(), s_scratch.begin() + kFftLengthBy2, - e.begin(), - [&](float a, float b) { return a - b * kScale; }); - std::for_each(e.begin(), e.end(), [](float& a) { - a = rtc::SafeClamp(a, -32768.f, 32767.f); - }); - fft.ZeroPaddedFft(e, Aec3Fft::Window::kRectangular, &E); - for (auto& o : output) { - for (size_t k = 0; k < kBlockSize; ++k) { - o.s_main[k] = kScale * s_scratch[k + kFftLengthBy2]; + for (size_t ch = 0; ch < num_render_channels; ++ch) { + x_hp_filter[ch]->Process(x[0][ch]); } + y_hp_filter.Process(y); + + render_delay_buffer->Insert(x); + if (j == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureProcessing(); + auto* const render_buffer = render_delay_buffer->GetRenderBuffer(); + + render_signal_analyzer.Update(*render_buffer, + aec_state.MinDirectPathFilterDelay()); + + filter.Filter(*render_buffer, &S); + fft.Ifft(S, &s_scratch); + std::transform(y.begin(), y.end(), s_scratch.begin() + kFftLengthBy2, + e.begin(), + [&](float a, float b) { return a - b * kScale; }); + std::for_each(e.begin(), e.end(), [](float& a) { + a = rtc::SafeClamp(a, -32768.f, 32767.f); + }); + fft.ZeroPaddedFft(e, Aec3Fft::Window::kRectangular, &E); + for (auto& o : output) { + for (size_t k = 0; k < kBlockSize; ++k) { + o.s_main[k] = kScale * s_scratch[k + kFftLengthBy2]; + } + } + + std::array render_power; + render_buffer->SpectralSum(filter.SizePartitions(), &render_power); + gain.Compute(render_power, render_signal_analyzer, E, + filter.SizePartitions(), false, &G); + filter.Adapt(*render_buffer, G, &h[0]); + aec_state.HandleEchoPathChange(EchoPathVariability( + false, EchoPathVariability::DelayAdjustment::kNone, false)); + + filter.ComputeFrequencyResponse(&H2[0]); + aec_state.Update(delay_estimate, H2, h, *render_buffer, E2_main, Y2, + output); } - - std::array render_power; - render_buffer->SpectralSum(filter.SizePartitions(), &render_power); - gain.Compute(render_power, render_signal_analyzer, E, - filter.SizePartitions(), false, &G); - filter.Adapt(*render_buffer, G, &h[0]); - aec_state.HandleEchoPathChange(EchoPathVariability( - false, EchoPathVariability::DelayAdjustment::kNone, false)); - - filter.ComputeFrequencyResponse(&H2[0]); - aec_state.Update(delay_estimate, H2, h, *render_buffer, E2_main, Y2, - output); + // Verify that the filter is able to perform well. + EXPECT_LT(1000 * std::inner_product(e.begin(), e.end(), e.begin(), 0.f), + std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); } - // Verify that the filter is able to perform well. - EXPECT_LT(1000 * std::inner_product(e.begin(), e.end(), e.begin(), 0.f), - std::inner_product(y.begin(), y.end(), y.begin(), 0.f)); } } } diff --git a/modules/audio_processing/aec3/aec_state.h b/modules/audio_processing/aec3/aec_state.h index 53b8be03e2..250091e5dd 100644 --- a/modules/audio_processing/aec3/aec_state.h +++ b/modules/audio_processing/aec3/aec_state.h @@ -127,8 +127,7 @@ class AecState { } // Updates the aec state. - // TODO(bugs.webrtc.org/10913): Handle multi-channel adaptive filter response. - // TODO(bugs.webrtc.org/10913): Compute multi-channel ERL, ERLE, and reverb. + // TODO(bugs.webrtc.org/10913): Compute multi-channel ERL. void Update( const absl::optional& external_delay, rtc::ArrayView>> diff --git a/modules/audio_processing/aec3/comfort_noise_generator.cc b/modules/audio_processing/aec3/comfort_noise_generator.cc index fd12a7171e..005c25c3fd 100644 --- a/modules/audio_processing/aec3/comfort_noise_generator.cc +++ b/modules/audio_processing/aec3/comfort_noise_generator.cc @@ -105,7 +105,7 @@ ComfortNoiseGenerator::ComfortNoiseGenerator(Aec3Optimization optimization, ComfortNoiseGenerator::~ComfortNoiseGenerator() = default; void ComfortNoiseGenerator::Compute( - const AecState& aec_state, + bool saturated_capture, const std::array& capture_spectrum, FftData* lower_band_noise, FftData* upper_band_noise) { @@ -113,7 +113,7 @@ void ComfortNoiseGenerator::Compute( RTC_DCHECK(upper_band_noise); const auto& Y2 = capture_spectrum; - if (!aec_state.SaturatedCapture()) { + if (!saturated_capture) { // Smooth Y2. std::transform(Y2_smoothed_.begin(), Y2_smoothed_.end(), Y2.begin(), Y2_smoothed_.begin(), diff --git a/modules/audio_processing/aec3/comfort_noise_generator.h b/modules/audio_processing/aec3/comfort_noise_generator.h index 77967d8011..31360d2a83 100644 --- a/modules/audio_processing/aec3/comfort_noise_generator.h +++ b/modules/audio_processing/aec3/comfort_noise_generator.h @@ -45,7 +45,7 @@ class ComfortNoiseGenerator { ~ComfortNoiseGenerator(); // Computes the comfort noise. - void Compute(const AecState& aec_state, + void Compute(bool saturated_capture, const std::array& capture_spectrum, FftData* lower_band_noise, FftData* upper_band_noise); diff --git a/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc b/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc index 7abbb794b7..2d87cd8d9c 100644 --- a/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc +++ b/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc @@ -36,19 +36,17 @@ float Power(const FftData& N) { TEST(ComfortNoiseGenerator, NullLowerBandNoise) { std::array N2; FftData noise; - EXPECT_DEATH( - ComfortNoiseGenerator(DetectOptimization(), 42) - .Compute(AecState(EchoCanceller3Config{}, 1), N2, nullptr, &noise), - ""); + EXPECT_DEATH(ComfortNoiseGenerator(DetectOptimization(), 42) + .Compute(false, N2, nullptr, &noise), + ""); } TEST(ComfortNoiseGenerator, NullUpperBandNoise) { std::array N2; FftData noise; - EXPECT_DEATH( - ComfortNoiseGenerator(DetectOptimization(), 42) - .Compute(AecState(EchoCanceller3Config{}, 1), N2, &noise, nullptr), - ""); + EXPECT_DEATH(ComfortNoiseGenerator(DetectOptimization(), 42) + .Compute(false, N2, &noise, nullptr), + ""); } #endif @@ -68,12 +66,12 @@ TEST(ComfortNoiseGenerator, CorrectLevel) { n_upper.im.fill(0.f); // Ensure instantaneous updata to nonzero noise. - cng.Compute(aec_state, N2, &n_lower, &n_upper); + cng.Compute(false, N2, &n_lower, &n_upper); EXPECT_LT(0.f, Power(n_lower)); EXPECT_LT(0.f, Power(n_upper)); for (int k = 0; k < 10000; ++k) { - cng.Compute(aec_state, N2, &n_lower, &n_upper); + cng.Compute(false, N2, &n_lower, &n_upper); } EXPECT_NEAR(2.f * N2[0], Power(n_lower), N2[0] / 10.f); EXPECT_NEAR(2.f * N2[0], Power(n_upper), N2[0] / 10.f); diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc b/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc index b962d6430b..b38b9090e4 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc +++ b/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc @@ -36,37 +36,44 @@ std::string ProduceDebugText(size_t delay, size_t down_sampling_factor) { // Verifies that the basic API calls work. TEST(EchoPathDelayEstimator, BasicApiCalls) { - constexpr size_t kNumChannels = 1; constexpr int kSampleRateHz = 48000; constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); - ApmDataDumper data_dumper(0); - EchoCanceller3Config config; - std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); - EchoPathDelayEstimator estimator(&data_dumper, config); - std::vector>> render( - kNumBands, std::vector>( - kNumChannels, std::vector(kBlockSize))); - std::vector> capture(1, std::vector(kBlockSize)); - for (size_t k = 0; k < 100; ++k) { - render_delay_buffer->Insert(render); - estimator.EstimateDelay(render_delay_buffer->GetDownsampledRenderBuffer(), - capture); + for (size_t num_capture_channels : {1, 2, 4}) { + for (size_t num_render_channels : {1, 2, 3, 6, 8}) { + ApmDataDumper data_dumper(0); + EchoCanceller3Config config; + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, kSampleRateHz, + num_render_channels)); + EchoPathDelayEstimator estimator(&data_dumper, config); + std::vector>> render( + kNumBands, std::vector>( + num_render_channels, std::vector(kBlockSize))); + std::vector> capture(num_capture_channels, + std::vector(kBlockSize)); + for (size_t k = 0; k < 100; ++k) { + render_delay_buffer->Insert(render); + estimator.EstimateDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), capture); + } + } } } // Verifies that the delay estimator produces correct delay for artificially // delayed signals. TEST(EchoPathDelayEstimator, DelayEstimation) { - constexpr size_t kNumChannels = 1; + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; constexpr int kSampleRateHz = 48000; constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); Random random_generator(42U); std::vector>> render( kNumBands, std::vector>( - kNumChannels, std::vector(kBlockSize))); - std::vector> capture(1, std::vector(kBlockSize)); + kNumRenderChannels, std::vector(kBlockSize))); + std::vector> capture(kNumCaptureChannels, + std::vector(kBlockSize)); ApmDataDumper data_dumper(0); constexpr size_t kDownSamplingFactors[] = {2, 4, 8}; for (auto down_sampling_factor : kDownSamplingFactors) { @@ -76,7 +83,7 @@ TEST(EchoPathDelayEstimator, DelayEstimation) { for (size_t delay_samples : {30, 64, 150, 200, 800, 4000}) { SCOPED_TRACE(ProduceDebugText(delay_samples, down_sampling_factor)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, kSampleRateHz, kNumChannels)); + RenderDelayBuffer::Create(config, kSampleRateHz, kNumRenderChannels)); DelayBuffer signal_delay_buffer(delay_samples); EchoPathDelayEstimator estimator(&data_dumper, config); @@ -117,20 +124,22 @@ TEST(EchoPathDelayEstimator, DelayEstimation) { // Verifies that the delay estimator does not produce delay estimates for render // signals of low level. TEST(EchoPathDelayEstimator, NoDelayEstimatesForLowLevelRenderSignals) { - constexpr size_t kNumChannels = 1; + constexpr size_t kNumRenderChannels = 1; + constexpr size_t kNumCaptureChannels = 1; constexpr int kSampleRateHz = 48000; constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz); Random random_generator(42U); EchoCanceller3Config config; std::vector>> render( kNumBands, std::vector>( - kNumChannels, std::vector(kBlockSize))); - std::vector> capture(1, std::vector(kBlockSize)); + kNumRenderChannels, std::vector(kBlockSize))); + std::vector> capture(kNumCaptureChannels, + std::vector(kBlockSize)); ApmDataDumper data_dumper(0); EchoPathDelayEstimator estimator(&data_dumper, config); std::unique_ptr render_delay_buffer( RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz, - kNumChannels)); + kNumRenderChannels)); for (size_t k = 0; k < 100; ++k) { RandomizeSampleVector(&random_generator, render[0][0]); for (auto& render_k : render[0][0]) { diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index b508c95f39..602a353e03 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -316,15 +316,12 @@ void EchoRemoverImpl::ProcessCapture( subtractor_output_heap_.data(), num_capture_channels_); } - const std::vector& x0 = x[0][0]; - std::vector& y0 = (*y)[0][0]; - - data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, &y0[0], - 16000, 1); - data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, &x0[0], - 16000, 1); - data_dumper_->DumpRaw("aec3_echo_remover_capture_input", y0); - data_dumper_->DumpRaw("aec3_echo_remover_render_input", x0); + data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, + &(*y)[0][0][0], 16000, 1); + data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, + &x[0][0][0], 16000, 1); + data_dumper_->DumpRaw("aec3_echo_remover_capture_input", (*y)[0][0]); + data_dumper_->DumpRaw("aec3_echo_remover_render_input", x[0][0]); aec_state_.UpdateCaptureSaturation(capture_signal_saturation); @@ -374,12 +371,10 @@ void EchoRemoverImpl::ProcessCapture( subtractor_.Process(*render_buffer, (*y)[0], render_signal_analyzer_, aec_state_, subtractor_output); + // Compute spectra. for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - auto& y_low = (*y)[0][ch]; - - // Compute spectra. FormLinearFilterOutput(subtractor_output[ch], e[ch]); - WindowedPaddedFft(fft_, y_low, y_old_[ch], &Y[ch]); + WindowedPaddedFft(fft_, (*y)[0][ch], y_old_[ch], &Y[ch]); WindowedPaddedFft(fft_, e[ch], e_old_[ch], &E[ch]); LinearEchoPower(E[ch], Y[ch], &S2_linear[ch]); Y[ch].Spectrum(optimization_, Y2[ch]); @@ -387,15 +382,15 @@ void EchoRemoverImpl::ProcessCapture( } // Update the AEC state information. - // TODO(bugs.webrtc.org/10913): Take all subtractors into account. - aec_state_.Update(external_delay, subtractor_.FilterFrequencyResponse(), - subtractor_.FilterImpulseResponse(), *render_buffer, E2, Y2, - subtractor_output); + aec_state_.Update(external_delay, subtractor_.FilterFrequencyResponses(), + subtractor_.FilterImpulseResponses(), *render_buffer, E2, + Y2, subtractor_output); // Choose the linear output. const auto& Y_fft = aec_state_.UseLinearFilterOutput() ? E : Y; - data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0], 16000, 1); + data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &(*y)[0][0][0], 16000, + 1); data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0][0], 16000, 1); float high_bands_gain = 1.f; @@ -408,8 +403,8 @@ void EchoRemoverImpl::ProcessCapture( for (size_t ch = 0; ch < num_capture_channels_; ++ch) { // Estimate the comfort noise. - cngs_[ch]->Compute(aec_state_, Y2[ch], &comfort_noise[ch], - &high_band_comfort_noise[ch]); + cngs_[ch]->Compute(aec_state_.SaturatedCapture(), Y2[ch], + &comfort_noise[ch], &high_band_comfort_noise[ch]); // Suppressor echo estimate. const auto& echo_spectrum = @@ -448,13 +443,14 @@ void EchoRemoverImpl::ProcessCapture( // Debug outputs for the purpose of development and analysis. data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize, &subtractor_output[0].s_main[0], 16000, 1); - data_dumper_->DumpRaw("aec3_output", y0); + data_dumper_->DumpRaw("aec3_output", (*y)[0][0]); data_dumper_->DumpRaw("aec3_narrow_render", render_signal_analyzer_.NarrowPeakBand() ? 1 : 0); data_dumper_->DumpRaw("aec3_N2", cngs_[0]->NoiseSpectrum()); data_dumper_->DumpRaw("aec3_suppressor_gain", G); - data_dumper_->DumpWav( - "aec3_output", rtc::ArrayView(&y0[0], kBlockSize), 16000, 1); + data_dumper_->DumpWav("aec3_output", + rtc::ArrayView(&(*y)[0][0][0], kBlockSize), + 16000, 1); data_dumper_->DumpRaw("aec3_using_subtractor_output[0]", aec_state_.UseLinearFilterOutput() ? 1 : 0); data_dumper_->DumpRaw("aec3_E2", E2[0]); diff --git a/modules/audio_processing/aec3/filter_analyzer_unittest.cc b/modules/audio_processing/aec3/filter_analyzer_unittest.cc index 34104c39b2..f1e2e4c188 100644 --- a/modules/audio_processing/aec3/filter_analyzer_unittest.cc +++ b/modules/audio_processing/aec3/filter_analyzer_unittest.cc @@ -21,11 +21,13 @@ namespace webrtc { TEST(FilterAnalyzer, FilterResize) { EchoCanceller3Config c; std::vector filter(65, 0.f); - FilterAnalyzer fa(c, 1); - fa.SetRegionToAnalyze(filter.size()); - fa.SetRegionToAnalyze(filter.size()); - filter.resize(32); - fa.SetRegionToAnalyze(filter.size()); + for (size_t num_capture_channels : {1, 2, 4}) { + FilterAnalyzer fa(c, num_capture_channels); + fa.SetRegionToAnalyze(filter.size()); + fa.SetRegionToAnalyze(filter.size()); + filter.resize(32); + fa.SetRegionToAnalyze(filter.size()); + } } } // namespace webrtc diff --git a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc index fa3b263502..f79b2d6e84 100644 --- a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc +++ b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc @@ -54,12 +54,12 @@ void RunFilterUpdateTest(int num_blocks_to_process, config.filter.shadow.length_blocks = filter_length_blocks; AdaptiveFirFilter main_filter(config.filter.main.length_blocks, config.filter.main.length_blocks, - config.filter.config_change_duration_blocks, 1, - optimization, &data_dumper); - AdaptiveFirFilter shadow_filter(config.filter.shadow.length_blocks, - config.filter.shadow.length_blocks, - config.filter.config_change_duration_blocks, - 1, optimization, &data_dumper); + config.filter.config_change_duration_blocks, + kNumRenderChannels, optimization, &data_dumper); + AdaptiveFirFilter shadow_filter( + config.filter.shadow.length_blocks, config.filter.shadow.length_blocks, + config.filter.config_change_duration_blocks, kNumRenderChannels, + optimization, &data_dumper); std::vector>> H2( kNumCaptureChannels, std::vector>( main_filter.max_filter_size_partitions(), diff --git a/modules/audio_processing/aec3/render_delay_controller_unittest.cc b/modules/audio_processing/aec3/render_delay_controller_unittest.cc index 6cee5c957c..de195cc5a2 100644 --- a/modules/audio_processing/aec3/render_delay_controller_unittest.cc +++ b/modules/audio_processing/aec3/render_delay_controller_unittest.cc @@ -46,24 +46,27 @@ constexpr size_t kDownSamplingFactors[] = {2, 4, 8}; // Verifies the output of GetDelay when there are no AnalyzeRender calls. TEST(RenderDelayController, NoRenderSignal) { - std::vector> block(1, std::vector(kBlockSize, 0.f)); - EchoCanceller3Config config; - for (size_t num_matched_filters = 4; num_matched_filters == 10; - num_matched_filters++) { - for (auto down_sampling_factor : kDownSamplingFactors) { - config.delay.down_sampling_factor = down_sampling_factor; - config.delay.num_filters = num_matched_filters; - for (auto rate : {16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - std::unique_ptr delay_buffer( - RenderDelayBuffer::Create(config, rate, 1)); - std::unique_ptr delay_controller( - RenderDelayController::Create(config, rate)); - for (size_t k = 0; k < 100; ++k) { - auto delay = delay_controller->GetDelay( - delay_buffer->GetDownsampledRenderBuffer(), delay_buffer->Delay(), - block); - EXPECT_FALSE(delay->delay); + for (size_t num_render_channels : {1, 2, 8}) { + std::vector> block(1, + std::vector(kBlockSize, 0.f)); + EchoCanceller3Config config; + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; + for (auto rate : {16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(config, rate, num_render_channels)); + std::unique_ptr delay_controller( + RenderDelayController::Create(config, rate)); + for (size_t k = 0; k < 100; ++k) { + auto delay = delay_controller->GetDelay( + delay_buffer->GetDownsampledRenderBuffer(), + delay_buffer->Delay(), block); + EXPECT_FALSE(delay->delay); + } } } } @@ -72,35 +75,38 @@ TEST(RenderDelayController, NoRenderSignal) { // Verifies the basic API call sequence. TEST(RenderDelayController, BasicApiCalls) { - constexpr size_t kNumChannels = 1; - std::vector> capture_block( - 1, std::vector(kBlockSize, 0.f)); - absl::optional delay_blocks; - for (size_t num_matched_filters = 4; num_matched_filters == 10; - num_matched_filters++) { - for (auto down_sampling_factor : kDownSamplingFactors) { - EchoCanceller3Config config; - config.delay.down_sampling_factor = down_sampling_factor; - config.delay.num_filters = num_matched_filters; - for (auto rate : {16000, 32000, 48000}) { - std::vector>> render_block( - NumBandsForRate(rate), - std::vector>( - kNumChannels, std::vector(kBlockSize, 0.f))); - std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate, kNumChannels)); - std::unique_ptr delay_controller( - RenderDelayController::Create(EchoCanceller3Config(), rate)); - for (size_t k = 0; k < 10; ++k) { - render_delay_buffer->Insert(render_block); - render_delay_buffer->PrepareCaptureProcessing(); + for (size_t num_capture_channels : {1, 2, 4}) { + for (size_t num_render_channels : {1, 2, 8}) { + std::vector> capture_block( + num_capture_channels, std::vector(kBlockSize, 0.f)); + absl::optional delay_blocks; + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; + for (auto rate : {16000, 32000, 48000}) { + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>( + num_render_channels, std::vector(kBlockSize, 0.f))); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, rate, num_render_channels)); + std::unique_ptr delay_controller( + RenderDelayController::Create(EchoCanceller3Config(), rate)); + for (size_t k = 0; k < 10; ++k) { + render_delay_buffer->Insert(render_block); + render_delay_buffer->PrepareCaptureProcessing(); - delay_blocks = delay_controller->GetDelay( - render_delay_buffer->GetDownsampledRenderBuffer(), - render_delay_buffer->Delay(), capture_block); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), + render_delay_buffer->Delay(), capture_block); + } + EXPECT_TRUE(delay_blocks); + EXPECT_FALSE(delay_blocks->delay); + } } - EXPECT_TRUE(delay_blocks); - EXPECT_FALSE(delay_blocks->delay); } } } @@ -110,53 +116,55 @@ TEST(RenderDelayController, BasicApiCalls) { // simple timeshifts between the signals. TEST(RenderDelayController, Alignment) { Random random_generator(42U); - std::vector> capture_block( - 1, std::vector(kBlockSize, 0.f)); - for (size_t num_matched_filters = 4; num_matched_filters == 10; - num_matched_filters++) { - for (auto down_sampling_factor : kDownSamplingFactors) { - EchoCanceller3Config config; - config.delay.down_sampling_factor = down_sampling_factor; - config.delay.num_filters = num_matched_filters; + for (size_t num_capture_channels : {1, 2, 4}) { + std::vector> capture_block( + num_capture_channels, std::vector(kBlockSize, 0.f)); + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; - for (size_t num_render_channels : {1, 2}) { - for (auto rate : {16000, 32000, 48000}) { - std::vector>> render_block( - NumBandsForRate(rate), - std::vector>( - num_render_channels, std::vector(kBlockSize, 0.f))); + for (size_t num_render_channels : {1, 2, 8}) { + for (auto rate : {16000, 32000, 48000}) { + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>( + num_render_channels, std::vector(kBlockSize, 0.f))); - for (size_t delay_samples : {15, 50, 150, 200, 800, 4000}) { - absl::optional delay_blocks; - SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); - std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate, num_render_channels)); - std::unique_ptr delay_controller( - RenderDelayController::Create(config, rate)); - DelayBuffer signal_delay_buffer(delay_samples); - for (size_t k = 0; k < (400 + delay_samples / kBlockSize); ++k) { - for (size_t band = 0; band < render_block.size(); ++band) { - for (size_t channel = 0; channel < render_block[band].size(); - ++channel) { - RandomizeSampleVector(&random_generator, - render_block[band][channel]); + for (size_t delay_samples : {15, 50, 150, 200, 800, 4000}) { + absl::optional delay_blocks; + SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, rate, num_render_channels)); + std::unique_ptr delay_controller( + RenderDelayController::Create(config, rate)); + DelayBuffer signal_delay_buffer(delay_samples); + for (size_t k = 0; k < (400 + delay_samples / kBlockSize); ++k) { + for (size_t band = 0; band < render_block.size(); ++band) { + for (size_t channel = 0; channel < render_block[band].size(); + ++channel) { + RandomizeSampleVector(&random_generator, + render_block[band][channel]); + } } + signal_delay_buffer.Delay(render_block[0][0], capture_block[0]); + render_delay_buffer->Insert(render_block); + render_delay_buffer->PrepareCaptureProcessing(); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), + render_delay_buffer->Delay(), capture_block); } - signal_delay_buffer.Delay(render_block[0][0], capture_block[0]); - render_delay_buffer->Insert(render_block); - render_delay_buffer->PrepareCaptureProcessing(); - delay_blocks = delay_controller->GetDelay( - render_delay_buffer->GetDownsampledRenderBuffer(), - render_delay_buffer->Delay(), capture_block); + ASSERT_TRUE(!!delay_blocks); + + constexpr int kDelayHeadroomBlocks = 1; + size_t expected_delay_blocks = + std::max(0, static_cast(delay_samples / kBlockSize) - + kDelayHeadroomBlocks); + + EXPECT_EQ(expected_delay_blocks, delay_blocks->delay); } - ASSERT_TRUE(!!delay_blocks); - - constexpr int kDelayHeadroomBlocks = 1; - size_t expected_delay_blocks = - std::max(0, static_cast(delay_samples / kBlockSize) - - kDelayHeadroomBlocks); - - EXPECT_EQ(expected_delay_blocks, delay_blocks->delay); } } } @@ -168,44 +176,48 @@ TEST(RenderDelayController, Alignment) { // delays. TEST(RenderDelayController, NonCausalAlignment) { Random random_generator(42U); - constexpr size_t kNumRenderChannels = 1; - constexpr size_t kNumCaptureChannels = 1; - for (size_t num_matched_filters = 4; num_matched_filters == 10; - num_matched_filters++) { - for (auto down_sampling_factor : kDownSamplingFactors) { - EchoCanceller3Config config; - config.delay.down_sampling_factor = down_sampling_factor; - config.delay.num_filters = num_matched_filters; - for (auto rate : {16000, 32000, 48000}) { - std::vector>> render_block( - NumBandsForRate(rate), - std::vector>( - kNumRenderChannels, std::vector(kBlockSize, 0.f))); - std::vector>> capture_block( - NumBandsForRate(rate), - std::vector>( - kNumCaptureChannels, std::vector(kBlockSize, 0.f))); + for (size_t num_capture_channels : {1, 2, 4}) { + for (size_t num_render_channels : {1, 2, 8}) { + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; + for (auto rate : {16000, 32000, 48000}) { + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>( + num_render_channels, std::vector(kBlockSize, 0.f))); + std::vector>> capture_block( + NumBandsForRate(rate), + std::vector>( + num_capture_channels, std::vector(kBlockSize, 0.f))); - for (int delay_samples : {-15, -50, -150, -200}) { - absl::optional delay_blocks; - SCOPED_TRACE(ProduceDebugText(rate, -delay_samples)); - std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate, kNumRenderChannels)); - std::unique_ptr delay_controller( - RenderDelayController::Create(EchoCanceller3Config(), rate)); - DelayBuffer signal_delay_buffer(-delay_samples); - for (int k = 0; - k < (400 - delay_samples / static_cast(kBlockSize)); ++k) { - RandomizeSampleVector(&random_generator, capture_block[0][0]); - signal_delay_buffer.Delay(capture_block[0][0], render_block[0][0]); - render_delay_buffer->Insert(render_block); - render_delay_buffer->PrepareCaptureProcessing(); - delay_blocks = delay_controller->GetDelay( - render_delay_buffer->GetDownsampledRenderBuffer(), - render_delay_buffer->Delay(), capture_block[0]); + for (int delay_samples : {-15, -50, -150, -200}) { + absl::optional delay_blocks; + SCOPED_TRACE(ProduceDebugText(rate, -delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, rate, num_render_channels)); + std::unique_ptr delay_controller( + RenderDelayController::Create(EchoCanceller3Config(), rate)); + DelayBuffer signal_delay_buffer(-delay_samples); + for (int k = 0; + k < (400 - delay_samples / static_cast(kBlockSize)); + ++k) { + RandomizeSampleVector(&random_generator, capture_block[0][0]); + signal_delay_buffer.Delay(capture_block[0][0], + render_block[0][0]); + render_delay_buffer->Insert(render_block); + render_delay_buffer->PrepareCaptureProcessing(); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), + render_delay_buffer->Delay(), capture_block[0]); + } + + ASSERT_FALSE(delay_blocks); + } } - - ASSERT_FALSE(delay_blocks); } } } @@ -216,86 +228,69 @@ TEST(RenderDelayController, NonCausalAlignment) { // simple timeshifts between the signals when there is jitter in the API calls. TEST(RenderDelayController, AlignmentWithJitter) { Random random_generator(42U); - constexpr size_t kNumRenderChannels = 1; - std::vector> capture_block( - 1, std::vector(kBlockSize, 0.f)); - for (size_t num_matched_filters = 4; num_matched_filters == 10; - num_matched_filters++) { - for (auto down_sampling_factor : kDownSamplingFactors) { - EchoCanceller3Config config; - config.delay.down_sampling_factor = down_sampling_factor; - config.delay.num_filters = num_matched_filters; - for (auto rate : {16000, 32000, 48000}) { - std::vector>> render_block( - NumBandsForRate(rate), - std::vector>( - kNumRenderChannels, std::vector(kBlockSize, 0.f))); - for (size_t delay_samples : {15, 50, 300, 800}) { - absl::optional delay_blocks; - SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); - std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate, kNumRenderChannels)); - std::unique_ptr delay_controller( - RenderDelayController::Create(config, rate)); - DelayBuffer signal_delay_buffer(delay_samples); - constexpr size_t kMaxTestJitterBlocks = 26; - for (size_t j = 0; - j < - (1000 + delay_samples / kBlockSize) / kMaxTestJitterBlocks + 1; - ++j) { - std::vector>> capture_block_buffer; - for (size_t k = 0; k < (kMaxTestJitterBlocks - 1); ++k) { - RandomizeSampleVector(&random_generator, render_block[0][0]); - signal_delay_buffer.Delay(render_block[0][0], capture_block[0]); - capture_block_buffer.push_back(capture_block); - render_delay_buffer->Insert(render_block); - } - for (size_t k = 0; k < (kMaxTestJitterBlocks - 1); ++k) { - render_delay_buffer->PrepareCaptureProcessing(); - delay_blocks = delay_controller->GetDelay( - render_delay_buffer->GetDownsampledRenderBuffer(), - render_delay_buffer->Delay(), capture_block_buffer[k]); + for (size_t num_capture_channels : {1, 2, 4}) { + for (size_t num_render_channels : {1, 2, 8}) { + std::vector> capture_block( + num_capture_channels, std::vector(kBlockSize, 0.f)); + for (size_t num_matched_filters = 4; num_matched_filters == 10; + num_matched_filters++) { + for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; + for (auto rate : {16000, 32000, 48000}) { + std::vector>> render_block( + NumBandsForRate(rate), + std::vector>( + num_render_channels, std::vector(kBlockSize, 0.f))); + for (size_t delay_samples : {15, 50, 300, 800}) { + absl::optional delay_blocks; + SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, rate, num_render_channels)); + std::unique_ptr delay_controller( + RenderDelayController::Create(config, rate)); + DelayBuffer signal_delay_buffer(delay_samples); + constexpr size_t kMaxTestJitterBlocks = 26; + for (size_t j = 0; j < (1000 + delay_samples / kBlockSize) / + kMaxTestJitterBlocks + + 1; + ++j) { + std::vector>> + capture_block_buffer; + for (size_t k = 0; k < (kMaxTestJitterBlocks - 1); ++k) { + RandomizeSampleVector(&random_generator, render_block[0][0]); + signal_delay_buffer.Delay(render_block[0][0], + capture_block[0]); + capture_block_buffer.push_back(capture_block); + render_delay_buffer->Insert(render_block); + } + for (size_t k = 0; k < (kMaxTestJitterBlocks - 1); ++k) { + render_delay_buffer->PrepareCaptureProcessing(); + delay_blocks = delay_controller->GetDelay( + render_delay_buffer->GetDownsampledRenderBuffer(), + render_delay_buffer->Delay(), capture_block_buffer[k]); + } + } + + constexpr int kDelayHeadroomBlocks = 1; + size_t expected_delay_blocks = + std::max(0, static_cast(delay_samples / kBlockSize) - + kDelayHeadroomBlocks); + if (expected_delay_blocks < 2) { + expected_delay_blocks = 0; + } + + ASSERT_TRUE(delay_blocks); + EXPECT_EQ(expected_delay_blocks, delay_blocks->delay); } } - - constexpr int kDelayHeadroomBlocks = 1; - size_t expected_delay_blocks = - std::max(0, static_cast(delay_samples / kBlockSize) - - kDelayHeadroomBlocks); - if (expected_delay_blocks < 2) { - expected_delay_blocks = 0; - } - - ASSERT_TRUE(delay_blocks); - EXPECT_EQ(expected_delay_blocks, delay_blocks->delay); } } } } } -// Verifies the initial value for the AlignmentHeadroomSamples. -TEST(RenderDelayController, InitialHeadroom) { - std::vector render_block(kBlockSize, 0.f); - std::vector capture_block(kBlockSize, 0.f); - for (size_t num_matched_filters = 4; num_matched_filters == 10; - num_matched_filters++) { - for (auto down_sampling_factor : kDownSamplingFactors) { - EchoCanceller3Config config; - config.delay.down_sampling_factor = down_sampling_factor; - config.delay.num_filters = num_matched_filters; - for (auto rate : {16000, 32000, 48000}) { - SCOPED_TRACE(ProduceDebugText(rate)); - std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(config, rate, 1)); - - std::unique_ptr delay_controller( - RenderDelayController::Create(config, rate)); - } - } - } -} - #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for the capture signal block size. diff --git a/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc index a73a539c74..d2d100588e 100644 --- a/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc +++ b/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc @@ -41,14 +41,14 @@ void RunFilterUpdateTest(int num_blocks_to_process, ApmDataDumper data_dumper(42); EchoCanceller3Config config; config.filter.main.length_blocks = filter_length_blocks; - AdaptiveFirFilter main_filter(config.filter.main.length_blocks, - config.filter.main.length_blocks, - config.filter.config_change_duration_blocks, 1, - DetectOptimization(), &data_dumper); - AdaptiveFirFilter shadow_filter(config.filter.shadow.length_blocks, - config.filter.shadow.length_blocks, - config.filter.config_change_duration_blocks, - 1, DetectOptimization(), &data_dumper); + AdaptiveFirFilter main_filter( + config.filter.main.length_blocks, config.filter.main.length_blocks, + config.filter.config_change_duration_blocks, num_render_channels, + DetectOptimization(), &data_dumper); + AdaptiveFirFilter shadow_filter( + config.filter.shadow.length_blocks, config.filter.shadow.length_blocks, + config.filter.config_change_duration_blocks, num_render_channels, + DetectOptimization(), &data_dumper); Aec3Fft fft; constexpr int kSampleRateHz = 48000; @@ -158,8 +158,7 @@ TEST(ShadowFilterUpdateGain, GainCausesFilterToConverge) { std::vector blocks_with_echo_path_changes; std::vector blocks_with_saturation; - // TODO(http://bugs.webrtc.org/10913): Test multiple render channel counts. - for (size_t num_render_channels : {1}) { + for (size_t num_render_channels : {1, 2, 8}) { for (size_t filter_length_blocks : {12, 20, 30}) { for (size_t delay_samples : {0, 64, 150, 200, 301}) { SCOPED_TRACE(ProduceDebugText(delay_samples, filter_length_blocks)); @@ -168,7 +167,7 @@ TEST(ShadowFilterUpdateGain, GainCausesFilterToConverge) { std::array y; FftData G; - RunFilterUpdateTest(1000, delay_samples, num_render_channels, + RunFilterUpdateTest(5000, delay_samples, num_render_channels, filter_length_blocks, blocks_with_saturation, &e, &y, &G); @@ -190,8 +189,7 @@ TEST(ShadowFilterUpdateGain, GainCausesFilterToConverge) { // Verifies that the magnitude of the gain on average decreases for a // persistently exciting signal. TEST(ShadowFilterUpdateGain, DecreasingGain) { - // TODO(http://bugs.webrtc.org/10913): Test multiple render channel counts. - for (size_t num_render_channels : {1}) { + for (size_t num_render_channels : {1, 2, 4}) { for (size_t filter_length_blocks : {12, 20, 30}) { SCOPED_TRACE(ProduceDebugText(filter_length_blocks)); std::vector blocks_with_echo_path_changes; @@ -233,8 +231,7 @@ TEST(ShadowFilterUpdateGain, SaturationBehavior) { for (int k = 99; k < 200; ++k) { blocks_with_saturation.push_back(k); } - // TODO(http://bugs.webrtc.org/10913): Test multiple render channel counts. - for (size_t num_render_channels : {1}) { + for (size_t num_render_channels : {1, 2, 8}) { for (size_t filter_length_blocks : {12, 20, 30}) { SCOPED_TRACE(ProduceDebugText(filter_length_blocks)); diff --git a/modules/audio_processing/aec3/subtractor.cc b/modules/audio_processing/aec3/subtractor.cc index 5e995656e8..27cc424e07 100644 --- a/modules/audio_processing/aec3/subtractor.cc +++ b/modules/audio_processing/aec3/subtractor.cc @@ -66,26 +66,26 @@ Subtractor::Subtractor(const EchoCanceller3Config& config, optimization_(optimization), config_(config), num_capture_channels_(num_capture_channels), - main_filter_(num_capture_channels_), + main_filters_(num_capture_channels_), shadow_filter_(num_capture_channels_), - G_main_(num_capture_channels_), - G_shadow_(num_capture_channels_), - filter_misadjustment_estimator_(num_capture_channels_), - poor_shadow_filter_counter_(num_capture_channels_, 0), - main_frequency_response_( + main_gains_(num_capture_channels_), + shadow_gains_(num_capture_channels_), + filter_misadjustment_estimators_(num_capture_channels_), + poor_shadow_filter_counters_(num_capture_channels_, 0), + main_frequency_responses_( num_capture_channels_, std::vector>( std::max(config_.filter.main_initial.length_blocks, config_.filter.main.length_blocks), std::array())), - main_impulse_response_( + main_impulse_responses_( num_capture_channels_, std::vector(GetTimeDomainLength(std::max( config_.filter.main_initial.length_blocks, config_.filter.main.length_blocks)), 0.f)) { for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - main_filter_[ch] = std::make_unique( + main_filters_[ch] = std::make_unique( config_.filter.main.length_blocks, config_.filter.main_initial.length_blocks, config.filter.config_change_duration_blocks, num_render_channels, @@ -96,17 +96,17 @@ Subtractor::Subtractor(const EchoCanceller3Config& config, config_.filter.shadow_initial.length_blocks, config.filter.config_change_duration_blocks, num_render_channels, optimization, data_dumper_); - G_main_[ch] = std::make_unique( + main_gains_[ch] = std::make_unique( config_.filter.main_initial, config_.filter.config_change_duration_blocks); - G_shadow_[ch] = std::make_unique( + shadow_gains_[ch] = std::make_unique( config_.filter.shadow_initial, config.filter.config_change_duration_blocks); } RTC_DCHECK(data_dumper_); for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - for (auto& H2_k : main_frequency_response_[ch]) { + for (auto& H2_k : main_frequency_responses_[ch]) { H2_k.fill(0.f); } } @@ -118,13 +118,13 @@ void Subtractor::HandleEchoPathChange( const EchoPathVariability& echo_path_variability) { const auto full_reset = [&]() { for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - main_filter_[ch]->HandleEchoPathChange(); + main_filters_[ch]->HandleEchoPathChange(); shadow_filter_[ch]->HandleEchoPathChange(); - G_main_[ch]->HandleEchoPathChange(echo_path_variability); - G_shadow_[ch]->HandleEchoPathChange(); - G_main_[ch]->SetConfig(config_.filter.main_initial, true); - G_shadow_[ch]->SetConfig(config_.filter.shadow_initial, true); - main_filter_[ch]->SetSizePartitions( + main_gains_[ch]->HandleEchoPathChange(echo_path_variability); + shadow_gains_[ch]->HandleEchoPathChange(); + main_gains_[ch]->SetConfig(config_.filter.main_initial, true); + shadow_gains_[ch]->SetConfig(config_.filter.shadow_initial, true); + main_filters_[ch]->SetSizePartitions( config_.filter.main_initial.length_blocks, true); shadow_filter_[ch]->SetSizePartitions( config_.filter.shadow_initial.length_blocks, true); @@ -138,17 +138,17 @@ void Subtractor::HandleEchoPathChange( if (echo_path_variability.gain_change) { for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - G_main_[ch]->HandleEchoPathChange(echo_path_variability); + main_gains_[ch]->HandleEchoPathChange(echo_path_variability); } } } void Subtractor::ExitInitialState() { for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - G_main_[ch]->SetConfig(config_.filter.main, false); - G_shadow_[ch]->SetConfig(config_.filter.shadow, false); - main_filter_[ch]->SetSizePartitions(config_.filter.main.length_blocks, - false); + main_gains_[ch]->SetConfig(config_.filter.main, false); + shadow_gains_[ch]->SetConfig(config_.filter.shadow, false); + main_filters_[ch]->SetSizePartitions(config_.filter.main.length_blocks, + false); shadow_filter_[ch]->SetSizePartitions(config_.filter.shadow.length_blocks, false); } @@ -163,19 +163,19 @@ void Subtractor::Process(const RenderBuffer& render_buffer, // Compute the render powers. const bool same_filter_sizes = - main_filter_[0]->SizePartitions() == shadow_filter_[0]->SizePartitions(); + main_filters_[0]->SizePartitions() == shadow_filter_[0]->SizePartitions(); std::array X2_main; std::array X2_shadow_data; auto& X2_shadow = same_filter_sizes ? X2_main : X2_shadow_data; if (same_filter_sizes) { - render_buffer.SpectralSum(main_filter_[0]->SizePartitions(), &X2_main); - } else if (main_filter_[0]->SizePartitions() > + render_buffer.SpectralSum(main_filters_[0]->SizePartitions(), &X2_main); + } else if (main_filters_[0]->SizePartitions() > shadow_filter_[0]->SizePartitions()) { render_buffer.SpectralSums(shadow_filter_[0]->SizePartitions(), - main_filter_[0]->SizePartitions(), &X2_shadow, + main_filters_[0]->SizePartitions(), &X2_shadow, &X2_main); } else { - render_buffer.SpectralSums(main_filter_[0]->SizePartitions(), + render_buffer.SpectralSums(main_filters_[0]->SizePartitions(), shadow_filter_[0]->SizePartitions(), &X2_main, &X2_shadow); } @@ -194,7 +194,7 @@ void Subtractor::Process(const RenderBuffer& render_buffer, FftData& G = S; // Form the outputs of the main and shadow filters. - main_filter_[ch]->Filter(render_buffer, &S); + main_filters_[ch]->Filter(render_buffer, &S); PredictionError(fft_, S, y, &e_main, &output.s_main); shadow_filter_[ch]->Filter(render_buffer, &S); @@ -204,17 +204,17 @@ void Subtractor::Process(const RenderBuffer& render_buffer, output.ComputeMetrics(y); // Adjust the filter if needed. - bool main_filter_adjusted = false; - filter_misadjustment_estimator_[ch].Update(output); - if (filter_misadjustment_estimator_[ch].IsAdjustmentNeeded()) { - float scale = filter_misadjustment_estimator_[ch].GetMisadjustment(); - main_filter_[ch]->ScaleFilter(scale); - for (auto& h_k : main_impulse_response_[ch]) { + bool main_filters_adjusted = false; + filter_misadjustment_estimators_[ch].Update(output); + if (filter_misadjustment_estimators_[ch].IsAdjustmentNeeded()) { + float scale = filter_misadjustment_estimators_[ch].GetMisadjustment(); + main_filters_[ch]->ScaleFilter(scale); + for (auto& h_k : main_impulse_responses_[ch]) { h_k *= scale; } ScaleFilterOutput(y, scale, e_main, output.s_main); - filter_misadjustment_estimator_[ch].Reset(); - main_filter_adjusted = true; + filter_misadjustment_estimators_[ch].Reset(); + main_filters_adjusted = true; } // Compute the FFts of the main and shadow filter outputs. @@ -226,18 +226,18 @@ void Subtractor::Process(const RenderBuffer& render_buffer, E_main.Spectrum(optimization_, output.E2_main); // Update the main filter. - if (!main_filter_adjusted) { + if (!main_filters_adjusted) { std::array erl; - ComputeErl(optimization_, main_frequency_response_[ch], erl); - G_main_[ch]->Compute(X2_main, render_signal_analyzer, output, erl, - main_filter_[ch]->SizePartitions(), - aec_state.SaturatedCapture(), &G); + ComputeErl(optimization_, main_frequency_responses_[ch], erl); + main_gains_[ch]->Compute(X2_main, render_signal_analyzer, output, erl, + main_filters_[ch]->SizePartitions(), + aec_state.SaturatedCapture(), &G); } else { G.re.fill(0.f); G.im.fill(0.f); } - main_filter_[ch]->Adapt(render_buffer, G, &main_impulse_response_[ch]); - main_filter_[ch]->ComputeFrequencyResponse(&main_frequency_response_[ch]); + main_filters_[ch]->Adapt(render_buffer, G, &main_impulse_responses_[ch]); + main_filters_[ch]->ComputeFrequencyResponse(&main_frequency_responses_[ch]); if (ch == 0) { data_dumper_->DumpRaw("aec3_subtractor_G_main", G.re); @@ -245,27 +245,27 @@ void Subtractor::Process(const RenderBuffer& render_buffer, } // Update the shadow filter. - poor_shadow_filter_counter_[ch] = output.e2_main < output.e2_shadow - ? poor_shadow_filter_counter_[ch] + 1 + poor_shadow_filter_counters_[ch] = + output.e2_main < output.e2_shadow ? poor_shadow_filter_counters_[ch] + 1 : 0; - if (poor_shadow_filter_counter_[ch] < 5) { - G_shadow_[ch]->Compute(X2_shadow, render_signal_analyzer, E_shadow, - shadow_filter_[ch]->SizePartitions(), - aec_state.SaturatedCapture(), &G); + if (poor_shadow_filter_counters_[ch] < 5) { + shadow_gains_[ch]->Compute(X2_shadow, render_signal_analyzer, E_shadow, + shadow_filter_[ch]->SizePartitions(), + aec_state.SaturatedCapture(), &G); } else { - poor_shadow_filter_counter_[ch] = 0; - shadow_filter_[ch]->SetFilter(main_filter_[ch]->SizePartitions(), - main_filter_[ch]->GetFilter()); - G_shadow_[ch]->Compute(X2_shadow, render_signal_analyzer, E_main, - shadow_filter_[ch]->SizePartitions(), - aec_state.SaturatedCapture(), &G); + poor_shadow_filter_counters_[ch] = 0; + shadow_filter_[ch]->SetFilter(main_filters_[ch]->SizePartitions(), + main_filters_[ch]->GetFilter()); + shadow_gains_[ch]->Compute(X2_shadow, render_signal_analyzer, E_main, + shadow_filter_[ch]->SizePartitions(), + aec_state.SaturatedCapture(), &G); } shadow_filter_[ch]->Adapt(render_buffer, G); if (ch == 0) { data_dumper_->DumpRaw("aec3_subtractor_G_shadow", G.re); data_dumper_->DumpRaw("aec3_subtractor_G_shadow", G.im); - filter_misadjustment_estimator_[ch].Dump(data_dumper_); + filter_misadjustment_estimators_[ch].Dump(data_dumper_); DumpFilters(); } @@ -273,7 +273,7 @@ void Subtractor::Process(const RenderBuffer& render_buffer, [](float& a) { a = rtc::SafeClamp(a, -32768.f, 32767.f); }); if (ch == 0) { - data_dumper_->DumpWav("aec3_main_filter_output", kBlockSize, &e_main[0], + data_dumper_->DumpWav("aec3_main_filters_output", kBlockSize, &e_main[0], 16000, 1); data_dumper_->DumpWav("aec3_shadow_filter_output", kBlockSize, &e_shadow[0], 16000, 1); diff --git a/modules/audio_processing/aec3/subtractor.h b/modules/audio_processing/aec3/subtractor.h index 01d2eef403..32c42cccc1 100644 --- a/modules/audio_processing/aec3/subtractor.h +++ b/modules/audio_processing/aec3/subtractor.h @@ -60,25 +60,25 @@ class Subtractor { // Returns the block-wise frequency responses for the main adaptive filters. const std::vector>>& - FilterFrequencyResponse() const { - return main_frequency_response_; + FilterFrequencyResponses() const { + return main_frequency_responses_; } // Returns the estimates of the impulse responses for the main adaptive // filters. - const std::vector>& FilterImpulseResponse() const { - return main_impulse_response_; + const std::vector>& FilterImpulseResponses() const { + return main_impulse_responses_; } void DumpFilters() { data_dumper_->DumpRaw( "aec3_subtractor_h_main", rtc::ArrayView( - main_impulse_response_[0].data(), + main_impulse_responses_[0].data(), GetTimeDomainLength( - main_filter_[0]->max_filter_size_partitions()))); + main_filters_[0]->max_filter_size_partitions()))); - main_filter_[0]->DumpFilter("aec3_subtractor_H_main"); + main_filters_[0]->DumpFilter("aec3_subtractor_H_main"); shadow_filter_[0]->DumpFilter("aec3_subtractor_H_shadow"); } @@ -120,15 +120,15 @@ class Subtractor { const EchoCanceller3Config config_; const size_t num_capture_channels_; - std::vector> main_filter_; + std::vector> main_filters_; std::vector> shadow_filter_; - std::vector> G_main_; - std::vector> G_shadow_; - std::vector filter_misadjustment_estimator_; - std::vector poor_shadow_filter_counter_; + std::vector> main_gains_; + std::vector> shadow_gains_; + std::vector filter_misadjustment_estimators_; + std::vector poor_shadow_filter_counters_; std::vector>> - main_frequency_response_; - std::vector> main_impulse_response_; + main_frequency_responses_; + std::vector> main_impulse_responses_; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/subtractor_unittest.cc b/modules/audio_processing/aec3/subtractor_unittest.cc index b59fa7b5e8..507d70c39a 100644 --- a/modules/audio_processing/aec3/subtractor_unittest.cc +++ b/modules/audio_processing/aec3/subtractor_unittest.cc @@ -150,8 +150,8 @@ std::vector RunSubtractorTest( aec_state.HandleEchoPathChange(EchoPathVariability( false, EchoPathVariability::DelayAdjustment::kNone, false)); - aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(), - subtractor.FilterImpulseResponse(), + aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponses(), + subtractor.FilterImpulseResponses(), *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, output); } diff --git a/modules/audio_processing/aec3/suppression_gain.cc b/modules/audio_processing/aec3/suppression_gain.cc index 89ebe0f4f3..6ec70bfade 100644 --- a/modules/audio_processing/aec3/suppression_gain.cc +++ b/modules/audio_processing/aec3/suppression_gain.cc @@ -114,6 +114,7 @@ float SuppressionGain::UpperBandsGain( if (render.size() == 1) { return 1.f; } + const size_t num_render_channels = render[0].size(); if (narrow_peak_band && (*narrow_peak_band > static_cast(kFftLengthBy2Plus1 - 10))) { @@ -131,13 +132,19 @@ float SuppressionGain::UpperBandsGain( // Compute the upper and lower band energies. const auto sum_of_squares = [](float a, float b) { return a + b * b; }; - const float low_band_energy = std::accumulate( - render[0][0].begin(), render[0][0].end(), 0.f, sum_of_squares); + float low_band_energy = 0.f; + for (size_t ch = 0; ch < num_render_channels; ++ch) { + const float channel_energy = std::accumulate( + render[0][0].begin(), render[0][0].end(), 0.f, sum_of_squares); + low_band_energy = std::max(low_band_energy, channel_energy); + } float high_band_energy = 0.f; for (size_t k = 1; k < render.size(); ++k) { - const float energy = std::accumulate( - render[k][0].begin(), render[k][0].end(), 0.f, sum_of_squares); - high_band_energy = std::max(high_band_energy, energy); + for (size_t ch = 0; ch < num_render_channels; ++ch) { + const float energy = std::accumulate( + render[k][ch].begin(), render[k][ch].end(), 0.f, sum_of_squares); + high_band_energy = std::max(high_band_energy, energy); + } } // If there is more power in the lower frequencies than the upper frequencies, @@ -369,11 +376,16 @@ bool SuppressionGain::LowNoiseRenderDetector::Detect( const std::vector>>& render) { float x2_sum = 0.f; float x2_max = 0.f; - for (auto x_k : render[0][0]) { - const float x2 = x_k * x_k; - x2_sum += x2; - x2_max = std::max(x2_max, x2); + for (auto x_ch : render[0]) { + for (auto x_k : x_ch) { + const float x2 = x_k * x_k; + x2_sum += x2; + x2_max = std::max(x2_max, x2); + } } + const size_t num_render_channels = render[0].size(); + x2_sum = x2_sum / num_render_channels; + ; constexpr float kThreshold = 50.f * 50.f * 64.f; const bool low_noise_render = diff --git a/modules/audio_processing/aec3/suppression_gain_unittest.cc b/modules/audio_processing/aec3/suppression_gain_unittest.cc index d068328772..6396af8e3a 100644 --- a/modules/audio_processing/aec3/suppression_gain_unittest.cc +++ b/modules/audio_processing/aec3/suppression_gain_unittest.cc @@ -102,14 +102,14 @@ TEST(SuppressionGain, BasicGainComputation) { // Ensure that the gain is no longer forced to zero. for (int k = 0; k <= kNumBlocksPerSecond / 5 + 1; ++k) { - aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(), - subtractor.FilterImpulseResponse(), + aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponses(), + subtractor.FilterImpulseResponses(), *render_delay_buffer->GetRenderBuffer(), E2, Y2, output); } for (int k = 0; k < 100; ++k) { - aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(), - subtractor.FilterImpulseResponse(), + aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponses(), + subtractor.FilterImpulseResponses(), *render_delay_buffer->GetRenderBuffer(), E2, Y2, output); suppression_gain.GetGain(E2[0], S2, R2, N2, analyzer, aec_state, x, &high_bands_gain, &g); @@ -129,8 +129,8 @@ TEST(SuppressionGain, BasicGainComputation) { N2.fill(0.f); for (int k = 0; k < 100; ++k) { - aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(), - subtractor.FilterImpulseResponse(), + aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponses(), + subtractor.FilterImpulseResponses(), *render_delay_buffer->GetRenderBuffer(), E2, Y2, output); suppression_gain.GetGain(E2[0], S2, R2, N2, analyzer, aec_state, x, &high_bands_gain, &g);