From 2e8e1c699ef5af1f8b3e4dd02a50feb015ed3faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=85hgren?= Date: Fri, 20 Dec 2019 00:42:22 +0100 Subject: [PATCH] Open up for do the noise suppressor analysis on the linear AEC output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL allows the noise suppressor to use the linear AEC output for analysis whenever that is available. This will potentially lower the risk that the noise suppressor estimates the noise based on echo. The feature is off by default. Bug: webrtc:5298,b:132164318 Change-Id: Idc6c8e197d96209d213819d87a8fb2533b7303ec Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/162900 Reviewed-by: Sam Zackrisson Commit-Queue: Per Åhgren Cr-Commit-Position: refs/heads/master@{#30116} --- .../audio_processing/audio_processing_impl.cc | 24 +++++++++++++++---- .../include/audio_processing.h | 1 + .../test/audio_processing_simulator.cc | 4 ++++ .../test/audio_processing_simulator.h | 1 + .../test/audioproc_float_impl.cc | 7 ++++++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 6d8b209740..cffbfbede3 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -1206,6 +1206,7 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { 1); AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity. + AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get(); if (submodules_.high_pass_filter && config_.high_pass_filter.apply_in_full_band && @@ -1292,10 +1293,14 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { RETURN_ON_ERR(submodules_.gain_control->AnalyzeCaptureAudio(*capture_buffer)); RTC_DCHECK( !(submodules_.legacy_noise_suppressor && submodules_.noise_suppressor)); - if (submodules_.noise_suppressor) { - submodules_.noise_suppressor->Analyze(*capture_buffer); - } else if (submodules_.legacy_noise_suppressor) { - submodules_.legacy_noise_suppressor->AnalyzeCaptureAudio(capture_buffer); + + if (!config_.noise_suppression.analyze_linear_aec_output_when_available || + !linear_aec_buffer || submodules_.echo_control_mobile) { + if (submodules_.noise_suppressor) { + submodules_.noise_suppressor->Analyze(*capture_buffer); + } else if (submodules_.legacy_noise_suppressor) { + submodules_.legacy_noise_suppressor->AnalyzeCaptureAudio(capture_buffer); + } } if (submodules_.echo_control_mobile) { @@ -1322,11 +1327,20 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { submodules_.echo_controller->SetAudioBufferDelay(stream_delay_ms()); } - AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get(); submodules_.echo_controller->ProcessCapture( capture_buffer, linear_aec_buffer, capture_.echo_path_gain_change); } + if (config_.noise_suppression.analyze_linear_aec_output_when_available && + linear_aec_buffer) { + if (submodules_.noise_suppressor) { + submodules_.noise_suppressor->Analyze(*linear_aec_buffer); + } else if (submodules_.legacy_noise_suppressor) { + submodules_.legacy_noise_suppressor->AnalyzeCaptureAudio( + linear_aec_buffer); + } + } + if (submodules_.noise_suppressor) { submodules_.noise_suppressor->Process(capture_buffer); } else if (submodules_.legacy_noise_suppressor) { diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index 5ab591bc7f..3d278cde63 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -241,6 +241,7 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { bool enabled = false; enum Level { kLow, kModerate, kHigh, kVeryHigh }; Level level = kModerate; + bool analyze_linear_aec_output_when_available = false; // Recommended not to use. Will be removed in the future. bool use_legacy_ns = false; } noise_suppression; diff --git a/modules/audio_processing/test/audio_processing_simulator.cc b/modules/audio_processing/test/audio_processing_simulator.cc index 89a9f134fc..02e0867641 100644 --- a/modules/audio_processing/test/audio_processing_simulator.cc +++ b/modules/audio_processing/test/audio_processing_simulator.cc @@ -528,6 +528,10 @@ void AudioProcessingSimulator::CreateAudioProcessor() { apm_config.noise_suppression.level = static_cast(level); } + if (settings_.ns_analysis_on_linear_aec_output) { + apm_config.noise_suppression.analyze_linear_aec_output_when_available = + *settings_.ns_analysis_on_linear_aec_output; + } RTC_CHECK(ap_builder_); if (echo_control_factory) { diff --git a/modules/audio_processing/test/audio_processing_simulator.h b/modules/audio_processing/test/audio_processing_simulator.h index abef2fa398..c902d7c9ea 100644 --- a/modules/audio_processing/test/audio_processing_simulator.h +++ b/modules/audio_processing/test/audio_processing_simulator.h @@ -75,6 +75,7 @@ struct SimulationSettings { agc2_adaptive_level_estimator; absl::optional pre_amplifier_gain_factor; absl::optional ns_level; + absl::optional ns_analysis_on_linear_aec_output; absl::optional maximum_internal_processing_rate; int initial_mic_level; bool simulate_mic_gain = false; diff --git a/modules/audio_processing/test/audioproc_float_impl.cc b/modules/audio_processing/test/audioproc_float_impl.cc index 6cfcef2f8c..c4d2ec26b5 100644 --- a/modules/audio_processing/test/audioproc_float_impl.cc +++ b/modules/audio_processing/test/audioproc_float_impl.cc @@ -173,6 +173,11 @@ ABSL_FLAG(int, ns_level, kParameterNotSpecifiedValue, "Specify the NS level (0-3)"); +ABSL_FLAG(int, + ns_analysis_on_linear_aec_output, + kParameterNotSpecifiedValue, + "Specifies whether the noise suppression analysis is done on the " + "linear AEC output"); ABSL_FLAG(int, maximum_internal_processing_rate, kParameterNotSpecifiedValue, @@ -402,6 +407,8 @@ SimulationSettings CreateSettings() { SetSettingIfSpecified(absl::GetFlag(FLAGS_pre_amplifier_gain_factor), &settings.pre_amplifier_gain_factor); SetSettingIfSpecified(absl::GetFlag(FLAGS_ns_level), &settings.ns_level); + SetSettingIfFlagSet(absl::GetFlag(FLAGS_ns_analysis_on_linear_aec_output), + &settings.ns_analysis_on_linear_aec_output); SetSettingIfSpecified(absl::GetFlag(FLAGS_maximum_internal_processing_rate), &settings.maximum_internal_processing_rate); SetSettingIfSpecified(absl::GetFlag(FLAGS_stream_delay),