From f204fafdb42faa9ac04d417ef39864e3db197d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=85hgren?= Date: Thu, 25 Apr 2019 15:18:06 +0200 Subject: [PATCH] Only create AEC2 when needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL ensures that the AEC2 is only created when needed. The changes in the CL are bitexact when running AEC2 via audioproc_f Bug: webrtc:8671 Change-Id: I5f6d33e45a7031c69ac53098781635c415668e49 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/129740 Reviewed-by: Sam Zackrisson Commit-Queue: Per Åhgren Cr-Commit-Position: refs/heads/master@{#27772} --- .../audio_processing/audio_processing_impl.cc | 218 ++++++++++-------- .../audio_processing/audio_processing_impl.h | 6 +- .../echo_cancellation_bit_exact_unittest.cc | 2 +- .../echo_cancellation_impl.cc | 14 +- .../audio_processing/echo_cancellation_impl.h | 7 +- .../echo_cancellation_impl_unittest.cc | 11 +- 6 files changed, 144 insertions(+), 114 deletions(-) diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 4d24ee662d..6950623523 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -429,7 +429,6 @@ AudioProcessingImpl::AudioProcessingImpl( new rtc::RefCountedObject(); } - private_submodules_->echo_cancellation.reset(new EchoCancellationImpl()); private_submodules_->echo_control_mobile.reset(new EchoControlMobileImpl()); // TODO(alessiob): Move the injected gain controller once injection is // implemented. @@ -548,15 +547,8 @@ int AudioProcessingImpl::InitializeLocked() { formats_.api_format.output_stream().num_channels(), formats_.api_format.output_stream().num_frames())); - private_submodules_->echo_cancellation->Initialize( - proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), - num_proc_channels()); AllocateRenderQueue(); - int success = private_submodules_->echo_cancellation->enable_metrics(true); - RTC_DCHECK_EQ(0, success); - success = private_submodules_->echo_cancellation->enable_delay_logging(true); - RTC_DCHECK_EQ(0, success); private_submodules_->echo_control_mobile->Initialize( proc_split_sample_rate_hz(), num_reverse_channels(), num_output_channels()); @@ -802,7 +794,12 @@ void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) { rtc::CritScope cs_render(&crit_render_); rtc::CritScope cs_capture(&crit_capture_); - private_submodules_->echo_cancellation->SetExtraOptions(config); + capture_nonlocked_.use_aec2_extended_filter = + config.Get().enabled; + capture_nonlocked_.use_aec2_delay_agnostic = + config.Get().enabled; + capture_nonlocked_.use_aec2_refined_adaptive_filter = + config.Get().enabled; if (capture_.transient_suppressor_enabled != config.Get().enabled) { @@ -1035,13 +1032,16 @@ void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) { RTC_DCHECK_GE(160, audio->num_frames_per_band()); // Insert the samples into the queue. - if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) { - // The data queue is full and needs to be emptied. - EmptyQueuedRenderAudio(); + if (private_submodules_->echo_cancellation) { + RTC_DCHECK(aec_render_signal_queue_); + if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); - // Retry the insert (should always work). - bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_); - RTC_DCHECK(result); + // Retry the insert (should always work). + bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_); + RTC_DCHECK(result); + } } EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(), @@ -1087,12 +1087,6 @@ void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) { } void AudioProcessingImpl::AllocateRenderQueue() { - const size_t new_aec_render_queue_element_max_size = - std::max(static_cast(1), - kMaxAllowedValuesOfSamplesPerBand * - EchoCancellationImpl::NumCancellersRequired( - num_output_channels(), num_reverse_channels())); - const size_t new_aecm_render_queue_element_max_size = std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerBand * @@ -1107,25 +1101,6 @@ void AudioProcessingImpl::AllocateRenderQueue() { // Reallocate the queues if the queue item sizes are too small to fit the // data to put in the queues. - if (aec_render_queue_element_max_size_ < - new_aec_render_queue_element_max_size) { - aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size; - - std::vector template_queue_element( - aec_render_queue_element_max_size_); - - aec_render_signal_queue_.reset( - new SwapQueue, RenderQueueItemVerifier>( - kMaxNumFramesToBuffer, template_queue_element, - RenderQueueItemVerifier( - aec_render_queue_element_max_size_))); - - aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_); - aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_); - } else { - aec_render_signal_queue_->Clear(); - } - if (aecm_render_queue_element_max_size_ < new_aecm_render_queue_element_max_size) { aecm_render_queue_element_max_size_ = @@ -1187,9 +1162,12 @@ void AudioProcessingImpl::AllocateRenderQueue() { void AudioProcessingImpl::EmptyQueuedRenderAudio() { rtc::CritScope cs_capture(&crit_capture_); - while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) { - private_submodules_->echo_cancellation->ProcessRenderAudio( - aec_capture_queue_buffer_); + if (private_submodules_->echo_cancellation) { + RTC_DCHECK(aec_render_signal_queue_); + while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) { + private_submodules_->echo_cancellation->ProcessRenderAudio( + aec_capture_queue_buffer_); + } } while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) { @@ -1283,7 +1261,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { // Ensure that not both the AEC and AECM are active at the same time. // TODO(peah): Simplify once the public API Enable functions for these // are moved to APM. - RTC_DCHECK(!(private_submodules_->echo_cancellation->is_enabled() && + RTC_DCHECK(!((private_submodules_->echo_cancellation && + private_submodules_->echo_cancellation->is_enabled()) && private_submodules_->echo_control_mobile->is_enabled())); MaybeUpdateHistograms(); @@ -1368,7 +1347,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { // Ensure that the stream delay was set before the call to the // AEC ProcessCaptureAudio function. - if (private_submodules_->echo_cancellation->is_enabled() && + if (private_submodules_->echo_cancellation && + private_submodules_->echo_cancellation->is_enabled() && !private_submodules_->echo_controller && !was_stream_delay_set()) { return AudioProcessing::kStreamParameterNotSetError; } @@ -1383,7 +1363,7 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { private_submodules_->echo_controller->ProcessCapture( capture_buffer, capture_.echo_path_gain_change); - } else { + } else if (private_submodules_->echo_cancellation) { RETURN_ON_ERR(private_submodules_->echo_cancellation->ProcessCaptureAudio( capture_buffer, stream_delay_ms())); } @@ -1402,7 +1382,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { } if (!(private_submodules_->echo_controller || - private_submodules_->echo_cancellation->is_enabled())) { + (private_submodules_->echo_cancellation && + private_submodules_->echo_cancellation->is_enabled()))) { RETURN_ON_ERR(private_submodules_->echo_control_mobile->ProcessCaptureAudio( capture_buffer, stream_delay_ms())); } @@ -1426,7 +1407,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { // TODO(peah): Add reporting from AEC3 whether there is echo. RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( capture_buffer, - private_submodules_->echo_cancellation->stream_has_echo())); + private_submodules_->echo_cancellation && + private_submodules_->echo_cancellation->stream_has_echo())); if (submodule_states_.CaptureMultiBandProcessingActive() && SampleRateSupportsMultiBand( @@ -1756,20 +1738,6 @@ AudioProcessingStats AudioProcessingImpl::GetStatistics( stats.echo_return_loss_enhancement = ec_metrics.echo_return_loss_enhancement; stats.delay_ms = ec_metrics.delay_ms; - } else if (private_submodules_->echo_cancellation->GetMetrics(&metrics) == - Error::kNoError) { - if (metrics.divergent_filter_fraction != -1.0f) { - stats.divergent_filter_fraction = - absl::optional(metrics.divergent_filter_fraction); - } - if (metrics.echo_return_loss.instant != -100) { - stats.echo_return_loss = - absl::optional(metrics.echo_return_loss.instant); - } - if (metrics.echo_return_loss_enhancement.instant != -100) { - stats.echo_return_loss_enhancement = - absl::optional(metrics.echo_return_loss_enhancement.instant); - } } if (config_.residual_echo_detector.enabled) { RTC_DCHECK(private_submodules_->echo_detector); @@ -1778,18 +1746,6 @@ AudioProcessingStats AudioProcessingImpl::GetStatistics( stats.residual_echo_likelihood_recent_max = ed_metrics.echo_likelihood_recent_max; } - int delay_median, delay_std; - float fraction_poor_delays; - if (private_submodules_->echo_cancellation->GetDelayMetrics( - &delay_median, &delay_std, &fraction_poor_delays) == - Error::kNoError) { - if (delay_median >= 0) { - stats.delay_median_ms = absl::optional(delay_median); - } - if (delay_std >= 0) { - stats.delay_standard_deviation_ms = absl::optional(delay_std); - } - } return stats; } @@ -1826,7 +1782,8 @@ AudioProcessing::Config AudioProcessingImpl::GetConfig() const { bool AudioProcessingImpl::UpdateActiveSubmoduleStates() { return submodule_states_.Update( config_.high_pass_filter.enabled, - private_submodules_->echo_cancellation->is_enabled(), + private_submodules_->echo_cancellation && + private_submodules_->echo_cancellation->is_enabled(), private_submodules_->echo_control_mobile->is_enabled(), config_.residual_echo_detector.enabled, public_submodules_->noise_suppression->is_enabled(), @@ -1872,20 +1829,82 @@ void AudioProcessingImpl::InitializeEchoController() { } capture_nonlocked_.echo_controller_enabled = true; - } else { - private_submodules_->echo_cancellation->Enable( - config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode); - private_submodules_->echo_control_mobile->Enable( - config_.echo_canceller.enabled && config_.echo_canceller.mobile_mode); - private_submodules_->echo_cancellation->set_suppression_level( - config_.echo_canceller.legacy_moderate_suppression_level - ? EchoCancellationImpl::SuppressionLevel::kModerateSuppression - : EchoCancellationImpl::SuppressionLevel::kHighSuppression); - - private_submodules_->echo_controller.reset(); - capture_nonlocked_.echo_controller_enabled = false; + private_submodules_->echo_cancellation.reset(); + aec_render_signal_queue_.reset(); + private_submodules_->echo_control_mobile->Enable(false); + return; } + + private_submodules_->echo_controller.reset(); + capture_nonlocked_.echo_controller_enabled = false; + + if (!config_.echo_canceller.enabled) { + private_submodules_->echo_cancellation.reset(); + aec_render_signal_queue_.reset(); + private_submodules_->echo_control_mobile->Enable(false); + return; + } + + if (config_.echo_canceller.mobile_mode) { + // Create and activate AECM. + // TODO(peah): Add an on-demand creation of AECmobile similar to AEC2. + private_submodules_->echo_control_mobile->Enable(true); + + private_submodules_->echo_cancellation.reset(); + aec_render_signal_queue_.reset(); + return; + } + + // Create and activate AEC2. + private_submodules_->echo_control_mobile->Enable(false); + private_submodules_->echo_cancellation.reset(new EchoCancellationImpl()); + private_submodules_->echo_cancellation->SetExtraOptions( + capture_nonlocked_.use_aec2_extended_filter, + capture_nonlocked_.use_aec2_delay_agnostic, + capture_nonlocked_.use_aec2_refined_adaptive_filter); + + const size_t new_aec_render_queue_element_max_size = + std::max(static_cast(1), + kMaxAllowedValuesOfSamplesPerBand * + EchoCancellationImpl::NumCancellersRequired( + num_output_channels(), num_reverse_channels())); + + // Reallocate the queues if the queue item sizes are too small to fit the + // data to put in the queues. + if (aec_render_queue_element_max_size_ < + new_aec_render_queue_element_max_size || + !aec_render_signal_queue_) { + aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size; + + std::vector template_queue_element( + aec_render_queue_element_max_size_); + + aec_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + aec_render_queue_element_max_size_))); + + aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_); + aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_); + } else { + aec_render_signal_queue_->Clear(); + } + + private_submodules_->echo_cancellation->Initialize( + proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), + num_proc_channels()); + + private_submodules_->echo_cancellation->Enable(true); + + private_submodules_->echo_cancellation->set_suppression_level( + config_.echo_canceller.legacy_moderate_suppression_level + ? EchoCancellationImpl::SuppressionLevel::kModerateSuppression + : EchoCancellationImpl::SuppressionLevel::kHighSuppression); + + private_submodules_->echo_controller.reset(); + capture_nonlocked_.echo_controller_enabled = false; } void AudioProcessingImpl::InitializeGainController2() { @@ -1935,7 +1954,8 @@ void AudioProcessingImpl::InitializePreProcessor() { void AudioProcessingImpl::MaybeUpdateHistograms() { static const int kMinDiffDelayMs = 60; - if (private_submodules_->echo_cancellation->is_enabled()) { + if (private_submodules_->echo_cancellation && + private_submodules_->echo_cancellation->is_enabled()) { // Activate delay_jumps_ counters if we know echo_cancellation is running. // If a stream has echo we know that the echo_cancellation is in process. if (capture_.stream_delay_jumps == -1 && @@ -2009,8 +2029,12 @@ void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) { if (!aec_dump_) { return; } - std::string experiments_description = - private_submodules_->echo_cancellation->GetExperimentsDescription(); + + std::string experiments_description = ""; + if (private_submodules_->echo_cancellation) { + experiments_description += + private_submodules_->echo_cancellation->GetExperimentsDescription(); + } // TODO(peah): Add semicolon-separated concatenations of experiment // descriptions for other submodules. if (constants_.agc_clipped_level_min != kClippedLevelMin) { @@ -2027,13 +2051,19 @@ void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) { apm_config.aec_enabled = config_.echo_canceller.enabled; apm_config.aec_delay_agnostic_enabled = + private_submodules_->echo_cancellation && private_submodules_->echo_cancellation->is_delay_agnostic_enabled(); apm_config.aec_drift_compensation_enabled = + private_submodules_->echo_cancellation && private_submodules_->echo_cancellation->is_drift_compensation_enabled(); apm_config.aec_extended_filter_enabled = + private_submodules_->echo_cancellation && private_submodules_->echo_cancellation->is_extended_filter_enabled(); - apm_config.aec_suppression_level = static_cast( - private_submodules_->echo_cancellation->suppression_level()); + apm_config.aec_suppression_level = + private_submodules_->echo_cancellation + ? static_cast( + private_submodules_->echo_cancellation->suppression_level()) + : 0; apm_config.aecm_enabled = private_submodules_->echo_control_mobile->is_enabled(); @@ -2115,7 +2145,9 @@ void AudioProcessingImpl::RecordAudioProcessingState() { AecDump::AudioProcessingState audio_proc_state; audio_proc_state.delay = capture_nonlocked_.stream_delay_ms; audio_proc_state.drift = - private_submodules_->echo_cancellation->stream_drift_samples(); + private_submodules_->echo_cancellation + ? private_submodules_->echo_cancellation->stream_drift_samples() + : 0; audio_proc_state.level = agc1()->stream_analog_level(); audio_proc_state.keypress = capture_.key_pressed; aec_dump_->AddAudioProcessingState(audio_proc_state); diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h index 9cd4f78b3f..33cc559abb 100644 --- a/modules/audio_processing/audio_processing_impl.h +++ b/modules/audio_processing/audio_processing_impl.h @@ -246,7 +246,8 @@ class AudioProcessingImpl : public AudioProcessing { void InitializeResidualEchoDetector() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void InitializeLowCutFilter() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); - void InitializeEchoController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeEchoController() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); @@ -417,6 +418,9 @@ class AudioProcessingImpl : public AudioProcessing { int split_rate; int stream_delay_ms; bool echo_controller_enabled = false; + bool use_aec2_extended_filter = false; + bool use_aec2_delay_agnostic = false; + bool use_aec2_refined_adaptive_filter = false; } capture_nonlocked_; struct ApmRenderState { diff --git a/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc b/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc index af0741ee17..37898c0054 100644 --- a/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc +++ b/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc @@ -33,7 +33,7 @@ void SetupComponent(int sample_rate_hz, Config config; config.Set(new DelayAgnostic(true)); config.Set(new ExtendedFilter(true)); - echo_canceller->SetExtraOptions(config); + echo_canceller->SetExtraOptions(true, true, false); } void ProcessOneFrame(int sample_rate_hz, diff --git a/modules/audio_processing/echo_cancellation_impl.cc b/modules/audio_processing/echo_cancellation_impl.cc index 96b978973c..ac13ebe077 100644 --- a/modules/audio_processing/echo_cancellation_impl.cc +++ b/modules/audio_processing/echo_cancellation_impl.cc @@ -16,7 +16,6 @@ #include "modules/audio_processing/aec/aec_core.h" #include "modules/audio_processing/aec/echo_cancellation.h" #include "modules/audio_processing/audio_buffer.h" -#include "modules/audio_processing/include/config.h" #include "rtc_base/checks.h" #include "system_wrappers/include/field_trial.h" @@ -432,13 +431,12 @@ void EchoCancellationImpl::PackRenderAudioBuffer( } } -void EchoCancellationImpl::SetExtraOptions(const webrtc::Config& config) { - { - extended_filter_enabled_ = config.Get().enabled; - delay_agnostic_enabled_ = config.Get().enabled; - refined_adaptive_filter_enabled_ = - config.Get().enabled; - } +void EchoCancellationImpl::SetExtraOptions(bool use_extended_filter, + bool use_delay_agnostic, + bool use_refined_adaptive_filter) { + extended_filter_enabled_ = use_extended_filter; + delay_agnostic_enabled_ = use_delay_agnostic; + refined_adaptive_filter_enabled_ = use_refined_adaptive_filter; Configure(); } diff --git a/modules/audio_processing/echo_cancellation_impl.h b/modules/audio_processing/echo_cancellation_impl.h index 79be73bb6c..25412cd8dd 100644 --- a/modules/audio_processing/echo_cancellation_impl.h +++ b/modules/audio_processing/echo_cancellation_impl.h @@ -17,10 +17,7 @@ #include #include "api/array_view.h" -#include "modules/audio_processing/include/audio_processing.h" #include "rtc_base/constructor_magic.h" -#include "rtc_base/critical_section.h" -#include "rtc_base/thread_annotations.h" namespace webrtc { @@ -136,7 +133,9 @@ class EchoCancellationImpl { size_t num_reverse_channels_, size_t num_output_channels_, size_t num_proc_channels_); - void SetExtraOptions(const webrtc::Config& config); + void SetExtraOptions(bool use_extended_filter, + bool use_delay_agnostic, + bool use_refined_adaptive_filter); bool is_delay_agnostic_enabled() const; bool is_extended_filter_enabled() const; std::string GetExperimentsDescription(); diff --git a/modules/audio_processing/echo_cancellation_impl_unittest.cc b/modules/audio_processing/echo_cancellation_impl_unittest.cc index 841ea0d9fb..22741ee1f9 100644 --- a/modules/audio_processing/echo_cancellation_impl_unittest.cc +++ b/modules/audio_processing/echo_cancellation_impl_unittest.cc @@ -31,16 +31,14 @@ TEST(EchoCancellationInternalTest, ExtendedFilter) { EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(aec_core)); Config config; - config.Set(new ExtendedFilter(true)); - echo_canceller.SetExtraOptions(config); + echo_canceller.SetExtraOptions(true, false, false); EXPECT_EQ(1, WebRtcAec_extended_filter_enabled(aec_core)); // Retains setting after initialization. echo_canceller.Initialize(AudioProcessing::kSampleRate16kHz, 2, 2, 2); EXPECT_EQ(1, WebRtcAec_extended_filter_enabled(aec_core)); - config.Set(new ExtendedFilter(false)); - echo_canceller.SetExtraOptions(config); + echo_canceller.SetExtraOptions(false, false, false); EXPECT_EQ(0, WebRtcAec_extended_filter_enabled(aec_core)); // Retains setting after initialization. @@ -63,8 +61,7 @@ TEST(EchoCancellationInternalTest, DelayAgnostic) { EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(aec_core)); Config config; - config.Set(new DelayAgnostic(true)); - echo_canceller.SetExtraOptions(config); + echo_canceller.SetExtraOptions(false, true, false); EXPECT_EQ(1, WebRtcAec_delay_agnostic_enabled(aec_core)); // Retains setting after initialization. @@ -72,7 +69,7 @@ TEST(EchoCancellationInternalTest, DelayAgnostic) { EXPECT_EQ(1, WebRtcAec_delay_agnostic_enabled(aec_core)); config.Set(new DelayAgnostic(false)); - echo_canceller.SetExtraOptions(config); + echo_canceller.SetExtraOptions(false, false, false); EXPECT_EQ(0, WebRtcAec_delay_agnostic_enabled(aec_core)); // Retains setting after initialization.