From 9f4a4a096bf11e26ce6446620dcd3dd825807d3e Mon Sep 17 00:00:00 2001 From: ivoc Date: Fri, 28 Oct 2016 05:39:16 -0700 Subject: [PATCH] Add empty residual echo detector. This CL does not contain the actual algorithm, but only creates an empty processing component and connects the right signals to it. The algorithm will be added in a follow-up CL. BUG=webrtc:6525 Review-Url: https://codereview.webrtc.org/2405403003 Cr-Commit-Position: refs/heads/master@{#14820} --- webrtc/modules/audio_processing/BUILD.gn | 2 + .../audio_processing/audio_processing.gypi | 2 + .../audio_processing/audio_processing_impl.cc | 60 +++++++++++++++++++ .../audio_processing/audio_processing_impl.h | 14 ++++- .../include/audio_processing.h | 3 + .../residual_echo_detector.cc | 52 ++++++++++++++++ .../audio_processing/residual_echo_detector.h | 47 +++++++++++++++ 7 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 webrtc/modules/audio_processing/residual_echo_detector.cc create mode 100644 webrtc/modules/audio_processing/residual_echo_detector.h diff --git a/webrtc/modules/audio_processing/BUILD.gn b/webrtc/modules/audio_processing/BUILD.gn index bd6adf6542..4090f6d38b 100644 --- a/webrtc/modules/audio_processing/BUILD.gn +++ b/webrtc/modules/audio_processing/BUILD.gn @@ -99,6 +99,8 @@ rtc_static_library("audio_processing") { "noise_suppression_impl.cc", "noise_suppression_impl.h", "render_queue_item_verifier.h", + "residual_echo_detector.cc", + "residual_echo_detector.h", "rms_level.cc", "rms_level.h", "splitting_filter.cc", diff --git a/webrtc/modules/audio_processing/audio_processing.gypi b/webrtc/modules/audio_processing/audio_processing.gypi index d7d84d1695..d25f9e578e 100644 --- a/webrtc/modules/audio_processing/audio_processing.gypi +++ b/webrtc/modules/audio_processing/audio_processing.gypi @@ -111,6 +111,8 @@ 'noise_suppression_impl.cc', 'noise_suppression_impl.h', 'render_queue_item_verifier.h', + 'residual_echo_detector.cc', + 'residual_echo_detector.h', 'rms_level.cc', 'rms_level.h', 'splitting_filter.cc', diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index acbc80c9dd..32f1d9f5a1 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -35,6 +35,7 @@ #include "webrtc/modules/audio_processing/level_controller/level_controller.h" #include "webrtc/modules/audio_processing/level_estimator_impl.h" #include "webrtc/modules/audio_processing/noise_suppression_impl.h" +#include "webrtc/modules/audio_processing/residual_echo_detector.h" #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" #include "webrtc/modules/audio_processing/voice_detection_impl.h" #include "webrtc/modules/include/module_common_types.h" @@ -137,6 +138,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update( bool high_pass_filter_enabled, bool echo_canceller_enabled, bool mobile_echo_controller_enabled, + bool residual_echo_detector_enabled, bool noise_suppressor_enabled, bool intelligibility_enhancer_enabled, bool beamformer_enabled, @@ -150,6 +152,8 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update( changed |= (echo_canceller_enabled != echo_canceller_enabled_); changed |= (mobile_echo_controller_enabled != mobile_echo_controller_enabled_); + changed |= + (residual_echo_detector_enabled != residual_echo_detector_enabled_); changed |= (noise_suppressor_enabled != noise_suppressor_enabled_); changed |= (intelligibility_enhancer_enabled != intelligibility_enhancer_enabled_); @@ -165,6 +169,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update( high_pass_filter_enabled_ = high_pass_filter_enabled; echo_canceller_enabled_ = echo_canceller_enabled; mobile_echo_controller_enabled_ = mobile_echo_controller_enabled; + residual_echo_detector_enabled_ = residual_echo_detector_enabled; noise_suppressor_enabled_ = noise_suppressor_enabled; intelligibility_enhancer_enabled_ = intelligibility_enhancer_enabled; beamformer_enabled_ = beamformer_enabled; @@ -239,6 +244,7 @@ struct AudioProcessingImpl::ApmPrivateSubmodules { std::unique_ptr beamformer; std::unique_ptr agc_manager; std::unique_ptr level_controller; + std::unique_ptr residual_echo_detector; }; AudioProcessing* AudioProcessing::Create() { @@ -304,6 +310,8 @@ AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config, public_submodules_->gain_control_for_experimental_agc.reset( new GainControlForExperimentalAgc( public_submodules_->gain_control.get(), &crit_capture_)); + private_submodules_->residual_echo_detector.reset( + new ResidualEchoDetector()); // TODO(peah): Move this creation to happen only when the level controller // is enabled. @@ -470,6 +478,7 @@ int AudioProcessingImpl::InitializeLocked() { public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); public_submodules_->level_estimator->Initialize(); InitializeLevelController(); + InitializeResidualEchoDetector(); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP if (debug_dump_.debug_file->is_open()) { @@ -809,6 +818,18 @@ void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) { RTC_DCHECK(result); } } + + ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_); + + // Insert the samples into the queue. + if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_); + RTC_DCHECK(result); + } } void AudioProcessingImpl::AllocateRenderQueue() { @@ -827,6 +848,9 @@ void AudioProcessingImpl::AllocateRenderQueue() { const size_t new_agc_render_queue_element_max_size = std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame); + const size_t new_red_render_queue_element_max_size = + std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame); + // Reallocate the queues if the queue item sizes are too small to fit the // data to put in the queues. if (aec_render_queue_element_max_size_ < @@ -886,6 +910,25 @@ void AudioProcessingImpl::AllocateRenderQueue() { } else { agc_render_signal_queue_->Clear(); } + + if (red_render_queue_element_max_size_ < + new_red_render_queue_element_max_size) { + red_render_queue_element_max_size_ = new_red_render_queue_element_max_size; + + std::vector template_queue_element( + red_render_queue_element_max_size_); + + red_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + red_render_queue_element_max_size_))); + + red_render_queue_buffer_.resize(red_render_queue_element_max_size_); + red_capture_queue_buffer_.resize(red_render_queue_element_max_size_); + } else { + red_render_signal_queue_->Clear(); + } } void AudioProcessingImpl::EmptyQueuedRenderAudio() { @@ -904,6 +947,11 @@ void AudioProcessingImpl::EmptyQueuedRenderAudio() { public_submodules_->gain_control->ProcessRenderAudio( agc_capture_queue_buffer_); } + + while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) { + private_submodules_->residual_echo_detector->AnalyzeRenderAudio( + red_capture_queue_buffer_); + } } int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { @@ -1079,6 +1127,13 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( capture_buffer, stream_delay_ms())); + if (config_.residual_echo_detector.enabled) { + private_submodules_->residual_echo_detector->AnalyzeCaptureAudio( + rtc::ArrayView( + capture_buffer->split_bands_const_f(0)[kBand0To8kHz], + capture_buffer->num_frames_per_band())); + } + if (capture_nonlocked_.beamformer_enabled) { private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f()); } @@ -1454,6 +1509,7 @@ bool AudioProcessingImpl::UpdateActiveSubmoduleStates() { public_submodules_->high_pass_filter->is_enabled(), public_submodules_->echo_cancellation->is_enabled(), public_submodules_->echo_control_mobile->is_enabled(), + config_.residual_echo_detector.enabled, public_submodules_->noise_suppression->is_enabled(), capture_nonlocked_.intelligibility_enabled, capture_nonlocked_.beamformer_enabled, @@ -1503,6 +1559,10 @@ void AudioProcessingImpl::InitializeLevelController() { private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); } +void AudioProcessingImpl::InitializeResidualEchoDetector() { + private_submodules_->residual_echo_detector->Initialize(); +} + void AudioProcessingImpl::MaybeUpdateHistograms() { static const int kMinDiffDelayMs = 60; diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index aec4ed7086..fd8683fb57 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -150,6 +150,7 @@ class AudioProcessingImpl : public AudioProcessing { bool Update(bool high_pass_filter_enabled, bool echo_canceller_enabled, bool mobile_echo_controller_enabled, + bool residual_echo_detector_enabled, bool noise_suppressor_enabled, bool intelligibility_enhancer_enabled, bool beamformer_enabled, @@ -167,6 +168,7 @@ class AudioProcessingImpl : public AudioProcessing { bool high_pass_filter_enabled_ = false; bool echo_canceller_enabled_ = false; bool mobile_echo_controller_enabled_ = false; + bool residual_echo_detector_enabled_ = false; bool noise_suppressor_enabled_ = false; bool intelligibility_enhancer_enabled_ = false; bool beamformer_enabled_ = false; @@ -234,6 +236,8 @@ class AudioProcessingImpl : public AudioProcessing { int InitializeLocked(const ProcessingConfig& config) EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void InitializeLevelController() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeResidualEchoDetector() + EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void EmptyQueuedRenderAudio(); void AllocateRenderQueue() @@ -291,8 +295,7 @@ class AudioProcessingImpl : public AudioProcessing { // Structs containing the pointers to the submodules. std::unique_ptr public_submodules_; - std::unique_ptr private_submodules_ - GUARDED_BY(crit_capture_); + std::unique_ptr private_submodules_; // State that is written to while holding both the render and capture locks // but can be read without any lock being held. @@ -386,6 +389,11 @@ class AudioProcessingImpl : public AudioProcessing { std::vector agc_render_queue_buffer_ GUARDED_BY(crit_render_); std::vector agc_capture_queue_buffer_ GUARDED_BY(crit_capture_); + size_t red_render_queue_element_max_size_ GUARDED_BY(crit_render_) + GUARDED_BY(crit_capture_) = 0; + std::vector red_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector red_capture_queue_buffer_ GUARDED_BY(crit_capture_); + // Lock protection not needed. std::unique_ptr, RenderQueueItemVerifier>> aec_render_signal_queue_; @@ -395,6 +403,8 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr< SwapQueue, RenderQueueItemVerifier>> agc_render_signal_queue_; + std::unique_ptr, RenderQueueItemVerifier>> + red_render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/include/audio_processing.h b/webrtc/modules/audio_processing/include/audio_processing.h index 0fdbfd2d75..7629717247 100644 --- a/webrtc/modules/audio_processing/include/audio_processing.h +++ b/webrtc/modules/audio_processing/include/audio_processing.h @@ -258,6 +258,9 @@ class AudioProcessing { // the allowed range is [-100, 0]. float initial_peak_level_dbfs = -6.0206f; } level_controller; + struct ResidualEchoDetector { + bool enabled = false; + } residual_echo_detector; }; // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone. diff --git a/webrtc/modules/audio_processing/residual_echo_detector.cc b/webrtc/modules/audio_processing/residual_echo_detector.cc new file mode 100644 index 0000000000..df99de45bf --- /dev/null +++ b/webrtc/modules/audio_processing/residual_echo_detector.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/residual_echo_detector.h" + +#include "webrtc/modules/audio_processing/audio_buffer.h" + +namespace webrtc { + +ResidualEchoDetector::ResidualEchoDetector() = default; + +ResidualEchoDetector::~ResidualEchoDetector() = default; + +void ResidualEchoDetector::AnalyzeRenderAudio( + rtc::ArrayView render_audio) const { + // TODO(ivoc): Add implementation. +} + +void ResidualEchoDetector::AnalyzeCaptureAudio( + rtc::ArrayView render_audio) { + // TODO(ivoc): Add implementation. +} + +void ResidualEchoDetector::Initialize() { + // TODO(ivoc): Add implementation. +} + +void ResidualEchoDetector::PackRenderAudioBuffer( + AudioBuffer* audio, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + + packed_buffer->clear(); + packed_buffer->insert(packed_buffer->end(), + audio->split_bands_const_f(0)[kBand0To8kHz], + (audio->split_bands_const_f(0)[kBand0To8kHz] + + audio->num_frames_per_band())); +} + +float ResidualEchoDetector::get_echo_likelihood() const { + // TODO(ivoc): Add implementation. + return 0.0f; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/residual_echo_detector.h b/webrtc/modules/audio_processing/residual_echo_detector.h new file mode 100644 index 0000000000..723371c7ce --- /dev/null +++ b/webrtc/modules/audio_processing/residual_echo_detector.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ + +#include +#include + +#include "webrtc/base/array_view.h" + +namespace webrtc { + +class AudioBuffer; +class EchoDetector; + +class ResidualEchoDetector { + public: + ResidualEchoDetector(); + ~ResidualEchoDetector(); + + // This function should be called while holding the render lock. + void AnalyzeRenderAudio(rtc::ArrayView render_audio) const; + + // This function should be called while holding the capture lock. + void AnalyzeCaptureAudio(rtc::ArrayView capture_audio); + + // This function should be called while holding the capture lock. + void Initialize(); + + static void PackRenderAudioBuffer(AudioBuffer* audio, + std::vector* packed_buffer); + + // This function should be called while holding the capture lock. + float get_echo_likelihood() const; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_