diff --git a/webrtc/modules/audio_processing/BUILD.gn b/webrtc/modules/audio_processing/BUILD.gn index bd6adf6542..4090f6d38b 100644 --- a/webrtc/modules/audio_processing/BUILD.gn +++ b/webrtc/modules/audio_processing/BUILD.gn @@ -99,6 +99,8 @@ rtc_static_library("audio_processing") { "noise_suppression_impl.cc", "noise_suppression_impl.h", "render_queue_item_verifier.h", + "residual_echo_detector.cc", + "residual_echo_detector.h", "rms_level.cc", "rms_level.h", "splitting_filter.cc", diff --git a/webrtc/modules/audio_processing/audio_processing.gypi b/webrtc/modules/audio_processing/audio_processing.gypi index d7d84d1695..d25f9e578e 100644 --- a/webrtc/modules/audio_processing/audio_processing.gypi +++ b/webrtc/modules/audio_processing/audio_processing.gypi @@ -111,6 +111,8 @@ 'noise_suppression_impl.cc', 'noise_suppression_impl.h', 'render_queue_item_verifier.h', + 'residual_echo_detector.cc', + 'residual_echo_detector.h', 'rms_level.cc', 'rms_level.h', 'splitting_filter.cc', diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index acbc80c9dd..32f1d9f5a1 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -35,6 +35,7 @@ #include "webrtc/modules/audio_processing/level_controller/level_controller.h" #include "webrtc/modules/audio_processing/level_estimator_impl.h" #include "webrtc/modules/audio_processing/noise_suppression_impl.h" +#include "webrtc/modules/audio_processing/residual_echo_detector.h" #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" #include "webrtc/modules/audio_processing/voice_detection_impl.h" #include "webrtc/modules/include/module_common_types.h" @@ -137,6 +138,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update( bool high_pass_filter_enabled, bool echo_canceller_enabled, bool mobile_echo_controller_enabled, + bool residual_echo_detector_enabled, bool noise_suppressor_enabled, bool intelligibility_enhancer_enabled, bool beamformer_enabled, @@ -150,6 +152,8 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update( changed |= (echo_canceller_enabled != echo_canceller_enabled_); changed |= (mobile_echo_controller_enabled != mobile_echo_controller_enabled_); + changed |= + (residual_echo_detector_enabled != residual_echo_detector_enabled_); changed |= (noise_suppressor_enabled != noise_suppressor_enabled_); changed |= (intelligibility_enhancer_enabled != intelligibility_enhancer_enabled_); @@ -165,6 +169,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update( high_pass_filter_enabled_ = high_pass_filter_enabled; echo_canceller_enabled_ = echo_canceller_enabled; mobile_echo_controller_enabled_ = mobile_echo_controller_enabled; + residual_echo_detector_enabled_ = residual_echo_detector_enabled; noise_suppressor_enabled_ = noise_suppressor_enabled; intelligibility_enhancer_enabled_ = intelligibility_enhancer_enabled; beamformer_enabled_ = beamformer_enabled; @@ -239,6 +244,7 @@ struct AudioProcessingImpl::ApmPrivateSubmodules { std::unique_ptr beamformer; std::unique_ptr agc_manager; std::unique_ptr level_controller; + std::unique_ptr residual_echo_detector; }; AudioProcessing* AudioProcessing::Create() { @@ -304,6 +310,8 @@ AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config, public_submodules_->gain_control_for_experimental_agc.reset( new GainControlForExperimentalAgc( public_submodules_->gain_control.get(), &crit_capture_)); + private_submodules_->residual_echo_detector.reset( + new ResidualEchoDetector()); // TODO(peah): Move this creation to happen only when the level controller // is enabled. @@ -470,6 +478,7 @@ int AudioProcessingImpl::InitializeLocked() { public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); public_submodules_->level_estimator->Initialize(); InitializeLevelController(); + InitializeResidualEchoDetector(); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP if (debug_dump_.debug_file->is_open()) { @@ -809,6 +818,18 @@ void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) { RTC_DCHECK(result); } } + + ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_); + + // Insert the samples into the queue. + if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_); + RTC_DCHECK(result); + } } void AudioProcessingImpl::AllocateRenderQueue() { @@ -827,6 +848,9 @@ void AudioProcessingImpl::AllocateRenderQueue() { const size_t new_agc_render_queue_element_max_size = std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame); + const size_t new_red_render_queue_element_max_size = + std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame); + // Reallocate the queues if the queue item sizes are too small to fit the // data to put in the queues. if (aec_render_queue_element_max_size_ < @@ -886,6 +910,25 @@ void AudioProcessingImpl::AllocateRenderQueue() { } else { agc_render_signal_queue_->Clear(); } + + if (red_render_queue_element_max_size_ < + new_red_render_queue_element_max_size) { + red_render_queue_element_max_size_ = new_red_render_queue_element_max_size; + + std::vector template_queue_element( + red_render_queue_element_max_size_); + + red_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + red_render_queue_element_max_size_))); + + red_render_queue_buffer_.resize(red_render_queue_element_max_size_); + red_capture_queue_buffer_.resize(red_render_queue_element_max_size_); + } else { + red_render_signal_queue_->Clear(); + } } void AudioProcessingImpl::EmptyQueuedRenderAudio() { @@ -904,6 +947,11 @@ void AudioProcessingImpl::EmptyQueuedRenderAudio() { public_submodules_->gain_control->ProcessRenderAudio( agc_capture_queue_buffer_); } + + while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) { + private_submodules_->residual_echo_detector->AnalyzeRenderAudio( + red_capture_queue_buffer_); + } } int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { @@ -1079,6 +1127,13 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( capture_buffer, stream_delay_ms())); + if (config_.residual_echo_detector.enabled) { + private_submodules_->residual_echo_detector->AnalyzeCaptureAudio( + rtc::ArrayView( + capture_buffer->split_bands_const_f(0)[kBand0To8kHz], + capture_buffer->num_frames_per_band())); + } + if (capture_nonlocked_.beamformer_enabled) { private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f()); } @@ -1454,6 +1509,7 @@ bool AudioProcessingImpl::UpdateActiveSubmoduleStates() { public_submodules_->high_pass_filter->is_enabled(), public_submodules_->echo_cancellation->is_enabled(), public_submodules_->echo_control_mobile->is_enabled(), + config_.residual_echo_detector.enabled, public_submodules_->noise_suppression->is_enabled(), capture_nonlocked_.intelligibility_enabled, capture_nonlocked_.beamformer_enabled, @@ -1503,6 +1559,10 @@ void AudioProcessingImpl::InitializeLevelController() { private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); } +void AudioProcessingImpl::InitializeResidualEchoDetector() { + private_submodules_->residual_echo_detector->Initialize(); +} + void AudioProcessingImpl::MaybeUpdateHistograms() { static const int kMinDiffDelayMs = 60; diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index aec4ed7086..fd8683fb57 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -150,6 +150,7 @@ class AudioProcessingImpl : public AudioProcessing { bool Update(bool high_pass_filter_enabled, bool echo_canceller_enabled, bool mobile_echo_controller_enabled, + bool residual_echo_detector_enabled, bool noise_suppressor_enabled, bool intelligibility_enhancer_enabled, bool beamformer_enabled, @@ -167,6 +168,7 @@ class AudioProcessingImpl : public AudioProcessing { bool high_pass_filter_enabled_ = false; bool echo_canceller_enabled_ = false; bool mobile_echo_controller_enabled_ = false; + bool residual_echo_detector_enabled_ = false; bool noise_suppressor_enabled_ = false; bool intelligibility_enhancer_enabled_ = false; bool beamformer_enabled_ = false; @@ -234,6 +236,8 @@ class AudioProcessingImpl : public AudioProcessing { int InitializeLocked(const ProcessingConfig& config) EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void InitializeLevelController() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeResidualEchoDetector() + EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void EmptyQueuedRenderAudio(); void AllocateRenderQueue() @@ -291,8 +295,7 @@ class AudioProcessingImpl : public AudioProcessing { // Structs containing the pointers to the submodules. std::unique_ptr public_submodules_; - std::unique_ptr private_submodules_ - GUARDED_BY(crit_capture_); + std::unique_ptr private_submodules_; // State that is written to while holding both the render and capture locks // but can be read without any lock being held. @@ -386,6 +389,11 @@ class AudioProcessingImpl : public AudioProcessing { std::vector agc_render_queue_buffer_ GUARDED_BY(crit_render_); std::vector agc_capture_queue_buffer_ GUARDED_BY(crit_capture_); + size_t red_render_queue_element_max_size_ GUARDED_BY(crit_render_) + GUARDED_BY(crit_capture_) = 0; + std::vector red_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector red_capture_queue_buffer_ GUARDED_BY(crit_capture_); + // Lock protection not needed. std::unique_ptr, RenderQueueItemVerifier>> aec_render_signal_queue_; @@ -395,6 +403,8 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr< SwapQueue, RenderQueueItemVerifier>> agc_render_signal_queue_; + std::unique_ptr, RenderQueueItemVerifier>> + red_render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/include/audio_processing.h b/webrtc/modules/audio_processing/include/audio_processing.h index 0fdbfd2d75..7629717247 100644 --- a/webrtc/modules/audio_processing/include/audio_processing.h +++ b/webrtc/modules/audio_processing/include/audio_processing.h @@ -258,6 +258,9 @@ class AudioProcessing { // the allowed range is [-100, 0]. float initial_peak_level_dbfs = -6.0206f; } level_controller; + struct ResidualEchoDetector { + bool enabled = false; + } residual_echo_detector; }; // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone. diff --git a/webrtc/modules/audio_processing/residual_echo_detector.cc b/webrtc/modules/audio_processing/residual_echo_detector.cc new file mode 100644 index 0000000000..df99de45bf --- /dev/null +++ b/webrtc/modules/audio_processing/residual_echo_detector.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/residual_echo_detector.h" + +#include "webrtc/modules/audio_processing/audio_buffer.h" + +namespace webrtc { + +ResidualEchoDetector::ResidualEchoDetector() = default; + +ResidualEchoDetector::~ResidualEchoDetector() = default; + +void ResidualEchoDetector::AnalyzeRenderAudio( + rtc::ArrayView render_audio) const { + // TODO(ivoc): Add implementation. +} + +void ResidualEchoDetector::AnalyzeCaptureAudio( + rtc::ArrayView render_audio) { + // TODO(ivoc): Add implementation. +} + +void ResidualEchoDetector::Initialize() { + // TODO(ivoc): Add implementation. +} + +void ResidualEchoDetector::PackRenderAudioBuffer( + AudioBuffer* audio, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + + packed_buffer->clear(); + packed_buffer->insert(packed_buffer->end(), + audio->split_bands_const_f(0)[kBand0To8kHz], + (audio->split_bands_const_f(0)[kBand0To8kHz] + + audio->num_frames_per_band())); +} + +float ResidualEchoDetector::get_echo_likelihood() const { + // TODO(ivoc): Add implementation. + return 0.0f; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/residual_echo_detector.h b/webrtc/modules/audio_processing/residual_echo_detector.h new file mode 100644 index 0000000000..723371c7ce --- /dev/null +++ b/webrtc/modules/audio_processing/residual_echo_detector.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ + +#include +#include + +#include "webrtc/base/array_view.h" + +namespace webrtc { + +class AudioBuffer; +class EchoDetector; + +class ResidualEchoDetector { + public: + ResidualEchoDetector(); + ~ResidualEchoDetector(); + + // This function should be called while holding the render lock. + void AnalyzeRenderAudio(rtc::ArrayView render_audio) const; + + // This function should be called while holding the capture lock. + void AnalyzeCaptureAudio(rtc::ArrayView capture_audio); + + // This function should be called while holding the capture lock. + void Initialize(); + + static void PackRenderAudioBuffer(AudioBuffer* audio, + std::vector* packed_buffer); + + // This function should be called while holding the capture lock. + float get_echo_likelihood() const; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_