From b624d8c852fbe8eff8ff8888f673191b8a411421 Mon Sep 17 00:00:00 2001 From: peah Date: Sat, 5 Mar 2016 03:01:14 -0800 Subject: [PATCH] Removed the inheritance from ProcessingComponent for EchoCancellerImpl. BUG=webrtc:5352 Committed: https://crrev.com/3af0a009f8a7f2dfb630a4f4730044cbbd95bee8 Cr-Commit-Position: refs/heads/master@{#11876} Review URL: https://codereview.webrtc.org/1761813002 Cr-Commit-Position: refs/heads/master@{#11881} --- .../audio_processing/audio_processing_impl.cc | 21 ++- .../audio_processing/audio_processing_impl.h | 2 + .../echo_cancellation_impl.cc | 168 ++++++++++-------- .../audio_processing/echo_cancellation_impl.h | 24 +-- 4 files changed, 122 insertions(+), 93 deletions(-) diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index a92f13c671..9c587c25c7 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -85,7 +85,7 @@ struct AudioProcessingImpl::ApmPublicSubmodules { echo_control_mobile(nullptr), gain_control(nullptr) {} // Accessed externally of APM without any lock acquired. - EchoCancellationImpl* echo_cancellation; + std::unique_ptr echo_cancellation; EchoControlMobileImpl* echo_control_mobile; GainControlImpl* gain_control; std::unique_ptr high_pass_filter; @@ -168,8 +168,8 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config, rtc::CritScope cs_render(&crit_render_); rtc::CritScope cs_capture(&crit_capture_); - public_submodules_->echo_cancellation = - new EchoCancellationImpl(this, &crit_render_, &crit_capture_); + public_submodules_->echo_cancellation.reset( + new EchoCancellationImpl(this, &crit_render_, &crit_capture_)); public_submodules_->echo_control_mobile = new EchoControlMobileImpl(this, &crit_render_, &crit_capture_); public_submodules_->gain_control = @@ -185,9 +185,6 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config, public_submodules_->gain_control_for_experimental_agc.reset( new GainControlForExperimentalAgc(public_submodules_->gain_control, &crit_capture_)); - - private_submodules_->component_list.push_back( - public_submodules_->echo_cancellation); private_submodules_->component_list.push_back( public_submodules_->echo_control_mobile); private_submodules_->component_list.push_back( @@ -322,6 +319,7 @@ int AudioProcessingImpl::InitializeLocked() { } } + InitializeEchoCanceller(); InitializeExperimentalAgc(); InitializeTransient(); InitializeBeamformer(); @@ -423,6 +421,8 @@ void AudioProcessingImpl::SetExtraOptions(const Config& config) { item->SetExtraOptions(config); } + public_submodules_->echo_cancellation->SetExtraOptions(config); + if (capture_.transient_suppressor_enabled != config.Get().enabled) { capture_.transient_suppressor_enabled = @@ -1087,7 +1087,7 @@ int AudioProcessingImpl::StopDebugRecording() { EchoCancellation* AudioProcessingImpl::echo_cancellation() const { // Adding a lock here has no effect as it allows any access to the submodule // from the returned pointer. - return public_submodules_->echo_cancellation; + return public_submodules_->echo_cancellation.get(); } EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { @@ -1134,7 +1134,8 @@ bool AudioProcessingImpl::is_data_processed() const { // modify the data. if (capture_nonlocked_.beamformer_enabled || public_submodules_->high_pass_filter->is_enabled() || - public_submodules_->noise_suppression->is_enabled()) { + public_submodules_->noise_suppression->is_enabled() || + public_submodules_->echo_cancellation->is_enabled()) { return true; } @@ -1249,6 +1250,10 @@ void AudioProcessingImpl::InitializeNoiseSuppression() { proc_sample_rate_hz()); } +void AudioProcessingImpl::InitializeEchoCanceller() { + public_submodules_->echo_cancellation->Initialize(); +} + void AudioProcessingImpl::InitializeLevelEstimator() { public_submodules_->level_estimator->Initialize(); } diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index 4a28761af1..560920b1e1 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -196,6 +196,8 @@ class AudioProcessingImpl : public AudioProcessing { EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); void InitializeVoiceDetection() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeEchoCanceller() + EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); int InitializeLocked(const ProcessingConfig& config) EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.cc b/webrtc/modules/audio_processing/echo_cancellation_impl.cc index 99def9336c..cea905515a 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.cc +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.cc @@ -59,11 +59,38 @@ static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; static const size_t kMaxNumFramesToBuffer = 100; } // namespace +class EchoCancellationImpl::Canceller { + public: + explicit Canceller(int sample_rate_hz) { + state_ = WebRtcAec_Create(); + RTC_DCHECK(state_); + } + + ~Canceller() { + RTC_CHECK(state_); + WebRtcAec_Free(state_); + } + + Handle* state() { return state_; } + + void Initialize(int sample_rate_hz) { + // TODO(ajm): Drift compensation is disabled in practice. If restored, it + // should be managed internally and not depend on the hardware sample rate. + // For now, just hardcode a 48 kHz value. + const int error = WebRtcAec_Init(state_, sample_rate_hz, 48000); + RTC_DCHECK_EQ(0, error); + } + + private: + Handle* state_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(Canceller); +}; + EchoCancellationImpl::EchoCancellationImpl(const AudioProcessing* apm, rtc::CriticalSection* crit_render, rtc::CriticalSection* crit_capture) - : ProcessingComponent(), - apm_(apm), + : apm_(apm), crit_render_(crit_render), crit_capture_(crit_capture), drift_compensation_enabled_(false), @@ -86,12 +113,14 @@ EchoCancellationImpl::~EchoCancellationImpl() {} int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) { rtc::CritScope cs_render(crit_render_); - if (!is_component_enabled()) { + if (!enabled_) { return AudioProcessing::kNoError; } - assert(audio->num_frames_per_band() <= 160); - assert(audio->num_channels() == apm_->num_reverse_channels()); + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + RTC_DCHECK_EQ(audio->num_channels(), apm_->num_reverse_channels()); + RTC_DCHECK_GE(cancellers_.size(), + apm_->num_output_channels() * audio->num_channels()); int err = AudioProcessing::kNoError; @@ -100,9 +129,9 @@ int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) { render_queue_buffer_.clear(); for (size_t i = 0; i < apm_->num_output_channels(); i++) { for (size_t j = 0; j < audio->num_channels(); j++) { - Handle* my_handle = static_cast(handle(handle_index)); + Handle* my_handle = cancellers_[handle_index++]->state(); // Retrieve any error code produced by the buffering of the farend - // signal + // signal. err = WebRtcAec_GetBufferFarendError( my_handle, audio->split_bands_const_f(j)[kBand0To8kHz], audio->num_frames_per_band()); @@ -116,8 +145,6 @@ int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) { audio->split_bands_const_f(j)[kBand0To8kHz], (audio->split_bands_const_f(j)[kBand0To8kHz] + audio->num_frames_per_band())); - - handle_index++; } } @@ -137,7 +164,7 @@ int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) { // a queue. All the data chunks are buffered into the farend signal of the AEC. void EchoCancellationImpl::ReadQueuedRenderData() { rtc::CritScope cs_capture(crit_capture_); - if (!is_component_enabled()) { + if (!enabled_) { return; } @@ -149,12 +176,11 @@ void EchoCancellationImpl::ReadQueuedRenderData() { (apm_->num_output_channels() * apm_->num_reverse_channels()); for (size_t i = 0; i < apm_->num_output_channels(); i++) { for (size_t j = 0; j < apm_->num_reverse_channels(); j++) { - Handle* my_handle = static_cast(handle(handle_index)); + Handle* my_handle = cancellers_[handle_index++]->state(); WebRtcAec_BufferFarend(my_handle, &capture_queue_buffer_[buffer_index], num_frames_per_band); buffer_index += num_frames_per_band; - handle_index++; } } } @@ -162,7 +188,7 @@ void EchoCancellationImpl::ReadQueuedRenderData() { int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) { rtc::CritScope cs_capture(crit_capture_); - if (!is_component_enabled()) { + if (!enabled_) { return AudioProcessing::kNoError; } @@ -174,8 +200,8 @@ int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) { return AudioProcessing::kStreamParameterNotSetError; } - assert(audio->num_frames_per_band() <= 160); - assert(audio->num_channels() == apm_->num_proc_channels()); + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + RTC_DCHECK_EQ(audio->num_channels(), apm_->num_proc_channels()); int err = AudioProcessing::kNoError; @@ -184,7 +210,7 @@ int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) { stream_has_echo_ = false; for (size_t i = 0; i < audio->num_channels(); i++) { for (size_t j = 0; j < apm_->num_reverse_channels(); j++) { - Handle* my_handle = handle(handle_index); + Handle* my_handle = cancellers_[handle_index++]->state(); err = WebRtcAec_Process(my_handle, audio->split_bands_const_f(i), audio->num_bands(), audio->split_bands_f(i), audio->num_frames_per_band(), @@ -207,8 +233,6 @@ int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) { if (status == 1) { stream_has_echo_ = true; } - - handle_index++; } } @@ -227,12 +251,18 @@ int EchoCancellationImpl::Enable(bool enable) { return AudioProcessing::kBadParameterError; } - return EnableComponent(enable); + if (enable && !enabled_) { + enabled_ = enable; // Must be set before Initialize() is called. + Initialize(); + } else { + enabled_ = enable; + } + return AudioProcessing::kNoError; } bool EchoCancellationImpl::is_enabled() const { rtc::CritScope cs(crit_capture_); - return is_component_enabled(); + return enabled_; } int EchoCancellationImpl::set_suppression_level(SuppressionLevel level) { @@ -297,7 +327,7 @@ int EchoCancellationImpl::GetMetrics(Metrics* metrics) { return AudioProcessing::kNullPointerError; } - if (!is_component_enabled() || !metrics_enabled_) { + if (!enabled_ || !metrics_enabled_) { return AudioProcessing::kNotEnabledError; } @@ -305,7 +335,7 @@ int EchoCancellationImpl::GetMetrics(Metrics* metrics) { memset(&my_metrics, 0, sizeof(my_metrics)); memset(metrics, 0, sizeof(Metrics)); - Handle* my_handle = static_cast(handle(0)); + Handle* my_handle = cancellers_[0]->state(); int err = WebRtcAec_GetMetrics(my_handle, &my_metrics); if (err != AudioProcessing::kNoError) { return MapError(err); @@ -384,11 +414,11 @@ int EchoCancellationImpl::GetDelayMetrics(int* median, int* std, return AudioProcessing::kNullPointerError; } - if (!is_component_enabled() || !delay_logging_enabled_) { + if (!enabled_ || !delay_logging_enabled_) { return AudioProcessing::kNotEnabledError; } - Handle* my_handle = static_cast(handle(0)); + Handle* my_handle = cancellers_[0]->state(); const int err = WebRtcAec_GetDelayMetrics(my_handle, median, std, fraction_poor_delays); if (err != AudioProcessing::kNoError) { @@ -400,33 +430,45 @@ int EchoCancellationImpl::GetDelayMetrics(int* median, int* std, struct AecCore* EchoCancellationImpl::aec_core() const { rtc::CritScope cs(crit_capture_); - if (!is_component_enabled()) { + if (!enabled_) { return NULL; } - Handle* my_handle = static_cast(handle(0)); + Handle* my_handle = cancellers_[0]->state(); return WebRtcAec_aec_core(my_handle); } -int EchoCancellationImpl::Initialize() { - int err = ProcessingComponent::Initialize(); - { - rtc::CritScope cs(crit_capture_); - if (err != AudioProcessing::kNoError || !is_component_enabled()) { - return err; +void EchoCancellationImpl::Initialize() { + rtc::CritScope cs_render(crit_render_); + rtc::CritScope cs_capture(crit_capture_); + if (!enabled_) { + return; + } + const int sample_rate_hz = apm_->proc_sample_rate_hz(); + + if (num_handles_required() > cancellers_.size()) { + const size_t cancellers_old_size = cancellers_.size(); + cancellers_.resize(num_handles_required()); + + for (size_t i = cancellers_old_size; i < cancellers_.size(); ++i) { + cancellers_[i].reset(new Canceller(sample_rate_hz)); } } - AllocateRenderQueue(); + for (size_t i = 0; i < cancellers_.size(); ++i) { + cancellers_[i]->Initialize(sample_rate_hz); + } - return AudioProcessing::kNoError; + Configure(); + + AllocateRenderQueue(); } int EchoCancellationImpl::GetSystemDelayInSamples() const { rtc::CritScope cs(crit_capture_); - RTC_DCHECK(is_component_enabled()); + RTC_DCHECK(enabled_); // Report the delay for the first AEC component. return WebRtcAec_system_delay( - WebRtcAec_aec_core(static_cast(handle(0)))); + WebRtcAec_aec_core(cancellers_[0]->state())); } void EchoCancellationImpl::AllocateRenderQueue() { @@ -466,45 +508,30 @@ void EchoCancellationImpl::SetExtraOptions(const Config& config) { Configure(); } -void* EchoCancellationImpl::CreateHandle() const { - return WebRtcAec_Create(); -} - -void EchoCancellationImpl::DestroyHandle(void* handle) const { - assert(handle != NULL); - WebRtcAec_Free(static_cast(handle)); -} - -int EchoCancellationImpl::InitializeHandle(void* handle) const { - // Not locked as it only relies on APM public API which is threadsafe. - - assert(handle != NULL); - // TODO(ajm): Drift compensation is disabled in practice. If restored, it - // should be managed internally and not depend on the hardware sample rate. - // For now, just hardcode a 48 kHz value. - return WebRtcAec_Init(static_cast(handle), - apm_->proc_sample_rate_hz(), 48000); -} - -int EchoCancellationImpl::ConfigureHandle(void* handle) const { +int EchoCancellationImpl::Configure() { rtc::CritScope cs_render(crit_render_); rtc::CritScope cs_capture(crit_capture_); - assert(handle != NULL); AecConfig config; config.metricsMode = metrics_enabled_; config.nlpMode = MapSetting(suppression_level_); config.skewMode = drift_compensation_enabled_; config.delay_logging = delay_logging_enabled_; - WebRtcAec_enable_extended_filter( - WebRtcAec_aec_core(static_cast(handle)), - extended_filter_enabled_ ? 1 : 0); - WebRtcAec_enable_delay_agnostic( - WebRtcAec_aec_core(static_cast(handle)), - delay_agnostic_enabled_ ? 1 : 0); - WebRtcAec_enable_next_generation_aec( - WebRtcAec_aec_core(static_cast(handle)), - next_generation_aec_enabled_ ? 1 : 0); - return WebRtcAec_set_config(static_cast(handle), config); + + int error = AudioProcessing::kNoError; + for (size_t i = 0; i < cancellers_.size(); i++) { + Handle* my_handle = cancellers_[i]->state(); + WebRtcAec_enable_extended_filter(WebRtcAec_aec_core(my_handle), + extended_filter_enabled_ ? 1 : 0); + WebRtcAec_enable_delay_agnostic(WebRtcAec_aec_core(my_handle), + delay_agnostic_enabled_ ? 1 : 0); + WebRtcAec_enable_next_generation_aec(WebRtcAec_aec_core(my_handle), + next_generation_aec_enabled_ ? 1 : 0); + const int handle_error = WebRtcAec_set_config(my_handle, config); + if (handle_error != AudioProcessing::kNoError) { + error = AudioProcessing::kNoError; + } + } + return error; } size_t EchoCancellationImpl::num_handles_required() const { @@ -512,9 +539,4 @@ size_t EchoCancellationImpl::num_handles_required() const { return apm_->num_output_channels() * apm_->num_reverse_channels(); } -int EchoCancellationImpl::GetHandleError(void* handle) const { - // Not locked as it does not rely on anything in the state. - assert(handle != NULL); - return AudioProcessing::kUnspecifiedError; -} } // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.h b/webrtc/modules/audio_processing/echo_cancellation_impl.h index 2ef6d52fe6..ff7c3ae319 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.h +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.h @@ -13,6 +13,7 @@ #include +#include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" #include "webrtc/common_audio/swap_queue.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" @@ -22,8 +23,7 @@ namespace webrtc { class AudioBuffer; -class EchoCancellationImpl : public EchoCancellation, - public ProcessingComponent { +class EchoCancellationImpl : public EchoCancellation { public: EchoCancellationImpl(const AudioProcessing* apm, rtc::CriticalSection* crit_render, @@ -39,9 +39,8 @@ class EchoCancellationImpl : public EchoCancellation, SuppressionLevel suppression_level() const override; bool is_drift_compensation_enabled() const override; - // ProcessingComponent implementation. - int Initialize() override; - void SetExtraOptions(const Config& config) override; + void Initialize(); + void SetExtraOptions(const Config& config); bool is_delay_agnostic_enabled() const; bool is_extended_filter_enabled() const; bool is_next_generation_aec_enabled() const; @@ -54,6 +53,8 @@ class EchoCancellationImpl : public EchoCancellation, int GetSystemDelayInSamples() const; private: + class Canceller; + // EchoCancellation implementation. int Enable(bool enable) override; int enable_drift_compensation(bool enable) override; @@ -72,15 +73,10 @@ class EchoCancellationImpl : public EchoCancellation, struct AecCore* aec_core() const override; - // ProcessingComponent implementation. - void* CreateHandle() const override; - int InitializeHandle(void* handle) const override; - int ConfigureHandle(void* handle) const override; - void DestroyHandle(void* handle) const override; - size_t num_handles_required() const override; - int GetHandleError(void* handle) const override; + size_t num_handles_required() const; void AllocateRenderQueue(); + int Configure(); // Not guarded as its public API is thread safe. const AudioProcessing* apm_; @@ -88,6 +84,7 @@ class EchoCancellationImpl : public EchoCancellation, rtc::CriticalSection* const crit_render_ ACQUIRED_BEFORE(crit_capture_); rtc::CriticalSection* const crit_capture_; + bool enabled_ = false; bool drift_compensation_enabled_ GUARDED_BY(crit_capture_); bool metrics_enabled_ GUARDED_BY(crit_capture_); SuppressionLevel suppression_level_ GUARDED_BY(crit_capture_); @@ -107,6 +104,9 @@ class EchoCancellationImpl : public EchoCancellation, // Lock protection not needed. std::unique_ptr, RenderQueueItemVerifier>> render_signal_queue_; + + std::vector> cancellers_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(EchoCancellationImpl); }; } // namespace webrtc