diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index a38d73e2dc..805e05eeee 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -76,38 +76,6 @@ static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { } } // namespace -struct AudioProcessingImpl::ApmPublicSubmodules { - ApmPublicSubmodules() - : echo_cancellation(nullptr), - echo_control_mobile(nullptr), - gain_control(nullptr), - level_estimator(nullptr), - noise_suppression(nullptr), - voice_detection(nullptr) {} - // Accessed externally of APM without any lock acquired. - EchoCancellationImpl* echo_cancellation; - EchoControlMobileImpl* echo_control_mobile; - GainControlImpl* gain_control; - rtc::scoped_ptr high_pass_filter; - LevelEstimatorImpl* level_estimator; - NoiseSuppressionImpl* noise_suppression; - VoiceDetectionImpl* voice_detection; - rtc::scoped_ptr gain_control_for_new_agc; - - // Accessed internally from both render and capture. - rtc::scoped_ptr transient_suppressor; - rtc::scoped_ptr intelligibility_enhancer; -}; - -struct AudioProcessingImpl::ApmPrivateSubmodules { - explicit ApmPrivateSubmodules(Beamformer* beamformer) - : beamformer(beamformer) {} - // Accessed internally from capture or during initialization - std::list component_list; - rtc::scoped_ptr> beamformer; - rtc::scoped_ptr agc_manager; -}; - // Throughout webrtc, it's assumed that success is represented by zero. static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); @@ -177,6 +145,37 @@ class GainControlForNewAgc : public GainControl, public VolumeCallbacks { int volume_; }; +struct AudioProcessingImpl::ApmPublicSubmodules { + ApmPublicSubmodules() + : echo_cancellation(nullptr), + echo_control_mobile(nullptr), + gain_control(nullptr), + level_estimator(nullptr), + voice_detection(nullptr) {} + // Accessed externally of APM without any lock acquired. + EchoCancellationImpl* echo_cancellation; + EchoControlMobileImpl* echo_control_mobile; + GainControlImpl* gain_control; + rtc::scoped_ptr high_pass_filter; + LevelEstimatorImpl* level_estimator; + rtc::scoped_ptr noise_suppression; + VoiceDetectionImpl* voice_detection; + rtc::scoped_ptr gain_control_for_new_agc; + + // Accessed internally from both render and capture. + rtc::scoped_ptr transient_suppressor; + rtc::scoped_ptr intelligibility_enhancer; +}; + +struct AudioProcessingImpl::ApmPrivateSubmodules { + explicit ApmPrivateSubmodules(Beamformer* beamformer) + : beamformer(beamformer) {} + // Accessed internally from capture or during initialization + std::list component_list; + rtc::scoped_ptr> beamformer; + rtc::scoped_ptr agc_manager; +}; + const int AudioProcessing::kNativeSampleRatesHz[] = { AudioProcessing::kSampleRate8kHz, AudioProcessing::kSampleRate16kHz, @@ -246,8 +245,8 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config, new HighPassFilterImpl(&crit_capture_)); public_submodules_->level_estimator = new LevelEstimatorImpl(this, &crit_capture_); - public_submodules_->noise_suppression = - new NoiseSuppressionImpl(this, &crit_capture_); + public_submodules_->noise_suppression.reset( + new NoiseSuppressionImpl(&crit_capture_)); public_submodules_->voice_detection = new VoiceDetectionImpl(this, &crit_capture_); public_submodules_->gain_control_for_new_agc.reset( @@ -261,8 +260,6 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config, public_submodules_->gain_control); private_submodules_->component_list.push_back( public_submodules_->level_estimator); - private_submodules_->component_list.push_back( - public_submodules_->noise_suppression); private_submodules_->component_list.push_back( public_submodules_->voice_detection); } @@ -396,14 +393,11 @@ int AudioProcessingImpl::InitializeLocked() { } InitializeExperimentalAgc(); - InitializeTransient(); - InitializeBeamformer(); - InitializeIntelligibility(); - InitializeHighPassFilter(); + InitializeNoiseSuppression(); #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP if (debug_dump_.debug_file->Open()) { @@ -768,14 +762,14 @@ int AudioProcessingImpl::ProcessStreamLocked() { public_submodules_->high_pass_filter->ProcessCaptureAudio(ca); RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); - RETURN_ON_ERR(public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca)); + public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca); RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(ca)); if (public_submodules_->echo_control_mobile->is_enabled() && public_submodules_->noise_suppression->is_enabled()) { ca->CopyLowPassToReference(); } - RETURN_ON_ERR(public_submodules_->noise_suppression->ProcessCaptureAudio(ca)); + public_submodules_->noise_suppression->ProcessCaptureAudio(ca); RETURN_ON_ERR( public_submodules_->echo_control_mobile->ProcessCaptureAudio(ca)); RETURN_ON_ERR(public_submodules_->voice_detection->ProcessCaptureAudio(ca)); @@ -1158,7 +1152,7 @@ LevelEstimator* AudioProcessingImpl::level_estimator() const { NoiseSuppression* AudioProcessingImpl::noise_suppression() const { // Adding a lock here has no effect as it allows any access to the submodule // from the returned pointer. - return public_submodules_->noise_suppression; + return public_submodules_->noise_suppression.get(); } VoiceDetection* AudioProcessingImpl::voice_detection() const { @@ -1181,6 +1175,9 @@ bool AudioProcessingImpl::is_data_processed() const { if (public_submodules_->high_pass_filter->is_enabled()) { enabled_count++; } + if (public_submodules_->noise_suppression->is_enabled()) { + enabled_count++; + } // Data is unchanged if no components are enabled, or if only // public_submodules_->level_estimator @@ -1300,6 +1297,11 @@ void AudioProcessingImpl::InitializeHighPassFilter() { proc_sample_rate_hz()); } +void AudioProcessingImpl::InitializeNoiseSuppression() { + public_submodules_->noise_suppression->Initialize(num_output_channels(), + proc_sample_rate_hz()); +} + void AudioProcessingImpl::MaybeUpdateHistograms() { static const int kMinDiffDelayMs = 60; diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index 1ca5a8c19e..c4d5f08403 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -39,18 +39,6 @@ class AudioConverter; template class Beamformer; -class EchoCancellationImpl; -class EchoControlMobileImpl; -class GainControlImpl; -class GainControlForNewAgc; -class HighPassFilterImpl; -class LevelEstimatorImpl; -class NoiseSuppressionImpl; -class ProcessingComponent; -class TransientSuppressor; -class VoiceDetectionImpl; -class IntelligibilityEnhancer; - class AudioProcessingImpl : public AudioProcessing { public: // Methods forcing APM to run in a single-threaded manner. @@ -195,6 +183,8 @@ class AudioProcessingImpl : public AudioProcessing { EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void InitializeHighPassFilter() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeNoiseSuppression() + EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); int InitializeLocked(const ProcessingConfig& config) EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); diff --git a/webrtc/modules/audio_processing/high_pass_filter_impl.cc b/webrtc/modules/audio_processing/high_pass_filter_impl.cc index 795dcbd21c..0db794f94e 100644 --- a/webrtc/modules/audio_processing/high_pass_filter_impl.cc +++ b/webrtc/modules/audio_processing/high_pass_filter_impl.cc @@ -89,6 +89,7 @@ HighPassFilterImpl::HighPassFilterImpl(rtc::CriticalSection* crit) HighPassFilterImpl::~HighPassFilterImpl() {} void HighPassFilterImpl::Initialize(int channels, int sample_rate_hz) { + RTC_DCHECK_LE(0, channels); std::vector> new_filters(channels); for (int i = 0; i < channels; i++) { new_filters[i].reset(new BiquadFilter(sample_rate_hz)); @@ -98,6 +99,7 @@ void HighPassFilterImpl::Initialize(int channels, int sample_rate_hz) { } void HighPassFilterImpl::ProcessCaptureAudio(AudioBuffer* audio) { + RTC_DCHECK(audio); rtc::CritScope cs(crit_); if (!enabled_) { return; diff --git a/webrtc/modules/audio_processing/high_pass_filter_impl.h b/webrtc/modules/audio_processing/high_pass_filter_impl.h index b2a0717374..d6e84fd280 100644 --- a/webrtc/modules/audio_processing/high_pass_filter_impl.h +++ b/webrtc/modules/audio_processing/high_pass_filter_impl.h @@ -11,6 +11,7 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_HIGH_PASS_FILTER_IMPL_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_HIGH_PASS_FILTER_IMPL_H_ +#include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" @@ -37,6 +38,7 @@ class HighPassFilterImpl : public HighPassFilter { rtc::CriticalSection* const crit_ = nullptr; bool enabled_ GUARDED_BY(crit_) = false; std::vector> filters_ GUARDED_BY(crit_); + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(HighPassFilterImpl); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/noise_suppression_impl.cc b/webrtc/modules/audio_processing/noise_suppression_impl.cc index 837585fbd7..d4f8bbb60c 100644 --- a/webrtc/modules/audio_processing/noise_suppression_impl.cc +++ b/webrtc/modules/audio_processing/noise_suppression_impl.cc @@ -10,84 +10,96 @@ #include "webrtc/modules/audio_processing/noise_suppression_impl.h" -#include - #include "webrtc/modules/audio_processing/audio_buffer.h" #if defined(WEBRTC_NS_FLOAT) #include "webrtc/modules/audio_processing/ns/noise_suppression.h" +#define NS_CREATE WebRtcNs_Create +#define NS_FREE WebRtcNs_Free +#define NS_INIT WebRtcNs_Init +#define NS_SET_POLICY WebRtcNs_set_policy +typedef NsHandle NsState; #elif defined(WEBRTC_NS_FIXED) #include "webrtc/modules/audio_processing/ns/noise_suppression_x.h" +#define NS_CREATE WebRtcNsx_Create +#define NS_FREE WebRtcNsx_Free +#define NS_INIT WebRtcNsx_Init +#define NS_SET_POLICY WebRtcNsx_set_policy +typedef NsxHandle NsState; #endif - namespace webrtc { - -#if defined(WEBRTC_NS_FLOAT) -typedef NsHandle Handle; -#elif defined(WEBRTC_NS_FIXED) -typedef NsxHandle Handle; -#endif - -namespace { -int MapSetting(NoiseSuppression::Level level) { - switch (level) { - case NoiseSuppression::kLow: - return 0; - case NoiseSuppression::kModerate: - return 1; - case NoiseSuppression::kHigh: - return 2; - case NoiseSuppression::kVeryHigh: - return 3; +class NoiseSuppressionImpl::Suppressor { + public: + explicit Suppressor(int sample_rate_hz) { + state_ = NS_CREATE(); + RTC_CHECK(state_); + int error = NS_INIT(state_, sample_rate_hz); + RTC_DCHECK_EQ(0, error); } - assert(false); - return -1; -} -} // namespace + ~Suppressor() { + NS_FREE(state_); + } + NsState* state() { return state_; } + private: + NsState* state_ = nullptr; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(Suppressor); +}; -NoiseSuppressionImpl::NoiseSuppressionImpl(const AudioProcessing* apm, - rtc::CriticalSection* crit) - : ProcessingComponent(), apm_(apm), crit_(crit), level_(kModerate) { - RTC_DCHECK(apm); +NoiseSuppressionImpl::NoiseSuppressionImpl(rtc::CriticalSection* crit) + : crit_(crit) { RTC_DCHECK(crit); } NoiseSuppressionImpl::~NoiseSuppressionImpl() {} +void NoiseSuppressionImpl::Initialize(int channels, int sample_rate_hz) { + RTC_DCHECK_LE(0, channels); + std::vector> new_suppressors(channels); + for (int i = 0; i < channels; i++) { + new_suppressors[i].reset(new Suppressor(sample_rate_hz)); + } + rtc::CritScope cs(crit_); + suppressors_.swap(new_suppressors); + set_level(level_); +} + int NoiseSuppressionImpl::AnalyzeCaptureAudio(AudioBuffer* audio) { + RTC_DCHECK(audio); #if defined(WEBRTC_NS_FLOAT) - if (!is_component_enabled()) { + rtc::CritScope cs(crit_); + if (!enabled_) { return AudioProcessing::kNoError; } - assert(audio->num_frames_per_band() <= 160); - assert(audio->num_channels() == num_handles()); - for (int i = 0; i < num_handles(); ++i) { - Handle* my_handle = static_cast(handle(i)); - - WebRtcNs_Analyze(my_handle, audio->split_bands_const_f(i)[kBand0To8kHz]); + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + RTC_DCHECK_EQ(suppressors_.size(), + static_cast(audio->num_channels())); + for (size_t i = 0; i < suppressors_.size(); i++) { + WebRtcNs_Analyze(suppressors_[i]->state(), + audio->split_bands_const_f(i)[kBand0To8kHz]); } #endif return AudioProcessing::kNoError; } int NoiseSuppressionImpl::ProcessCaptureAudio(AudioBuffer* audio) { + RTC_DCHECK(audio); rtc::CritScope cs(crit_); - if (!is_component_enabled()) { + if (!enabled_) { return AudioProcessing::kNoError; } - assert(audio->num_frames_per_band() <= 160); - assert(audio->num_channels() == num_handles()); - for (int i = 0; i < num_handles(); ++i) { - Handle* my_handle = static_cast(handle(i)); + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + RTC_DCHECK_EQ(suppressors_.size(), + static_cast(audio->num_channels())); + for (size_t i = 0; i < suppressors_.size(); i++) { #if defined(WEBRTC_NS_FLOAT) - WebRtcNs_Process(my_handle, + WebRtcNs_Process(suppressors_[i]->state(), audio->split_bands_const_f(i), audio->num_bands(), audio->split_bands_f(i)); #elif defined(WEBRTC_NS_FIXED) - WebRtcNsx_Process(my_handle, + WebRtcNsx_Process(suppressors_[i]->state(), audio->split_bands_const(i), audio->num_bands(), audio->split_bands(i)); @@ -98,22 +110,40 @@ int NoiseSuppressionImpl::ProcessCaptureAudio(AudioBuffer* audio) { int NoiseSuppressionImpl::Enable(bool enable) { rtc::CritScope cs(crit_); - return EnableComponent(enable); + enabled_ = enable; + return AudioProcessing::kNoError; } bool NoiseSuppressionImpl::is_enabled() const { rtc::CritScope cs(crit_); - return is_component_enabled(); + return enabled_; } int NoiseSuppressionImpl::set_level(Level level) { rtc::CritScope cs(crit_); - if (MapSetting(level) == -1) { - return AudioProcessing::kBadParameterError; + int policy = 1; + switch (level) { + case NoiseSuppression::kLow: + policy = 0; + break; + case NoiseSuppression::kModerate: + policy = 1; + break; + case NoiseSuppression::kHigh: + policy = 2; + break; + case NoiseSuppression::kVeryHigh: + policy = 3; + break; + default: + RTC_NOTREACHED(); } - level_ = level; - return Configure(); + for (auto& suppressor : suppressors_) { + int error = NS_SET_POLICY(suppressor->state(), policy); + RTC_DCHECK_EQ(0, error); + } + return AudioProcessing::kNoError; } NoiseSuppression::Level NoiseSuppressionImpl::level() const { @@ -125,61 +155,18 @@ float NoiseSuppressionImpl::speech_probability() const { rtc::CritScope cs(crit_); #if defined(WEBRTC_NS_FLOAT) float probability_average = 0.0f; - for (int i = 0; i < num_handles(); i++) { - Handle* my_handle = static_cast(handle(i)); - probability_average += WebRtcNs_prior_speech_probability(my_handle); + for (auto& suppressor : suppressors_) { + probability_average += + WebRtcNs_prior_speech_probability(suppressor->state()); } - return probability_average / num_handles(); + if (suppressors_.size() > 0) { + probability_average /= suppressors_.size(); + } + return probability_average; #elif defined(WEBRTC_NS_FIXED) + // TODO(peah): Returning error code as a float! Remove this. // Currently not available for the fixed point implementation. return AudioProcessing::kUnsupportedFunctionError; #endif } - -void* NoiseSuppressionImpl::CreateHandle() const { -#if defined(WEBRTC_NS_FLOAT) - return WebRtcNs_Create(); -#elif defined(WEBRTC_NS_FIXED) - return WebRtcNsx_Create(); -#endif -} - -void NoiseSuppressionImpl::DestroyHandle(void* handle) const { -#if defined(WEBRTC_NS_FLOAT) - WebRtcNs_Free(static_cast(handle)); -#elif defined(WEBRTC_NS_FIXED) - WebRtcNsx_Free(static_cast(handle)); -#endif -} - -int NoiseSuppressionImpl::InitializeHandle(void* handle) const { -#if defined(WEBRTC_NS_FLOAT) - return WebRtcNs_Init(static_cast(handle), - apm_->proc_sample_rate_hz()); -#elif defined(WEBRTC_NS_FIXED) - return WebRtcNsx_Init(static_cast(handle), - apm_->proc_sample_rate_hz()); -#endif -} - -int NoiseSuppressionImpl::ConfigureHandle(void* handle) const { - rtc::CritScope cs(crit_); -#if defined(WEBRTC_NS_FLOAT) - return WebRtcNs_set_policy(static_cast(handle), - MapSetting(level_)); -#elif defined(WEBRTC_NS_FIXED) - return WebRtcNsx_set_policy(static_cast(handle), - MapSetting(level_)); -#endif -} - -int NoiseSuppressionImpl::num_handles_required() const { - return apm_->num_output_channels(); -} - -int NoiseSuppressionImpl::GetHandleError(void* handle) const { - // The NS has no get_error() function. - assert(handle != NULL); - return AudioProcessing::kUnspecifiedError; -} } // namespace webrtc diff --git a/webrtc/modules/audio_processing/noise_suppression_impl.h b/webrtc/modules/audio_processing/noise_suppression_impl.h index 1564fe586c..a3b346050f 100644 --- a/webrtc/modules/audio_processing/noise_suppression_impl.h +++ b/webrtc/modules/audio_processing/noise_suppression_impl.h @@ -11,49 +11,40 @@ #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_ #define WEBRTC_MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_ +#include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" +#include "webrtc/base/scoped_ptr.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" -#include "webrtc/modules/audio_processing/processing_component.h" namespace webrtc { class AudioBuffer; -class NoiseSuppressionImpl : public NoiseSuppression, - public ProcessingComponent { +class NoiseSuppressionImpl : public NoiseSuppression { public: - NoiseSuppressionImpl(const AudioProcessing* apm, rtc::CriticalSection* crit); - virtual ~NoiseSuppressionImpl(); + explicit NoiseSuppressionImpl(rtc::CriticalSection* crit); + ~NoiseSuppressionImpl() override; + // TODO(peah): Fold into ctor, once public API is removed. + void Initialize(int channels, int sample_rate_hz); int AnalyzeCaptureAudio(AudioBuffer* audio); int ProcessCaptureAudio(AudioBuffer* audio); // NoiseSuppression implementation. + int Enable(bool enable) override; bool is_enabled() const override; - float speech_probability() const override; + int set_level(Level level) override; Level level() const override; + float speech_probability() const override; private: - // NoiseSuppression implementation. - int Enable(bool enable) override; - int set_level(Level level) override; - - // ProcessingComponent implementation. - void* CreateHandle() const override; - int InitializeHandle(void* handle) const override; - int ConfigureHandle(void* handle) const override; - void DestroyHandle(void* handle) const override; - int num_handles_required() const override; - int GetHandleError(void* handle) const override; - - // Not guarded as its public API is thread safe. - const AudioProcessing* apm_; - + class Suppressor; rtc::CriticalSection* const crit_; - - Level level_ GUARDED_BY(crit_); + bool enabled_ GUARDED_BY(crit_) = false; + Level level_ GUARDED_BY(crit_) = kModerate; + std::vector> suppressors_ GUARDED_BY(crit_); + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(NoiseSuppressionImpl); }; - } // namespace webrtc #endif // WEBRTC_MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_