Make NoiseSuppression not a processing component (bit exact).

BUG=webrtc:5298

patch from issue 1490333004 at patchset 1 (http://crrev.com/1490333004#ps1)

Review URL: https://codereview.webrtc.org/1507683006

Cr-Commit-Position: refs/heads/master@{#10944}

webrtc/modules/audio_processing/audio_processing_impl.cc

@@ -76,38 +76,6 @@ static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
 }
 }  // namespace
 
-struct AudioProcessingImpl::ApmPublicSubmodules {
-  ApmPublicSubmodules()
-      : echo_cancellation(nullptr),
-        echo_control_mobile(nullptr),
-        gain_control(nullptr),
-        level_estimator(nullptr),
-        noise_suppression(nullptr),
-        voice_detection(nullptr) {}
-  // Accessed externally of APM without any lock acquired.
-  EchoCancellationImpl* echo_cancellation;
-  EchoControlMobileImpl* echo_control_mobile;
-  GainControlImpl* gain_control;
-  rtc::scoped_ptr<HighPassFilterImpl> high_pass_filter;
-  LevelEstimatorImpl* level_estimator;
-  NoiseSuppressionImpl* noise_suppression;
-  VoiceDetectionImpl* voice_detection;
-  rtc::scoped_ptr<GainControlForNewAgc> gain_control_for_new_agc;
-
-  // Accessed internally from both render and capture.
-  rtc::scoped_ptr<TransientSuppressor> transient_suppressor;
-  rtc::scoped_ptr<IntelligibilityEnhancer> intelligibility_enhancer;
-};
-
-struct AudioProcessingImpl::ApmPrivateSubmodules {
-  explicit ApmPrivateSubmodules(Beamformer<float>* beamformer)
-      : beamformer(beamformer) {}
-  // Accessed internally from capture or during initialization
-  std::list<ProcessingComponent*> component_list;
-  rtc::scoped_ptr<Beamformer<float>> beamformer;
-  rtc::scoped_ptr<AgcManagerDirect> agc_manager;
-};
-
 // Throughout webrtc, it's assumed that success is represented by zero.
 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
 
@@ -177,6 +145,37 @@ class GainControlForNewAgc : public GainControl, public VolumeCallbacks {
   int volume_;
 };
 
+struct AudioProcessingImpl::ApmPublicSubmodules {
+  ApmPublicSubmodules()
+      : echo_cancellation(nullptr),
+        echo_control_mobile(nullptr),
+        gain_control(nullptr),
+        level_estimator(nullptr),
+        voice_detection(nullptr) {}
+  // Accessed externally of APM without any lock acquired.
+  EchoCancellationImpl* echo_cancellation;
+  EchoControlMobileImpl* echo_control_mobile;
+  GainControlImpl* gain_control;
+  rtc::scoped_ptr<HighPassFilterImpl> high_pass_filter;
+  LevelEstimatorImpl* level_estimator;
+  rtc::scoped_ptr<NoiseSuppressionImpl> noise_suppression;
+  VoiceDetectionImpl* voice_detection;
+  rtc::scoped_ptr<GainControlForNewAgc> gain_control_for_new_agc;
+
+  // Accessed internally from both render and capture.
+  rtc::scoped_ptr<TransientSuppressor> transient_suppressor;
+  rtc::scoped_ptr<IntelligibilityEnhancer> intelligibility_enhancer;
+};
+
+struct AudioProcessingImpl::ApmPrivateSubmodules {
+  explicit ApmPrivateSubmodules(Beamformer<float>* beamformer)
+      : beamformer(beamformer) {}
+  // Accessed internally from capture or during initialization
+  std::list<ProcessingComponent*> component_list;
+  rtc::scoped_ptr<Beamformer<float>> beamformer;
+  rtc::scoped_ptr<AgcManagerDirect> agc_manager;
+};
+
 const int AudioProcessing::kNativeSampleRatesHz[] = {
     AudioProcessing::kSampleRate8kHz,
     AudioProcessing::kSampleRate16kHz,
@@ -246,8 +245,8 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config,
         new HighPassFilterImpl(&crit_capture_));
     public_submodules_->level_estimator =
         new LevelEstimatorImpl(this, &crit_capture_);
-    public_submodules_->noise_suppression =
+    public_submodules_->noise_suppression.reset(
-        new NoiseSuppressionImpl(this, &crit_capture_);
+        new NoiseSuppressionImpl(&crit_capture_));
     public_submodules_->voice_detection =
         new VoiceDetectionImpl(this, &crit_capture_);
     public_submodules_->gain_control_for_new_agc.reset(
@@ -261,8 +260,6 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config,
         public_submodules_->gain_control);
     private_submodules_->component_list.push_back(
         public_submodules_->level_estimator);
-    private_submodules_->component_list.push_back(
-        public_submodules_->noise_suppression);
     private_submodules_->component_list.push_back(
         public_submodules_->voice_detection);
   }
@@ -396,14 +393,11 @@ int AudioProcessingImpl::InitializeLocked() {
   }
 
   InitializeExperimentalAgc();
-
   InitializeTransient();
-
   InitializeBeamformer();
-
   InitializeIntelligibility();
-
   InitializeHighPassFilter();
+  InitializeNoiseSuppression();
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->Open()) {
@@ -768,14 +762,14 @@ int AudioProcessingImpl::ProcessStreamLocked() {
 
   public_submodules_->high_pass_filter->ProcessCaptureAudio(ca);
   RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca));
-  RETURN_ON_ERR(public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca));
+  public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca);
   RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(ca));
 
   if (public_submodules_->echo_control_mobile->is_enabled() &&
       public_submodules_->noise_suppression->is_enabled()) {
     ca->CopyLowPassToReference();
   }
-  RETURN_ON_ERR(public_submodules_->noise_suppression->ProcessCaptureAudio(ca));
+  public_submodules_->noise_suppression->ProcessCaptureAudio(ca);
   RETURN_ON_ERR(
       public_submodules_->echo_control_mobile->ProcessCaptureAudio(ca));
   RETURN_ON_ERR(public_submodules_->voice_detection->ProcessCaptureAudio(ca));
@@ -1158,7 +1152,7 @@ LevelEstimator* AudioProcessingImpl::level_estimator() const {
 NoiseSuppression* AudioProcessingImpl::noise_suppression() const {
   // Adding a lock here has no effect as it allows any access to the submodule
   // from the returned pointer.
-  return public_submodules_->noise_suppression;
+  return public_submodules_->noise_suppression.get();
 }
 
 VoiceDetection* AudioProcessingImpl::voice_detection() const {
@@ -1181,6 +1175,9 @@ bool AudioProcessingImpl::is_data_processed() const {
   if (public_submodules_->high_pass_filter->is_enabled()) {
     enabled_count++;
   }
+  if (public_submodules_->noise_suppression->is_enabled()) {
+    enabled_count++;
+  }
 
   // Data is unchanged if no components are enabled, or if only
   // public_submodules_->level_estimator
@@ -1300,6 +1297,11 @@ void AudioProcessingImpl::InitializeHighPassFilter() {
                                                    proc_sample_rate_hz());
 }
 
+void AudioProcessingImpl::InitializeNoiseSuppression() {
+  public_submodules_->noise_suppression->Initialize(num_output_channels(),
+                                                    proc_sample_rate_hz());
+}
+
 void AudioProcessingImpl::MaybeUpdateHistograms() {
   static const int kMinDiffDelayMs = 60;
 

webrtc/modules/audio_processing/audio_processing_impl.h

@@ -39,18 +39,6 @@ class AudioConverter;
 template<typename T>
 class Beamformer;
 
-class EchoCancellationImpl;
-class EchoControlMobileImpl;
-class GainControlImpl;
-class GainControlForNewAgc;
-class HighPassFilterImpl;
-class LevelEstimatorImpl;
-class NoiseSuppressionImpl;
-class ProcessingComponent;
-class TransientSuppressor;
-class VoiceDetectionImpl;
-class IntelligibilityEnhancer;
-
 class AudioProcessingImpl : public AudioProcessing {
  public:
   // Methods forcing APM to run in a single-threaded manner.
@@ -195,6 +183,8 @@ class AudioProcessingImpl : public AudioProcessing {
       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
   void InitializeHighPassFilter()
       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
+  void InitializeNoiseSuppression()
+      EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
   int InitializeLocked(const ProcessingConfig& config)
       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
 

webrtc/modules/audio_processing/high_pass_filter_impl.cc

@@ -89,6 +89,7 @@ HighPassFilterImpl::HighPassFilterImpl(rtc::CriticalSection* crit)
 HighPassFilterImpl::~HighPassFilterImpl() {}
 
 void HighPassFilterImpl::Initialize(int channels, int sample_rate_hz) {
+  RTC_DCHECK_LE(0, channels);
   std::vector<rtc::scoped_ptr<BiquadFilter>> new_filters(channels);
   for (int i = 0; i < channels; i++) {
     new_filters[i].reset(new BiquadFilter(sample_rate_hz));
@@ -98,6 +99,7 @@ void HighPassFilterImpl::Initialize(int channels, int sample_rate_hz) {
 }
 
 void HighPassFilterImpl::ProcessCaptureAudio(AudioBuffer* audio) {
+  RTC_DCHECK(audio);
   rtc::CritScope cs(crit_);
   if (!enabled_) {
     return;

webrtc/modules/audio_processing/high_pass_filter_impl.h

@@ -11,6 +11,7 @@
 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_HIGH_PASS_FILTER_IMPL_H_
 #define WEBRTC_MODULES_AUDIO_PROCESSING_HIGH_PASS_FILTER_IMPL_H_
 
+#include "webrtc/base/constructormagic.h"
 #include "webrtc/base/criticalsection.h"
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
@@ -37,6 +38,7 @@ class HighPassFilterImpl : public HighPassFilter {
   rtc::CriticalSection* const crit_ = nullptr;
   bool enabled_ GUARDED_BY(crit_) = false;
   std::vector<rtc::scoped_ptr<BiquadFilter>> filters_ GUARDED_BY(crit_);
+  RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(HighPassFilterImpl);
 };
 }  // namespace webrtc
 

webrtc/modules/audio_processing/noise_suppression_impl.cc

@@ -10,84 +10,96 @@
 
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
 
-#include <assert.h>
-
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 #if defined(WEBRTC_NS_FLOAT)
 #include "webrtc/modules/audio_processing/ns/noise_suppression.h"
+#define NS_CREATE WebRtcNs_Create
+#define NS_FREE WebRtcNs_Free
+#define NS_INIT WebRtcNs_Init
+#define NS_SET_POLICY WebRtcNs_set_policy
+typedef NsHandle NsState;
 #elif defined(WEBRTC_NS_FIXED)
 #include "webrtc/modules/audio_processing/ns/noise_suppression_x.h"
+#define NS_CREATE WebRtcNsx_Create
+#define NS_FREE WebRtcNsx_Free
+#define NS_INIT WebRtcNsx_Init
+#define NS_SET_POLICY WebRtcNsx_set_policy
+typedef NsxHandle NsState;
 #endif
 
-
 namespace webrtc {
-
+class NoiseSuppressionImpl::Suppressor {
-#if defined(WEBRTC_NS_FLOAT)
+ public:
-typedef NsHandle Handle;
+  explicit Suppressor(int sample_rate_hz) {
-#elif defined(WEBRTC_NS_FIXED)
+    state_ = NS_CREATE();
-typedef NsxHandle Handle;
+    RTC_CHECK(state_);
-#endif
+    int error = NS_INIT(state_, sample_rate_hz);
-
+    RTC_DCHECK_EQ(0, error);
-namespace {
-int MapSetting(NoiseSuppression::Level level) {
-  switch (level) {
-    case NoiseSuppression::kLow:
-      return 0;
-    case NoiseSuppression::kModerate:
-      return 1;
-    case NoiseSuppression::kHigh:
-      return 2;
-    case NoiseSuppression::kVeryHigh:
-      return 3;
   }
-  assert(false);
+  ~Suppressor() {
-  return -1;
+    NS_FREE(state_);
   }
-}  // namespace
+  NsState* state() { return state_; }
+ private:
+  NsState* state_ = nullptr;
+  RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(Suppressor);
+};
 
-NoiseSuppressionImpl::NoiseSuppressionImpl(const AudioProcessing* apm,
+NoiseSuppressionImpl::NoiseSuppressionImpl(rtc::CriticalSection* crit)
-                                           rtc::CriticalSection* crit)
+    : crit_(crit) {
-    : ProcessingComponent(), apm_(apm), crit_(crit), level_(kModerate) {
-  RTC_DCHECK(apm);
   RTC_DCHECK(crit);
 }
 
 NoiseSuppressionImpl::~NoiseSuppressionImpl() {}
 
+void NoiseSuppressionImpl::Initialize(int channels, int sample_rate_hz) {
+  RTC_DCHECK_LE(0, channels);
+  std::vector<rtc::scoped_ptr<Suppressor>> new_suppressors(channels);
+  for (int i = 0; i < channels; i++) {
+    new_suppressors[i].reset(new Suppressor(sample_rate_hz));
+  }
+  rtc::CritScope cs(crit_);
+  suppressors_.swap(new_suppressors);
+  set_level(level_);
+}
+
 int NoiseSuppressionImpl::AnalyzeCaptureAudio(AudioBuffer* audio) {
+  RTC_DCHECK(audio);
 #if defined(WEBRTC_NS_FLOAT)
-  if (!is_component_enabled()) {
+  rtc::CritScope cs(crit_);
+  if (!enabled_) {
     return AudioProcessing::kNoError;
   }
-  assert(audio->num_frames_per_band() <= 160);
-  assert(audio->num_channels() == num_handles());
 
-  for (int i = 0; i < num_handles(); ++i) {
+  RTC_DCHECK_GE(160u, audio->num_frames_per_band());
-    Handle* my_handle = static_cast<Handle*>(handle(i));
+  RTC_DCHECK_EQ(suppressors_.size(),
-
+                static_cast<size_t>(audio->num_channels()));
-    WebRtcNs_Analyze(my_handle, audio->split_bands_const_f(i)[kBand0To8kHz]);
+  for (size_t i = 0; i < suppressors_.size(); i++) {
+    WebRtcNs_Analyze(suppressors_[i]->state(),
+                     audio->split_bands_const_f(i)[kBand0To8kHz]);
   }
 #endif
   return AudioProcessing::kNoError;
 }
 
 int NoiseSuppressionImpl::ProcessCaptureAudio(AudioBuffer* audio) {
+  RTC_DCHECK(audio);
   rtc::CritScope cs(crit_);
-  if (!is_component_enabled()) {
+  if (!enabled_) {
     return AudioProcessing::kNoError;
   }
-  assert(audio->num_frames_per_band() <= 160);
-  assert(audio->num_channels() == num_handles());
 
-  for (int i = 0; i < num_handles(); ++i) {
+  RTC_DCHECK_GE(160u, audio->num_frames_per_band());
-    Handle* my_handle = static_cast<Handle*>(handle(i));
+  RTC_DCHECK_EQ(suppressors_.size(),
+                static_cast<size_t>(audio->num_channels()));
+  for (size_t i = 0; i < suppressors_.size(); i++) {
 #if defined(WEBRTC_NS_FLOAT)
-    WebRtcNs_Process(my_handle,
+    WebRtcNs_Process(suppressors_[i]->state(),
                      audio->split_bands_const_f(i),
                      audio->num_bands(),
                      audio->split_bands_f(i));
 #elif defined(WEBRTC_NS_FIXED)
-    WebRtcNsx_Process(my_handle,
+    WebRtcNsx_Process(suppressors_[i]->state(),
                       audio->split_bands_const(i),
                       audio->num_bands(),
                       audio->split_bands(i));
@@ -98,22 +110,40 @@ int NoiseSuppressionImpl::ProcessCaptureAudio(AudioBuffer* audio) {
 
 int NoiseSuppressionImpl::Enable(bool enable) {
   rtc::CritScope cs(crit_);
-  return EnableComponent(enable);
+  enabled_ = enable;
+  return AudioProcessing::kNoError;
 }
 
 bool NoiseSuppressionImpl::is_enabled() const {
   rtc::CritScope cs(crit_);
-  return is_component_enabled();
+  return enabled_;
 }
 
 int NoiseSuppressionImpl::set_level(Level level) {
   rtc::CritScope cs(crit_);
-  if (MapSetting(level) == -1) {
+  int policy = 1;
-    return AudioProcessing::kBadParameterError;
+  switch (level) {
+    case NoiseSuppression::kLow:
+      policy = 0;
+      break;
+    case NoiseSuppression::kModerate:
+      policy = 1;
+      break;
+    case NoiseSuppression::kHigh:
+      policy = 2;
+      break;
+    case NoiseSuppression::kVeryHigh:
+      policy = 3;
+      break;
+    default:
+      RTC_NOTREACHED();
   }
-
   level_ = level;
-  return Configure();
+  for (auto& suppressor : suppressors_) {
+    int error = NS_SET_POLICY(suppressor->state(), policy);
+    RTC_DCHECK_EQ(0, error);
+  }
+  return AudioProcessing::kNoError;
 }
 
 NoiseSuppression::Level NoiseSuppressionImpl::level() const {
@@ -125,61 +155,18 @@ float NoiseSuppressionImpl::speech_probability() const {
   rtc::CritScope cs(crit_);
 #if defined(WEBRTC_NS_FLOAT)
   float probability_average = 0.0f;
-  for (int i = 0; i < num_handles(); i++) {
+  for (auto& suppressor : suppressors_) {
-    Handle* my_handle = static_cast<Handle*>(handle(i));
+    probability_average +=
-    probability_average += WebRtcNs_prior_speech_probability(my_handle);
+        WebRtcNs_prior_speech_probability(suppressor->state());
   }
-  return probability_average / num_handles();
+  if (suppressors_.size() > 0) {
+    probability_average /= suppressors_.size();
+  }
+  return probability_average;
 #elif defined(WEBRTC_NS_FIXED)
+  // TODO(peah): Returning error code as a float! Remove this.
   // Currently not available for the fixed point implementation.
   return AudioProcessing::kUnsupportedFunctionError;
 #endif
 }
-
-void* NoiseSuppressionImpl::CreateHandle() const {
-#if defined(WEBRTC_NS_FLOAT)
-  return WebRtcNs_Create();
-#elif defined(WEBRTC_NS_FIXED)
-  return WebRtcNsx_Create();
-#endif
-}
-
-void NoiseSuppressionImpl::DestroyHandle(void* handle) const {
-#if defined(WEBRTC_NS_FLOAT)
-  WebRtcNs_Free(static_cast<Handle*>(handle));
-#elif defined(WEBRTC_NS_FIXED)
-  WebRtcNsx_Free(static_cast<Handle*>(handle));
-#endif
-}
-
-int NoiseSuppressionImpl::InitializeHandle(void* handle) const {
-#if defined(WEBRTC_NS_FLOAT)
-  return WebRtcNs_Init(static_cast<Handle*>(handle),
-                       apm_->proc_sample_rate_hz());
-#elif defined(WEBRTC_NS_FIXED)
-  return WebRtcNsx_Init(static_cast<Handle*>(handle),
-                        apm_->proc_sample_rate_hz());
-#endif
-}
-
-int NoiseSuppressionImpl::ConfigureHandle(void* handle) const {
-  rtc::CritScope cs(crit_);
-#if defined(WEBRTC_NS_FLOAT)
-  return WebRtcNs_set_policy(static_cast<Handle*>(handle),
-                             MapSetting(level_));
-#elif defined(WEBRTC_NS_FIXED)
-  return WebRtcNsx_set_policy(static_cast<Handle*>(handle),
-                              MapSetting(level_));
-#endif
-}
-
-int NoiseSuppressionImpl::num_handles_required() const {
-  return apm_->num_output_channels();
-}
-
-int NoiseSuppressionImpl::GetHandleError(void* handle) const {
-  // The NS has no get_error() function.
-  assert(handle != NULL);
-  return AudioProcessing::kUnspecifiedError;
-}
 }  // namespace webrtc

webrtc/modules/audio_processing/noise_suppression_impl.h

@@ -11,49 +11,40 @@
 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_
 #define WEBRTC_MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_
 
+#include "webrtc/base/constructormagic.h"
 #include "webrtc/base/criticalsection.h"
+#include "webrtc/base/scoped_ptr.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
-#include "webrtc/modules/audio_processing/processing_component.h"
 
 namespace webrtc {
 
 class AudioBuffer;
 
-class NoiseSuppressionImpl : public NoiseSuppression,
+class NoiseSuppressionImpl : public NoiseSuppression {
-                             public ProcessingComponent {
  public:
-  NoiseSuppressionImpl(const AudioProcessing* apm, rtc::CriticalSection* crit);
+  explicit NoiseSuppressionImpl(rtc::CriticalSection* crit);
-  virtual ~NoiseSuppressionImpl();
+  ~NoiseSuppressionImpl() override;
 
+  // TODO(peah): Fold into ctor, once public API is removed.
+  void Initialize(int channels, int sample_rate_hz);
   int AnalyzeCaptureAudio(AudioBuffer* audio);
   int ProcessCaptureAudio(AudioBuffer* audio);
 
   // NoiseSuppression implementation.
+  int Enable(bool enable) override;
   bool is_enabled() const override;
-  float speech_probability() const override;
+  int set_level(Level level) override;
   Level level() const override;
+  float speech_probability() const override;
 
  private:
-  // NoiseSuppression implementation.
+  class Suppressor;
-  int Enable(bool enable) override;
-  int set_level(Level level) override;
-
-  // ProcessingComponent implementation.
-  void* CreateHandle() const override;
-  int InitializeHandle(void* handle) const override;
-  int ConfigureHandle(void* handle) const override;
-  void DestroyHandle(void* handle) const override;
-  int num_handles_required() const override;
-  int GetHandleError(void* handle) const override;
-
-  // Not guarded as its public API is thread safe.
-  const AudioProcessing* apm_;
-
   rtc::CriticalSection* const crit_;
-
+  bool enabled_ GUARDED_BY(crit_) = false;
-  Level level_ GUARDED_BY(crit_);
+  Level level_ GUARDED_BY(crit_) = kModerate;
+  std::vector<rtc::scoped_ptr<Suppressor>> suppressors_ GUARDED_BY(crit_);
+  RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(NoiseSuppressionImpl);
 };
-
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_AUDIO_PROCESSING_NOISE_SUPPRESSION_IMPL_H_