AEC3: Sub-band nearend detector

Implements an alternative to the dominant nearend detector.

Bug: b/130016532
Change-Id: If4867d58aad036ccf4e456ef81689b8db0284f7d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/159865
Reviewed-by: Per Åhgren <peah@webrtc.org>
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Gustaf Ullberg <gustaf@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29906}

api/audio/echo_canceller3_config.cc

@@ -214,6 +214,22 @@ bool EchoCanceller3Config::Validate(EchoCanceller3Config* config) {
   res = res & Limit(&c->suppressor.dominant_nearend_detection.trigger_threshold,
                     0, 10000);
 
+  res = res &
+        Limit(&c->suppressor.subband_nearend_detection.nearend_average_blocks,
+              1, 1024);
+  res =
+      res & Limit(&c->suppressor.subband_nearend_detection.subband1.low, 0, 65);
+  res = res & Limit(&c->suppressor.subband_nearend_detection.subband1.high,
+                    c->suppressor.subband_nearend_detection.subband1.low, 65);
+  res =
+      res & Limit(&c->suppressor.subband_nearend_detection.subband2.low, 0, 65);
+  res = res & Limit(&c->suppressor.subband_nearend_detection.subband2.high,
+                    c->suppressor.subband_nearend_detection.subband2.low, 65);
+  res = res & Limit(&c->suppressor.subband_nearend_detection.nearend_threshold,
+                    0.f, 1.e24f);
+  res = res & Limit(&c->suppressor.subband_nearend_detection.snr_threshold, 0.f,
+                    1.e24f);
+
   res = res & Limit(&c->suppressor.high_bands_suppression.enr_threshold, 0.f,
                     1000000.f);
   res = res & Limit(&c->suppressor.high_bands_suppression.max_gain_during_echo,

api/audio/echo_canceller3_config.h

@@ -180,6 +180,20 @@ struct RTC_EXPORT EchoCanceller3Config {
       bool use_during_initial_phase = true;
     } dominant_nearend_detection;
 
+    struct SubbandNearendDetection {
+      size_t nearend_average_blocks = 1;
+      struct SubbandRegion {
+        size_t low;
+        size_t high;
+      };
+      SubbandRegion subband1 = {1, 1};
+      SubbandRegion subband2 = {1, 1};
+      float nearend_threshold = 1.f;
+      float snr_threshold = 1.f;
+    } subband_nearend_detection;
+
+    bool use_subband_nearend_detection = false;
+
     struct HighBandsSuppression {
       float enr_threshold = 1.f;
       float max_gain_during_echo = 1.f;

api/audio/echo_canceller3_config_json.cc

@@ -92,6 +92,25 @@ void ReadParam(const Json::Value& root,
   }
 }
 
+void ReadParam(
+    const Json::Value& root,
+    std::string param_name,
+    EchoCanceller3Config::Suppressor::SubbandNearendDetection::SubbandRegion*
+        param) {
+  RTC_DCHECK(param);
+  Json::Value json_array;
+  if (rtc::GetValueFromJsonObject(root, param_name, &json_array)) {
+    std::vector<int> v;
+    rtc::JsonArrayToIntVector(json_array, &v);
+    if (v.size() != 2) {
+      RTC_LOG(LS_ERROR) << "Incorrect array size for " << param_name;
+      return;
+    }
+    param->low = static_cast<size_t>(v[0]);
+    param->high = static_cast<size_t>(v[1]);
+  }
+}
+
 void ReadParam(const Json::Value& root,
                std::string param_name,
                EchoCanceller3Config::Suppressor::MaskingThresholds* param) {
@@ -306,6 +325,24 @@ void Aec3ConfigFromJsonString(absl::string_view json_string,
           &cfg.suppressor.dominant_nearend_detection.use_during_initial_phase);
     }
 
+    if (rtc::GetValueFromJsonObject(section, "subband_nearend_detection",
+                                    &subsection)) {
+      ReadParam(
+          subsection, "nearend_average_blocks",
+          &cfg.suppressor.subband_nearend_detection.nearend_average_blocks);
+      ReadParam(subsection, "subband1",
+                &cfg.suppressor.subband_nearend_detection.subband1);
+      ReadParam(subsection, "subband2",
+                &cfg.suppressor.subband_nearend_detection.subband2);
+      ReadParam(subsection, "nearend_threshold",
+                &cfg.suppressor.subband_nearend_detection.nearend_threshold);
+      ReadParam(subsection, "snr_threshold",
+                &cfg.suppressor.subband_nearend_detection.snr_threshold);
+    }
+
+    ReadParam(section, "use_subband_nearend_detection",
+              &cfg.suppressor.use_subband_nearend_detection);
+
     if (rtc::GetValueFromJsonObject(section, "high_bands_suppression",
                                     &subsection)) {
       ReadParam(subsection, "enr_threshold",
@@ -542,6 +579,25 @@ std::string Aec3ConfigToJsonString(const EchoCanceller3Config& config) {
   ost << "\"use_during_initial_phase\": "
       << config.suppressor.dominant_nearend_detection.use_during_initial_phase;
   ost << "},";
+  ost << "\"subband_nearend_detection\": {";
+  ost << "\"nearend_average_blocks\": "
+      << config.suppressor.subband_nearend_detection.nearend_average_blocks
+      << ",";
+  ost << "\"subband1\": [";
+  ost << config.suppressor.subband_nearend_detection.subband1.low << ",";
+  ost << config.suppressor.subband_nearend_detection.subband1.high;
+  ost << "],";
+  ost << "\"subband2\": [";
+  ost << config.suppressor.subband_nearend_detection.subband2.low << ",";
+  ost << config.suppressor.subband_nearend_detection.subband2.high;
+  ost << "],";
+  ost << "\"nearend_threshold\": "
+      << config.suppressor.subband_nearend_detection.nearend_threshold << ",";
+  ost << "\"snr_threshold\": "
+      << config.suppressor.subband_nearend_detection.snr_threshold;
+  ost << "},";
+  ost << "\"use_subband_nearend_detection\": "
+      << config.suppressor.use_subband_nearend_detection << ",";
   ost << "\"high_bands_suppression\": {";
   ost << "\"enr_threshold\": "
       << config.suppressor.high_bands_suppression.enr_threshold << ",";

api/audio/test/echo_canceller3_config_json_unittest.cc

@@ -21,6 +21,11 @@ TEST(EchoCanceller3JsonHelpers, ToStringAndParseJson) {
   cfg.delay.log_warning_on_delay_changes = true;
   cfg.filter.shadow_initial.length_blocks = 7u;
   cfg.suppressor.normal_tuning.mask_hf.enr_suppress = .5f;
+  cfg.suppressor.subband_nearend_detection.nearend_average_blocks = 3;
+  cfg.suppressor.subband_nearend_detection.subband1 = {1, 3};
+  cfg.suppressor.subband_nearend_detection.subband1 = {4, 5};
+  cfg.suppressor.subband_nearend_detection.nearend_threshold = 2.f;
+  cfg.suppressor.subband_nearend_detection.snr_threshold = 100.f;
   std::string json_string = Aec3ConfigToJsonString(cfg);
   EchoCanceller3Config cfg_transformed = Aec3ConfigFromJsonString(json_string);
 
@@ -41,5 +46,21 @@ TEST(EchoCanceller3JsonHelpers, ToStringAndParseJson) {
             cfg_transformed.filter.shadow_initial.length_blocks);
   EXPECT_EQ(cfg.suppressor.normal_tuning.mask_hf.enr_suppress,
             cfg_transformed.suppressor.normal_tuning.mask_hf.enr_suppress);
+  EXPECT_EQ(cfg.suppressor.subband_nearend_detection.nearend_average_blocks,
+            cfg_transformed.suppressor.subband_nearend_detection
+                .nearend_average_blocks);
+  EXPECT_EQ(cfg.suppressor.subband_nearend_detection.subband1.low,
+            cfg_transformed.suppressor.subband_nearend_detection.subband1.low);
+  EXPECT_EQ(cfg.suppressor.subband_nearend_detection.subband1.high,
+            cfg_transformed.suppressor.subband_nearend_detection.subband1.high);
+  EXPECT_EQ(cfg.suppressor.subband_nearend_detection.subband2.low,
+            cfg_transformed.suppressor.subband_nearend_detection.subband2.low);
+  EXPECT_EQ(cfg.suppressor.subband_nearend_detection.subband2.high,
+            cfg_transformed.suppressor.subband_nearend_detection.subband2.high);
+  EXPECT_EQ(
+      cfg.suppressor.subband_nearend_detection.nearend_threshold,
+      cfg_transformed.suppressor.subband_nearend_detection.nearend_threshold);
+  EXPECT_EQ(cfg.suppressor.subband_nearend_detection.snr_threshold,
+            cfg_transformed.suppressor.subband_nearend_detection.snr_threshold);
 }
 }  // namespace webrtc

modules/audio_processing/aec3/BUILD.gn

@@ -78,6 +78,7 @@ rtc_library("aec3") {
     "matched_filter_lag_aggregator.h",
     "moving_average.cc",
     "moving_average.h",
+    "nearend_detector.h",
     "render_buffer.cc",
     "render_buffer.h",
     "render_delay_buffer.cc",
@@ -108,6 +109,8 @@ rtc_library("aec3") {
     "stationarity_estimator.h",
     "subband_erle_estimator.cc",
     "subband_erle_estimator.h",
+    "subband_nearend_detector.cc",
+    "subband_nearend_detector.h",
     "subtractor.cc",
     "subtractor.h",
     "subtractor_output.cc",

modules/audio_processing/aec3/dominant_nearend_detector.cc

@@ -10,12 +10,11 @@
 
 #include "modules/audio_processing/aec3/dominant_nearend_detector.h"
 
-#include <algorithm>
 #include <numeric>
 
 namespace webrtc {
 DominantNearendDetector::DominantNearendDetector(
-    const EchoCanceller3Config::Suppressor::DominantNearendDetection config,
+    const EchoCanceller3Config::Suppressor::DominantNearendDetection& config,
     size_t num_capture_channels)
     : enr_threshold_(config.enr_threshold),
       enr_exit_threshold_(config.enr_exit_threshold),

modules/audio_processing/aec3/dominant_nearend_detector.h

@@ -15,18 +15,18 @@
 
 #include "api/array_view.h"
 #include "api/audio/echo_canceller3_config.h"
-#include "modules/audio_processing/aec3/aec3_common.h"
+#include "modules/audio_processing/aec3/nearend_detector.h"
 
 namespace webrtc {
 // Class for selecting whether the suppressor is in the nearend or echo state.
-class DominantNearendDetector {
+class DominantNearendDetector : public NearendDetector {
  public:
   DominantNearendDetector(
-      const EchoCanceller3Config::Suppressor::DominantNearendDetection config,
+      const EchoCanceller3Config::Suppressor::DominantNearendDetection& config,
       size_t num_capture_channels);
 
   // Returns whether the current state is the nearend state.
-  bool IsNearendState() const { return nearend_state_; }
+  bool IsNearendState() const override { return nearend_state_; }
 
   // Updates the state selection based on latest spectral estimates.
   void Update(rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
@@ -35,7 +35,7 @@ class DominantNearendDetector {
                   residual_echo_spectrum,
               rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
                   comfort_noise_spectrum,
-              bool initial_state);
+              bool initial_state) override;
 
  private:
   const float enr_threshold_;

modules/audio_processing/aec3/nearend_detector.h

@@ -0,0 +1,42 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AEC3_NEAREND_DETECTOR_H_
+#define MODULES_AUDIO_PROCESSING_AEC3_NEAREND_DETECTOR_H_
+
+#include <vector>
+
+#include "api/array_view.h"
+#include "api/audio/echo_canceller3_config.h"
+#include "modules/audio_processing/aec3/aec3_common.h"
+
+namespace webrtc {
+// Class for selecting whether the suppressor is in the nearend or echo state.
+class NearendDetector {
+ public:
+  virtual ~NearendDetector() {}
+
+  // Returns whether the current state is the nearend state.
+  virtual bool IsNearendState() const = 0;
+
+  // Updates the state selection based on latest spectral estimates.
+  virtual void Update(
+      rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+          nearend_spectrum,
+      rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+          residual_echo_spectrum,
+      rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+          comfort_noise_spectrum,
+      bool initial_state) = 0;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_AUDIO_PROCESSING_AEC3_NEAREND_DETECTOR_H_

modules/audio_processing/aec3/subband_nearend_detector.cc

@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/aec3/subband_nearend_detector.h"
+
+#include <numeric>
+
+namespace webrtc {
+SubbandNearendDetector::SubbandNearendDetector(
+    const EchoCanceller3Config::Suppressor::SubbandNearendDetection& config,
+    size_t num_capture_channels)
+    : config_(config),
+      num_capture_channels_(num_capture_channels),
+      nearend_smoothers_(num_capture_channels_,
+                         aec3::MovingAverage(kFftLengthBy2Plus1,
+                                             config_.nearend_average_blocks)),
+      one_over_subband_length1_(
+          1.f / (config_.subband1.high - config_.subband1.low + 1)),
+      one_over_subband_length2_(
+          1.f / (config_.subband2.high - config_.subband2.low + 1)) {}
+
+void SubbandNearendDetector::Update(
+    rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+        nearend_spectrum,
+    rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+        residual_echo_spectrum,
+    rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+        comfort_noise_spectrum,
+    bool initial_state) {
+  nearend_state_ = false;
+  for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
+    const std::array<float, kFftLengthBy2Plus1>& noise =
+        comfort_noise_spectrum[ch];
+    std::array<float, kFftLengthBy2Plus1> nearend;
+    nearend_smoothers_[ch].Average(nearend_spectrum[ch], nearend);
+
+    // Noise power of the first region.
+    float noise_power =
+        std::accumulate(noise.begin() + config_.subband1.low,
+                        noise.begin() + config_.subband1.high + 1, 0.f) *
+        one_over_subband_length1_;
+
+    // Nearend power of the first region.
+    float nearend_power_subband1 =
+        std::accumulate(nearend.begin() + config_.subband1.low,
+                        nearend.begin() + config_.subband1.high + 1, 0.f) *
+        one_over_subband_length1_;
+
+    // Nearend power of the second region.
+    float nearend_power_subband2 =
+        std::accumulate(nearend.begin() + config_.subband2.low,
+                        nearend.begin() + config_.subband2.high + 1, 0.f) *
+        one_over_subband_length2_;
+
+    // One channel is sufficient to trigger nearend state.
+    nearend_state_ =
+        nearend_state_ ||
+        (nearend_power_subband1 <
+             config_.nearend_threshold * nearend_power_subband2 &&
+         (nearend_power_subband1 > config_.snr_threshold * noise_power));
+  }
+}
+}  // namespace webrtc

modules/audio_processing/aec3/subband_nearend_detector.h

@@ -0,0 +1,52 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AEC3_SUBBAND_NEAREND_DETECTOR_H_
+#define MODULES_AUDIO_PROCESSING_AEC3_SUBBAND_NEAREND_DETECTOR_H_
+
+#include <vector>
+
+#include "api/array_view.h"
+#include "api/audio/echo_canceller3_config.h"
+#include "modules/audio_processing/aec3/moving_average.h"
+#include "modules/audio_processing/aec3/nearend_detector.h"
+
+namespace webrtc {
+// Class for selecting whether the suppressor is in the nearend or echo state.
+class SubbandNearendDetector : public NearendDetector {
+ public:
+  SubbandNearendDetector(
+      const EchoCanceller3Config::Suppressor::SubbandNearendDetection& config,
+      size_t num_capture_channels);
+
+  // Returns whether the current state is the nearend state.
+  bool IsNearendState() const override { return nearend_state_; }
+
+  // Updates the state selection based on latest spectral estimates.
+  void Update(rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+                  nearend_spectrum,
+              rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+                  residual_echo_spectrum,
+              rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
+                  comfort_noise_spectrum,
+              bool initial_state) override;
+
+ private:
+  const EchoCanceller3Config::Suppressor::SubbandNearendDetection config_;
+  const size_t num_capture_channels_;
+  std::vector<aec3::MovingAverage> nearend_smoothers_;
+  const float one_over_subband_length1_;
+  const float one_over_subband_length2_;
+  bool nearend_state_ = false;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_AUDIO_PROCESSING_AEC3_SUBBAND_NEAREND_DETECTOR_H_

modules/audio_processing/aec3/suppression_gain.cc

@@ -16,7 +16,9 @@
 #include <algorithm>
 #include <numeric>
 
+#include "modules/audio_processing/aec3/dominant_nearend_detector.h"
 #include "modules/audio_processing/aec3/moving_average.h"
+#include "modules/audio_processing/aec3/subband_nearend_detector.h"
 #include "modules/audio_processing/aec3/vector_math.h"
 #include "modules/audio_processing/logging/apm_data_dumper.h"
 #include "rtc_base/atomic_ops.h"
@@ -160,7 +162,7 @@ float SuppressionGain::UpperBandsGain(
   }
 
   float gain_bound = 1.f;
-  if (!dominant_nearend_detector_.IsNearendState()) {
+  if (!dominant_nearend_detector_->IsNearendState()) {
     // Bound the upper gain during significant echo activity.
     const auto& cfg = config_.suppressor.high_bands_suppression;
     auto low_frequency_energy = [](rtc::ArrayView<const float> spectrum) {
@@ -187,8 +189,8 @@ void SuppressionGain::GainToNoAudibleEcho(
     const std::array<float, kFftLengthBy2Plus1>& echo,
     const std::array<float, kFftLengthBy2Plus1>& masker,
     std::array<float, kFftLengthBy2Plus1>* gain) const {
-  const auto& p = dominant_nearend_detector_.IsNearendState() ? nearend_params_
-                                                              : normal_params_;
+  const auto& p = dominant_nearend_detector_->IsNearendState() ? nearend_params_
+                                                               : normal_params_;
   for (size_t k = 0; k < gain->size(); ++k) {
     float enr = echo[k] / (nearend[k] + 1.f);  // Echo-to-nearend ratio.
     float emr = echo[k] / (masker[k] + 1.f);   // Echo-to-masker (noise) ratio.
@@ -222,10 +224,11 @@ void SuppressionGain::GetMinGain(
                         : 1.f;
       min_gain[k] = std::min(min_gain[k], 1.f);
     }
+
+    const bool is_nearend_state = dominant_nearend_detector_->IsNearendState();
     for (size_t k = 0; k < 6; ++k) {
-      const auto& dec = dominant_nearend_detector_.IsNearendState()
-                            ? nearend_params_.max_dec_factor_lf
-                            : normal_params_.max_dec_factor_lf;
+      const auto& dec = is_nearend_state ? nearend_params_.max_dec_factor_lf
+                                         : normal_params_.max_dec_factor_lf;
 
       // Make sure the gains of the low frequencies do not decrease too
       // quickly after strong nearend.
@@ -242,7 +245,7 @@ void SuppressionGain::GetMinGain(
 // Compute the maximum gain by limiting the gain increase from the previous
 // gain.
 void SuppressionGain::GetMaxGain(rtc::ArrayView<float> max_gain) const {
-  const auto& inc = dominant_nearend_detector_.IsNearendState()
+  const auto& inc = dominant_nearend_detector_->IsNearendState()
                         ? nearend_params_.max_inc_factor
                         : normal_params_.max_inc_factor;
   const auto& floor = config_.suppressor.floor_first_increase;
@@ -319,11 +322,17 @@ SuppressionGain::SuppressionGain(const EchoCanceller3Config& config,
           aec3::MovingAverage(kFftLengthBy2Plus1,
                               config.suppressor.nearend_average_blocks)),
       nearend_params_(config_.suppressor.nearend_tuning),
-      normal_params_(config_.suppressor.normal_tuning),
-      dominant_nearend_detector_(config_.suppressor.dominant_nearend_detection,
-                                 num_capture_channels_) {
+      normal_params_(config_.suppressor.normal_tuning) {
   RTC_DCHECK_LT(0, state_change_duration_blocks_);
   last_gain_.fill(1.f);
+  if (config_.suppressor.use_subband_nearend_detection) {
+    dominant_nearend_detector_ = std::make_unique<SubbandNearendDetector>(
+        config_.suppressor.subband_nearend_detection, num_capture_channels_);
+  } else {
+    dominant_nearend_detector_ = std::make_unique<DominantNearendDetector>(
+        config_.suppressor.dominant_nearend_detection, num_capture_channels_);
+  }
+  RTC_DCHECK(dominant_nearend_detector_);
 }
 
 SuppressionGain::~SuppressionGain() = default;
@@ -345,8 +354,8 @@ void SuppressionGain::GetGain(
   RTC_DCHECK(low_band_gain);
 
   // Update the nearend state selection.
-  dominant_nearend_detector_.Update(nearend_spectrum, residual_echo_spectrum,
-                                    comfort_noise_spectrum, initial_state_);
+  dominant_nearend_detector_->Update(nearend_spectrum, residual_echo_spectrum,
+                                     comfort_noise_spectrum, initial_state_);
 
   // Compute gain for the lower band.
   bool low_noise_render = low_render_detector_.Detect(render);

modules/audio_processing/aec3/suppression_gain.h

@@ -20,9 +20,9 @@
 #include "api/audio/echo_canceller3_config.h"
 #include "modules/audio_processing/aec3/aec3_common.h"
 #include "modules/audio_processing/aec3/aec_state.h"
-#include "modules/audio_processing/aec3/dominant_nearend_detector.h"
 #include "modules/audio_processing/aec3/fft_data.h"
 #include "modules/audio_processing/aec3/moving_average.h"
+#include "modules/audio_processing/aec3/nearend_detector.h"
 #include "modules/audio_processing/aec3/render_signal_analyzer.h"
 #include "modules/audio_processing/logging/apm_data_dumper.h"
 #include "rtc_base/constructor_magic.h"
@@ -120,7 +120,7 @@ class SuppressionGain {
   std::vector<aec3::MovingAverage> nearend_smoothers_;
   const GainParameters nearend_params_;
   const GainParameters normal_params_;
-  DominantNearendDetector dominant_nearend_detector_;
+  std::unique_ptr<NearendDetector> dominant_nearend_detector_;
 
   RTC_DISALLOW_COPY_AND_ASSIGN(SuppressionGain);
 };