From b237a87a250e24bf49127e9885910e92fcc932ee Mon Sep 17 00:00:00 2001 From: Alessio Bazzica Date: Fri, 11 Jun 2021 12:37:54 +0200 Subject: [PATCH] AGC analog ClippingPredictor refactoring 1/2 - ClippingPredictor API and docstring changes - Unified ClippingPredictor factory function Bug: webrtc:12774 Change-Id: Iafaddae52addc00eb790ac165bf407a4bdd1cb52 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/221540 Reviewed-by: Hanna Silen Commit-Queue: Alessio Bazzica Cr-Commit-Position: refs/heads/master@{#34279} --- .../agc/agc_manager_direct.cc | 22 +------ .../agc/clipping_predictor.cc | 59 ++++++++++--------- .../audio_processing/agc/clipping_predictor.h | 45 ++++++-------- .../agc/clipping_predictor_unittest.cc | 53 +++++++++++------ .../include/audio_processing.h | 21 +++---- 5 files changed, 92 insertions(+), 108 deletions(-) diff --git a/modules/audio_processing/agc/agc_manager_direct.cc b/modules/audio_processing/agc/agc_manager_direct.cc index 46304d2819..9d8dafa0bb 100644 --- a/modules/audio_processing/agc/agc_manager_direct.cc +++ b/modules/audio_processing/agc/agc_manager_direct.cc @@ -129,26 +129,6 @@ float ComputeClippedRatio(const float* const* audio, return static_cast(num_clipped) / (samples_per_channel); } -std::unique_ptr CreateClippingPredictor( - int num_capture_channels, - const ClippingPredictorConfig& config) { - if (config.enabled) { - RTC_LOG(LS_INFO) << "[agc] Clipping prediction enabled."; - switch (config.mode) { - case ClippingPredictorConfig::kClippingEventPrediction: - return CreateClippingEventPredictor(num_capture_channels, config); - case ClippingPredictorConfig::kAdaptiveStepClippingPeakPrediction: - return CreateAdaptiveStepClippingPeakPredictor(num_capture_channels, - config); - case ClippingPredictorConfig::kFixedStepClippingPeakPrediction: - return CreateFixedStepClippingPeakPredictor(num_capture_channels, - config); - } - } else { - return nullptr; - } -} - } // namespace MonoAgc::MonoAgc(ApmDataDumper* data_dumper, @@ -531,7 +511,7 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, if (!!clipping_predictor_) { AudioFrameView frame = AudioFrameView( audio, num_capture_channels_, static_cast(samples_per_channel)); - clipping_predictor_->Process(frame); + clipping_predictor_->Analyze(frame); } if (frames_since_clipped_ < clipped_wait_frames_) { diff --git a/modules/audio_processing/agc/clipping_predictor.cc b/modules/audio_processing/agc/clipping_predictor.cc index deb95f633e..982bbca2ee 100644 --- a/modules/audio_processing/agc/clipping_predictor.cc +++ b/modules/audio_processing/agc/clipping_predictor.cc @@ -25,9 +25,6 @@ namespace { constexpr int kClippingPredictorMaxGainChange = 15; -using ClippingPredictorConfig = AudioProcessing::Config::GainController1:: - AnalogGainController::ClippingPredictor; - // Estimates the new level from the gain error; a copy of the function // `LevelFromGainError` in agc_manager_direct.cc. int LevelFromGainError(int gain_error, @@ -110,7 +107,7 @@ class ClippingEventPredictor : public ClippingPredictor { // Analyzes a frame of audio and stores the framewise metrics in // `ch_buffers_`. - void Process(const AudioFrameView& frame) { + void Analyze(const AudioFrameView& frame) { const int num_channels = frame.num_channels(); RTC_DCHECK_EQ(num_channels, ch_buffers_.size()); const int samples_per_channel = frame.samples_per_channel(); @@ -249,7 +246,7 @@ class ClippingPeakPredictor : public ClippingPredictor { // Analyzes a frame of audio and stores the framewise metrics in // `ch_buffers_`. - void Process(const AudioFrameView& frame) { + void Analyze(const AudioFrameView& frame) { const int num_channels = frame.num_channels(); RTC_DCHECK_EQ(num_channels, ch_buffers_.size()); const int samples_per_channel = frame.samples_per_channel(); @@ -352,31 +349,35 @@ class ClippingPeakPredictor : public ClippingPredictor { } // namespace -std::unique_ptr CreateClippingEventPredictor( +std::unique_ptr CreateClippingPredictor( int num_channels, - const ClippingPredictorConfig& config) { - return std::make_unique( - num_channels, config.window_length, config.reference_window_length, - config.reference_window_delay, config.clipping_threshold, - config.crest_factor_margin); -} - -std::unique_ptr CreateFixedStepClippingPeakPredictor( - int num_channels, - const ClippingPredictorConfig& config) { - return std::make_unique( - num_channels, config.window_length, config.reference_window_length, - config.reference_window_delay, config.clipping_threshold, - /*adaptive_step_estimation=*/false); -} - -std::unique_ptr CreateAdaptiveStepClippingPeakPredictor( - int num_channels, - const ClippingPredictorConfig& config) { - return std::make_unique( - num_channels, config.window_length, config.reference_window_length, - config.reference_window_delay, config.clipping_threshold, - /*adaptive_step_estimation=*/true); + const AudioProcessing::Config::GainController1::AnalogGainController:: + ClippingPredictor& config) { + if (!config.enabled) { + RTC_LOG(LS_INFO) << "[agc] Clipping prediction disabled."; + return nullptr; + } + RTC_LOG(LS_INFO) << "[agc] Clipping prediction enabled."; + using ClippingPredictorMode = AudioProcessing::Config::GainController1:: + AnalogGainController::ClippingPredictor::Mode; + switch (config.mode) { + case ClippingPredictorMode::kClippingEventPrediction: + return std::make_unique( + num_channels, config.window_length, config.reference_window_length, + config.reference_window_delay, config.clipping_threshold, + config.crest_factor_margin); + case ClippingPredictorMode::kAdaptiveStepClippingPeakPrediction: + return std::make_unique( + num_channels, config.window_length, config.reference_window_length, + config.reference_window_delay, config.clipping_threshold, + /*adaptive_step_estimation=*/true); + case ClippingPredictorMode::kFixedStepClippingPeakPrediction: + return std::make_unique( + num_channels, config.window_length, config.reference_window_length, + config.reference_window_delay, config.clipping_threshold, + /*adaptive_step_estimation=*/false); + } + RTC_NOTREACHED(); } } // namespace webrtc diff --git a/modules/audio_processing/agc/clipping_predictor.h b/modules/audio_processing/agc/clipping_predictor.h index 0fe98273fb..ee2b6ef1e7 100644 --- a/modules/audio_processing/agc/clipping_predictor.h +++ b/modules/audio_processing/agc/clipping_predictor.h @@ -20,19 +20,26 @@ namespace webrtc { -// Frame-wise clipping prediction and clipped level step estimation. Processing -// is done in two steps: Calling `Process` analyses a frame of audio and stores -// the frame metrics and `EstimateClippedLevelStep` produces an estimate for the -// required analog gain level decrease if clipping is predicted. +// Frame-wise clipping prediction and clipped level step estimation. Analyzes +// 10 ms multi-channel frames and estimates an analog mic level decrease step +// to possibly avoid clipping when predicted. `Analyze()` and +// `EstimateClippedLevelStep()` can be called in any order. class ClippingPredictor { public: virtual ~ClippingPredictor() = default; virtual void Reset() = 0; - // Estimates the analog gain clipped level step for channel `channel`. - // Returns absl::nullopt if clipping is not predicted, otherwise returns the - // suggested decrease in the analog gain level. + // Analyzes a 10 ms multi-channel audio frame. + virtual void Analyze(const AudioFrameView& frame) = 0; + + // Predicts if clipping is going to occur for the specified `channel` in the + // near-future and, if so, it returns a recommended analog mic level decrease + // step. Returns absl::nullopt if clipping is not predicted. + // `level` is the current analog mic level, `default_step` is the amount the + // mic level is lowered by the analog controller with every clipping event and + // `min_mic_level` and `max_mic_level` is the range of allowed analog mic + // levels. virtual absl::optional EstimateClippedLevelStep( int channel, int level, @@ -40,27 +47,13 @@ class ClippingPredictor { int min_mic_level, int max_mic_level) const = 0; - // Analyses a frame of audio and stores the resulting metrics in `data_`. - virtual void Process(const AudioFrameView& frame) = 0; }; -// Creates a ClippingPredictor based on crest factor-based clipping event -// prediction. -std::unique_ptr CreateClippingEventPredictor( - int num_channels, - const AudioProcessing::Config::GainController1::AnalogGainController:: - ClippingPredictor& config); - -// Creates a ClippingPredictor based on crest factor-based peak estimation and -// fixed-step clipped level step estimation. -std::unique_ptr CreateFixedStepClippingPeakPredictor( - int num_channels, - const AudioProcessing::Config::GainController1::AnalogGainController:: - ClippingPredictor& config); - -// Creates a ClippingPredictor based on crest factor-based peak estimation and -// adaptive-step clipped level step estimation. -std::unique_ptr CreateAdaptiveStepClippingPeakPredictor( +// Creates a ClippingPredictor based on the provided `config`. When enabled, +// the following must hold for `config`: +// `window_length < reference_window_length + reference_window_delay`. +// Returns `nullptr` if `config.enabled` is false. +std::unique_ptr CreateClippingPredictor( int num_channels, const AudioProcessing::Config::GainController1::AnalogGainController:: ClippingPredictor& config); diff --git a/modules/audio_processing/agc/clipping_predictor_unittest.cc b/modules/audio_processing/agc/clipping_predictor_unittest.cc index ab76abab68..e5220ea258 100644 --- a/modules/audio_processing/agc/clipping_predictor_unittest.cc +++ b/modules/audio_processing/agc/clipping_predictor_unittest.cc @@ -34,12 +34,14 @@ constexpr int kDefaultClippedLevelStep = 15; using ClippingPredictorConfig = AudioProcessing::Config::GainController1:: AnalogGainController::ClippingPredictor; +using ClippingPredictorMode = AudioProcessing::Config::GainController1:: + AnalogGainController::ClippingPredictor::Mode; void CallProcess(int num_calls, const AudioFrameView& frame, ClippingPredictor& predictor) { for (int i = 0; i < num_calls; ++i) { - predictor.Process(frame); + predictor.Analyze(frame); } } @@ -149,12 +151,14 @@ TEST_P(ClippingPredictorParameterization, CheckClippingEventPredictorEstimateAfterCrestFactorDrop) { if (reference_window_length() + reference_window_delay() > window_length()) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kClippingEventPrediction; config.window_length = window_length(); config.reference_window_length = reference_window_length(); config.reference_window_delay = reference_window_delay(); config.clipping_threshold = -1.0f; config.crest_factor_margin = 0.5f; - auto predictor = CreateClippingEventPredictor(num_channels(), config); + auto predictor = CreateClippingPredictor(num_channels(), config); ProcessNonZeroCrestFactorAudio( reference_window_length() + reference_window_delay() - window_length(), num_channels(), /*peak_ratio=*/0.99f, *predictor); @@ -173,12 +177,14 @@ TEST_P(ClippingPredictorParameterization, CheckClippingEventPredictorNoEstimateAfterConstantCrestFactor) { if (reference_window_length() + reference_window_delay() > window_length()) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kClippingEventPrediction; config.window_length = window_length(); config.reference_window_length = reference_window_length(); config.reference_window_delay = reference_window_delay(); config.clipping_threshold = -1.0f; config.crest_factor_margin = 0.5f; - auto predictor = CreateClippingEventPredictor(num_channels(), config); + auto predictor = CreateClippingPredictor(num_channels(), config); ProcessNonZeroCrestFactorAudio( reference_window_length() + reference_window_delay() - window_length(), num_channels(), /*peak_ratio=*/0.99f, *predictor); @@ -197,12 +203,13 @@ TEST_P(ClippingPredictorParameterization, CheckClippingPeakPredictorEstimateAfterHighCrestFactor) { if (reference_window_length() + reference_window_delay() > window_length()) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kAdaptiveStepClippingPeakPrediction; config.window_length = window_length(); config.reference_window_length = reference_window_length(); config.reference_window_delay = reference_window_delay(); config.clipping_threshold = -1.0f; - auto predictor = - CreateAdaptiveStepClippingPeakPredictor(num_channels(), config); + auto predictor = CreateClippingPredictor(num_channels(), config); ProcessNonZeroCrestFactorAudio( reference_window_length() + reference_window_delay() - window_length(), num_channels(), /*peak_ratio=*/0.99f, *predictor); @@ -221,12 +228,13 @@ TEST_P(ClippingPredictorParameterization, CheckClippingPeakPredictorNoEstimateAfterLowCrestFactor) { if (reference_window_length() + reference_window_delay() > window_length()) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kAdaptiveStepClippingPeakPrediction; config.window_length = window_length(); config.reference_window_length = reference_window_length(); config.reference_window_delay = reference_window_delay(); config.clipping_threshold = -1.0f; - auto predictor = - CreateAdaptiveStepClippingPeakPredictor(num_channels(), config); + auto predictor = CreateClippingPredictor(num_channels(), config); ProcessZeroCrestFactorAudio( reference_window_length() + reference_window_delay() - window_length(), num_channels(), /*peak_ratio=*/0.99f, *predictor); @@ -251,12 +259,14 @@ INSTANTIATE_TEST_SUITE_P(GainController1ClippingPredictor, TEST_P(ClippingEventPredictorParameterization, CheckEstimateAfterCrestFactorDrop) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kClippingEventPrediction; config.window_length = kWindowLength; config.reference_window_length = kReferenceWindowLength; config.reference_window_delay = kReferenceWindowDelay; config.clipping_threshold = clipping_threshold(); config.crest_factor_margin = crest_factor_margin(); - auto predictor = CreateClippingEventPredictor(kNumChannels, config); + auto predictor = CreateClippingPredictor(kNumChannels, config); ProcessNonZeroCrestFactorAudio(kReferenceWindowLength, kNumChannels, /*peak_ratio=*/0.99f, *predictor); CheckChannelEstimatesWithoutValue(kNumChannels, /*level=*/255, @@ -284,14 +294,15 @@ INSTANTIATE_TEST_SUITE_P(GainController1ClippingPredictor, TEST_P(ClippingPeakPredictorParameterization, CheckEstimateAfterHighCrestFactor) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = adaptive_step_estimation() + ? ClippingPredictorMode::kAdaptiveStepClippingPeakPrediction + : ClippingPredictorMode::kFixedStepClippingPeakPrediction; config.window_length = kWindowLength; config.reference_window_length = kReferenceWindowLength; config.reference_window_delay = kReferenceWindowDelay; config.clipping_threshold = clipping_threshold(); - auto predictor = - adaptive_step_estimation() - ? CreateAdaptiveStepClippingPeakPredictor(kNumChannels, config) - : CreateFixedStepClippingPeakPredictor(kNumChannels, config); + auto predictor = CreateClippingPredictor(kNumChannels, config); ProcessNonZeroCrestFactorAudio(kReferenceWindowLength, kNumChannels, /*peak_ratio=*/0.99f, *predictor); CheckChannelEstimatesWithoutValue(kNumChannels, /*level=*/255, @@ -324,12 +335,14 @@ INSTANTIATE_TEST_SUITE_P(GainController1ClippingPredictor, TEST(ClippingEventPredictorTest, CheckEstimateAfterReset) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kClippingEventPrediction; config.window_length = kWindowLength; config.reference_window_length = kReferenceWindowLength; config.reference_window_delay = kReferenceWindowDelay; config.clipping_threshold = -1.0f; config.crest_factor_margin = 3.0f; - auto predictor = CreateClippingEventPredictor(kNumChannels, config); + auto predictor = CreateClippingPredictor(kNumChannels, config); ProcessNonZeroCrestFactorAudio(kReferenceWindowLength, kNumChannels, /*peak_ratio=*/0.99f, *predictor); CheckChannelEstimatesWithoutValue(kNumChannels, /*level=*/255, @@ -345,13 +358,14 @@ TEST(ClippingEventPredictorTest, CheckEstimateAfterReset) { TEST(ClippingPeakPredictorTest, CheckNoEstimateAfterReset) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kAdaptiveStepClippingPeakPrediction; config.window_length = kWindowLength; config.reference_window_length = kReferenceWindowLength; config.reference_window_delay = kReferenceWindowDelay; config.clipping_threshold = -1.0f; config.crest_factor_margin = 3.0f; - auto predictor = - CreateAdaptiveStepClippingPeakPredictor(kNumChannels, config); + auto predictor = CreateClippingPredictor(kNumChannels, config); ProcessNonZeroCrestFactorAudio(kReferenceWindowLength, kNumChannels, /*peak_ratio=*/0.99f, *predictor); CheckChannelEstimatesWithoutValue(kNumChannels, /*level=*/255, @@ -367,12 +381,13 @@ TEST(ClippingPeakPredictorTest, CheckNoEstimateAfterReset) { TEST(ClippingPeakPredictorTest, CheckAdaptiveStepEstimate) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kAdaptiveStepClippingPeakPrediction; config.window_length = kWindowLength; config.reference_window_length = kReferenceWindowLength; config.reference_window_delay = kReferenceWindowDelay; config.clipping_threshold = -1.0f; - auto predictor = - CreateAdaptiveStepClippingPeakPredictor(kNumChannels, config); + auto predictor = CreateClippingPredictor(kNumChannels, config); ProcessNonZeroCrestFactorAudio(kReferenceWindowLength, kNumChannels, /*peak_ratio=*/0.99f, *predictor); CheckChannelEstimatesWithoutValue(kNumChannels, /*level=*/255, @@ -387,11 +402,13 @@ TEST(ClippingPeakPredictorTest, CheckAdaptiveStepEstimate) { TEST(ClippingPeakPredictorTest, CheckFixedStepEstimate) { ClippingPredictorConfig config; + config.enabled = true; + config.mode = ClippingPredictorMode::kFixedStepClippingPeakPrediction; config.window_length = kWindowLength; config.reference_window_length = kReferenceWindowLength; config.reference_window_delay = kReferenceWindowDelay; config.clipping_threshold = -1.0f; - auto predictor = CreateFixedStepClippingPeakPredictor(kNumChannels, config); + auto predictor = CreateClippingPredictor(kNumChannels, config); ProcessNonZeroCrestFactorAudio(kReferenceWindowLength, kNumChannels, /*peak_ratio=*/0.99f, *predictor); CheckChannelEstimatesWithoutValue(kNumChannels, /*level=*/255, diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index 1f3c19c464..b03d6d864f 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -348,28 +348,21 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { struct ClippingPredictor { bool enabled = false; enum Mode { - // Sets clipping prediction for clipping event prediction with fixed - // step estimation. + // Clipping event prediction mode with fixed step estimation. kClippingEventPrediction, - // Sets clipping prediction for clipped peak estimation with - // adaptive step estimation. + // Clipped peak estimation mode with adaptive step estimation. kAdaptiveStepClippingPeakPrediction, - // Sets clipping prediction for clipped peak estimation with fixed - // step estimation. + // Clipped peak estimation mode with fixed step estimation. kFixedStepClippingPeakPrediction, }; Mode mode = kClippingEventPrediction; - // Number of frames in the sliding analysis window. Limited to values - // higher than zero. + // Number of frames in the sliding analysis window. int window_length = 5; - // Number of frames in the sliding reference window. Limited to values - // higher than zero. + // Number of frames in the sliding reference window. int reference_window_length = 5; - // Number of frames the reference window is delayed. Limited to values - // zero and higher. An additional requirement: - // |window_length < reference_window_length + reference_window_delay|. + // Reference window delay (unit: number of frames). int reference_window_delay = 5; - // Clipping predictor ste estimation threshold (dB). + // Clipping prediction threshold (dBFS). float clipping_threshold = -1.0f; // Crest factor drop threshold (dB). float crest_factor_margin = 3.0f;