From 42dacda82cb223c030f5c74529539611f7cfb5dc Mon Sep 17 00:00:00 2001 From: Alessio Bazzica Date: Thu, 17 Jun 2021 17:18:46 +0200 Subject: [PATCH] AGC analog clipping predictor: integrate evaluator Integrate ClippingPredictorEvaluator in AgcManagerDirect adding the possibility to run the predictor without affecting the analog gain adjustment process. The evaluator is used to compute precision, recall and F1 score. F1 score and the measured clipping prediction intervals are logged as `WebRTC.Audio.Agc.ClippingPredictor.F1Score` and `.PredictionInterval` histograms respectively. Bug: webrtc:12774 Change-Id: I708dcda9321f92d5bd17ec4c36ebce1165ead57f Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/221921 Commit-Queue: Alessio Bazzica Reviewed-by: Hanna Silen Cr-Commit-Position: refs/heads/master@{#34327} --- modules/audio_processing/agc/BUILD.gn | 1 + .../agc/agc_manager_direct.cc | 100 +++++++++++++----- .../audio_processing/agc/agc_manager_direct.h | 58 ++++++---- .../agc/agc_manager_direct_unittest.cc | 57 +++++++--- .../include/audio_processing.h | 4 + 5 files changed, 160 insertions(+), 60 deletions(-) diff --git a/modules/audio_processing/agc/BUILD.gn b/modules/audio_processing/agc/BUILD.gn index ca6d9bdb0f..4bb8c5494b 100644 --- a/modules/audio_processing/agc/BUILD.gn +++ b/modules/audio_processing/agc/BUILD.gn @@ -20,6 +20,7 @@ rtc_library("agc") { configs += [ "..:apm_debug_dump" ] deps = [ ":clipping_predictor", + ":clipping_predictor_evaluator", ":gain_control_interface", ":gain_map", ":level_estimation", diff --git a/modules/audio_processing/agc/agc_manager_direct.cc b/modules/audio_processing/agc/agc_manager_direct.cc index 9d8dafa0bb..817678801e 100644 --- a/modules/audio_processing/agc/agc_manager_direct.cc +++ b/modules/audio_processing/agc/agc_manager_direct.cc @@ -49,6 +49,10 @@ constexpr int kMaxResidualGainChange = 15; // restrictions from clipping events. constexpr int kSurplusCompressionGain = 6; +// History size for the clipping predictor evaluator (unit: number of 10 ms +// frames). +constexpr int kClippingPredictorEvaluatorHistorySize = 32; + using ClippingPredictorConfig = AudioProcessing::Config::GainController1:: AnalogGainController::ClippingPredictor; @@ -129,6 +133,33 @@ float ComputeClippedRatio(const float* const* audio, return static_cast(num_clipped) / (samples_per_channel); } +void LogClippingPredictorMetrics(const ClippingPredictorEvaluator& evaluator) { + RTC_LOG(LS_INFO) << "Clipping predictor metrics: TP " + << evaluator.true_positives() << " TN " + << evaluator.true_negatives() << " FP " + << evaluator.false_positives() << " FN " + << evaluator.false_negatives(); + const float precision_denominator = + evaluator.true_positives() + evaluator.false_positives(); + const float recall_denominator = + evaluator.true_positives() + evaluator.false_negatives(); + if (precision_denominator > 0 && recall_denominator > 0) { + const float precision = evaluator.true_positives() / precision_denominator; + const float recall = evaluator.true_positives() / recall_denominator; + RTC_LOG(LS_INFO) << "Clipping predictor metrics: P " << precision << " R " + << recall; + const float f1_score_denominator = precision + recall; + if (f1_score_denominator > 0.0f) { + const float f1_score = 2 * precision * recall / f1_score_denominator; + RTC_LOG(LS_INFO) << "Clipping predictor metrics: F1 " << f1_score; + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc.ClippingPredictor.F1Score", + std::round(f1_score * 100.0f), /*min=*/0, + /*max=*/100, + /*bucket_count=*/50); + } + } +} + } // namespace MonoAgc::MonoAgc(ApmDataDumper* data_dumper, @@ -398,14 +429,15 @@ void MonoAgc::UpdateCompressor() { int AgcManagerDirect::instance_counter_ = 0; -AgcManagerDirect::AgcManagerDirect(Agc* agc, - int startup_min_level, - int clipped_level_min, - int sample_rate_hz, - int clipped_level_step, - float clipped_ratio_threshold, - int clipped_wait_frames, - const ClippingPredictorConfig& clipping_cfg) +AgcManagerDirect::AgcManagerDirect( + Agc* agc, + int startup_min_level, + int clipped_level_min, + int sample_rate_hz, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const ClippingPredictorConfig& clipping_config) : AgcManagerDirect(/*num_capture_channels*/ 1, startup_min_level, clipped_level_min, @@ -414,21 +446,22 @@ AgcManagerDirect::AgcManagerDirect(Agc* agc, clipped_level_step, clipped_ratio_threshold, clipped_wait_frames, - clipping_cfg) { + clipping_config) { RTC_DCHECK(channel_agcs_[0]); RTC_DCHECK(agc); channel_agcs_[0]->set_agc(agc); } -AgcManagerDirect::AgcManagerDirect(int num_capture_channels, - int startup_min_level, - int clipped_level_min, - bool disable_digital_adaptive, - int sample_rate_hz, - int clipped_level_step, - float clipped_ratio_threshold, - int clipped_wait_frames, - const ClippingPredictorConfig& clipping_cfg) +AgcManagerDirect::AgcManagerDirect( + int num_capture_channels, + int startup_min_level, + int clipped_level_min, + bool disable_digital_adaptive, + int sample_rate_hz, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const ClippingPredictorConfig& clipping_config) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_counter_))), use_min_channel_level_(!UseMaxAnalogChannelLevel()), @@ -443,7 +476,11 @@ AgcManagerDirect::AgcManagerDirect(int num_capture_channels, channel_agcs_(num_capture_channels), new_compressions_to_set_(num_capture_channels), clipping_predictor_( - CreateClippingPredictor(num_capture_channels, clipping_cfg)) { + CreateClippingPredictor(num_capture_channels, clipping_config)), + use_clipping_predictor_step_(!!clipping_predictor_ && + clipping_config.use_predicted_step), + clipping_predictor_evaluator_(kClippingPredictorEvaluatorHistorySize), + clipping_predictor_log_counter_(0) { const int min_mic_level = GetMinMicLevel(); for (size_t ch = 0; ch < channel_agcs_.size(); ++ch) { ApmDataDumper* data_dumper_ch = ch == 0 ? data_dumper_.get() : nullptr; @@ -472,6 +509,8 @@ void AgcManagerDirect::Initialize() { capture_output_used_ = true; AggregateChannelLevels(); + clipping_predictor_evaluator_.Reset(); + clipping_predictor_log_counter_ = 0; } void AgcManagerDirect::SetupDigitalGainControl( @@ -538,11 +577,27 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, const auto step = clipping_predictor_->EstimateClippedLevelStep( channel, stream_analog_level_, clipped_level_step_, channel_agcs_[channel]->min_mic_level(), kMaxMicLevel); - if (step.has_value()) { + if (use_clipping_predictor_step_ && step.has_value()) { predicted_step = std::max(predicted_step, step.value()); clipping_predicted = true; } } + // Clipping prediction evaluation. + absl::optional prediction_interval = + clipping_predictor_evaluator_.Observe(clipping_detected, + clipping_predicted); + if (prediction_interval.has_value()) { + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.Agc.ClippingPredictor.PredictionInterval", + prediction_interval.value(), /*min=*/0, + /*max=*/49, /*bucket_count=*/50); + } + constexpr int kNumFramesIn30Seconds = 3000; + clipping_predictor_log_counter_++; + if (clipping_predictor_log_counter_ == kNumFramesIn30Seconds) { + LogClippingPredictorMetrics(clipping_predictor_evaluator_); + clipping_predictor_log_counter_ = 0; + } } if (clipping_detected || clipping_predicted) { int step = clipped_level_step_; @@ -560,6 +615,7 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, frames_since_clipped_ = 0; if (!!clipping_predictor_) { clipping_predictor_->Reset(); + clipping_predictor_evaluator_.Reset(); } } AggregateChannelLevels(); @@ -643,8 +699,4 @@ void AgcManagerDirect::AggregateChannelLevels() { } } -bool AgcManagerDirect::clipping_predictor_enabled() const { - return !!clipping_predictor_; -} - } // namespace webrtc diff --git a/modules/audio_processing/agc/agc_manager_direct.h b/modules/audio_processing/agc/agc_manager_direct.h index 55a7ffa2eb..7ac96a661c 100644 --- a/modules/audio_processing/agc/agc_manager_direct.h +++ b/modules/audio_processing/agc/agc_manager_direct.h @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "modules/audio_processing/agc/agc.h" #include "modules/audio_processing/agc/clipping_predictor.h" +#include "modules/audio_processing/agc/clipping_predictor_evaluator.h" #include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/gtest_prod_util.h" @@ -41,16 +42,17 @@ class AgcManagerDirect final { // samples required to declare a clipping event, limited to (0.f, 1.f). // `clipped_wait_frames` is the time in frames to wait after a clipping event // before checking again, limited to values higher than 0. - AgcManagerDirect(int num_capture_channels, - int startup_min_level, - int clipped_level_min, - bool disable_digital_adaptive, - int sample_rate_hz, - int clipped_level_step, - float clipped_ratio_threshold, - int clipped_wait_frames, - const AudioProcessing::Config::GainController1:: - AnalogGainController::ClippingPredictor& clipping_cfg); + AgcManagerDirect( + int num_capture_channels, + int startup_min_level, + int clipped_level_min, + bool disable_digital_adaptive, + int sample_rate_hz, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const AudioProcessing::Config::GainController1::AnalogGainController:: + ClippingPredictor& clipping_config); ~AgcManagerDirect(); AgcManagerDirect(const AgcManagerDirect&) = delete; @@ -72,12 +74,17 @@ class AgcManagerDirect final { int num_channels() const { return num_capture_channels_; } int sample_rate_hz() const { return sample_rate_hz_; } - // Returns true if clipping prediction was set to be used in ctor. - bool clipping_predictor_enabled() const; - // If available, returns a new compression gain for the digital gain control. absl::optional GetDigitalComressionGain(); + // Returns true if clipping prediction is enabled. + bool clipping_predictor_enabled() const { return !!clipping_predictor_; } + + // Returns true if clipping prediction is used to adjust the analog gain. + bool use_clipping_predictor_step() const { + return use_clipping_predictor_step_; + } + private: friend class AgcManagerDirectTest; @@ -99,20 +106,24 @@ class AgcManagerDirect final { ClippingParametersVerified); FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, DisableClippingPredictorDoesNotLowerVolume); + FRIEND_TEST_ALL_PREFIXES( + AgcManagerDirectStandaloneTest, + EnableClippingPredictorWithUnusedPredictedStepDoesNotLowerVolume); FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, EnableClippingPredictorLowersVolume); // Dependency injection for testing. Don't delete |agc| as the memory is owned // by the manager. - AgcManagerDirect(Agc* agc, - int startup_min_level, - int clipped_level_min, - int sample_rate_hz, - int clipped_level_step, - float clipped_ratio_threshold, - int clipped_wait_frames, - const AudioProcessing::Config::GainController1:: - AnalogGainController::ClippingPredictor& clipping_cfg); + AgcManagerDirect( + Agc* agc, + int startup_min_level, + int clipped_level_min, + int sample_rate_hz, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const AudioProcessing::Config::GainController1::AnalogGainController:: + ClippingPredictor& clipping_config); void AnalyzePreProcess(const float* const* audio, size_t samples_per_channel); @@ -138,6 +149,9 @@ class AgcManagerDirect final { std::vector> new_compressions_to_set_; const std::unique_ptr clipping_predictor_; + const bool use_clipping_predictor_step_; + ClippingPredictorEvaluator clipping_predictor_evaluator_; + int clipping_predictor_log_counter_; }; class MonoAgc { diff --git a/modules/audio_processing/agc/agc_manager_direct_unittest.cc b/modules/audio_processing/agc/agc_manager_direct_unittest.cc index 07bb04022b..bb284f9abc 100644 --- a/modules/audio_processing/agc/agc_manager_direct_unittest.cc +++ b/modules/audio_processing/agc/agc_manager_direct_unittest.cc @@ -907,33 +907,62 @@ TEST(AgcManagerDirectStandaloneTest, kClippedWaitFrames, default_config); manager->Initialize(); EXPECT_FALSE(manager->clipping_predictor_enabled()); + EXPECT_FALSE(manager->use_clipping_predictor_step()); +} + +TEST(AgcManagerDirectStandaloneTest, ClippingPredictorDisabledByDefault) { + constexpr ClippingPredictorConfig kDefaultConfig; + EXPECT_FALSE(kDefaultConfig.enabled); } TEST(AgcManagerDirectStandaloneTest, EnableClippingPredictorEnablesClippingPredictor) { - const ClippingPredictorConfig config( - {/*enabled=*/true, ClippingPredictorConfig::kClippingEventPrediction, - /*window_length=*/5, /*reference_window_length=*/5, - /*reference_window_delay=*/5, /*clipping_threshold=*/-1.0f, - /*crest_factor_margin=*/3.0f}); + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + ClippingPredictorConfig config; + config.enabled = true; + config.use_predicted_step = true; std::unique_ptr manager = CreateAgcManagerDirect( kInitialVolume, kClippedLevelStep, kClippedRatioThreshold, kClippedWaitFrames, config); manager->Initialize(); EXPECT_TRUE(manager->clipping_predictor_enabled()); + EXPECT_TRUE(manager->use_clipping_predictor_step()); } TEST(AgcManagerDirectStandaloneTest, DisableClippingPredictorDoesNotLowerVolume) { - const ClippingPredictorConfig default_config; - EXPECT_FALSE(default_config.enabled); + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + constexpr ClippingPredictorConfig kConfig{/*enabled=*/false}; AgcManagerDirect manager(new ::testing::NiceMock(), kInitialVolume, kClippedMin, kSampleRateHz, kClippedLevelStep, - kClippedRatioThreshold, kClippedWaitFrames, - default_config); + kClippedRatioThreshold, kClippedWaitFrames, kConfig); manager.Initialize(); manager.set_stream_analog_level(/*level=*/255); EXPECT_FALSE(manager.clipping_predictor_enabled()); + EXPECT_FALSE(manager.use_clipping_predictor_step()); + EXPECT_EQ(manager.stream_analog_level(), 255); + manager.Process(nullptr); + CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); + CallPreProcessAudioBuffer(/*num_calls=*/300, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); + CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); +} + +TEST(AgcManagerDirectStandaloneTest, + EnableClippingPredictorWithUnusedPredictedStepDoesNotLowerVolume) { + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + ClippingPredictorConfig config; + config.enabled = true; + config.use_predicted_step = false; + AgcManagerDirect manager(new ::testing::NiceMock(), kInitialVolume, + kClippedMin, kSampleRateHz, kClippedLevelStep, + kClippedRatioThreshold, kClippedWaitFrames, config); + manager.Initialize(); + manager.set_stream_analog_level(/*level=*/255); + EXPECT_TRUE(manager.clipping_predictor_enabled()); + EXPECT_FALSE(manager.use_clipping_predictor_step()); EXPECT_EQ(manager.stream_analog_level(), 255); manager.Process(nullptr); CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); @@ -945,17 +974,17 @@ TEST(AgcManagerDirectStandaloneTest, } TEST(AgcManagerDirectStandaloneTest, EnableClippingPredictorLowersVolume) { - const ClippingPredictorConfig config( - {/*enabled=*/true, ClippingPredictorConfig::kClippingEventPrediction, - /*window_length=*/5, /*reference_window_length=*/5, - /*reference_window_delay=*/5, /*clipping_threshold=*/-1.0f, - /*crest_factor_margin=*/3.0f}); + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + ClippingPredictorConfig config; + config.enabled = true; + config.use_predicted_step = true; AgcManagerDirect manager(new ::testing::NiceMock(), kInitialVolume, kClippedMin, kSampleRateHz, kClippedLevelStep, kClippedRatioThreshold, kClippedWaitFrames, config); manager.Initialize(); manager.set_stream_analog_level(/*level=*/255); EXPECT_TRUE(manager.clipping_predictor_enabled()); + EXPECT_TRUE(manager.use_clipping_predictor_step()); EXPECT_EQ(manager.stream_analog_level(), 255); manager.Process(nullptr); CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index b03d6d864f..64b1b5d107 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -366,6 +366,10 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { float clipping_threshold = -1.0f; // Crest factor drop threshold (dB). float crest_factor_margin = 3.0f; + // If true, the recommended clipped level step is used to modify the + // analog gain. Otherwise, the predictor runs without affecting the + // analog gain. + bool use_predicted_step = true; } clipping_predictor; } analog_gain_controller; } gain_controller1;