diff --git a/modules/audio_processing/agc/BUILD.gn b/modules/audio_processing/agc/BUILD.gn index ca6d9bdb0f..4bb8c5494b 100644 --- a/modules/audio_processing/agc/BUILD.gn +++ b/modules/audio_processing/agc/BUILD.gn @@ -20,6 +20,7 @@ rtc_library("agc") { configs += [ "..:apm_debug_dump" ] deps = [ ":clipping_predictor", + ":clipping_predictor_evaluator", ":gain_control_interface", ":gain_map", ":level_estimation", diff --git a/modules/audio_processing/agc/agc_manager_direct.cc b/modules/audio_processing/agc/agc_manager_direct.cc index 9d8dafa0bb..817678801e 100644 --- a/modules/audio_processing/agc/agc_manager_direct.cc +++ b/modules/audio_processing/agc/agc_manager_direct.cc @@ -49,6 +49,10 @@ constexpr int kMaxResidualGainChange = 15; // restrictions from clipping events. constexpr int kSurplusCompressionGain = 6; +// History size for the clipping predictor evaluator (unit: number of 10 ms +// frames). +constexpr int kClippingPredictorEvaluatorHistorySize = 32; + using ClippingPredictorConfig = AudioProcessing::Config::GainController1:: AnalogGainController::ClippingPredictor; @@ -129,6 +133,33 @@ float ComputeClippedRatio(const float* const* audio, return static_cast(num_clipped) / (samples_per_channel); } +void LogClippingPredictorMetrics(const ClippingPredictorEvaluator& evaluator) { + RTC_LOG(LS_INFO) << "Clipping predictor metrics: TP " + << evaluator.true_positives() << " TN " + << evaluator.true_negatives() << " FP " + << evaluator.false_positives() << " FN " + << evaluator.false_negatives(); + const float precision_denominator = + evaluator.true_positives() + evaluator.false_positives(); + const float recall_denominator = + evaluator.true_positives() + evaluator.false_negatives(); + if (precision_denominator > 0 && recall_denominator > 0) { + const float precision = evaluator.true_positives() / precision_denominator; + const float recall = evaluator.true_positives() / recall_denominator; + RTC_LOG(LS_INFO) << "Clipping predictor metrics: P " << precision << " R " + << recall; + const float f1_score_denominator = precision + recall; + if (f1_score_denominator > 0.0f) { + const float f1_score = 2 * precision * recall / f1_score_denominator; + RTC_LOG(LS_INFO) << "Clipping predictor metrics: F1 " << f1_score; + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc.ClippingPredictor.F1Score", + std::round(f1_score * 100.0f), /*min=*/0, + /*max=*/100, + /*bucket_count=*/50); + } + } +} + } // namespace MonoAgc::MonoAgc(ApmDataDumper* data_dumper, @@ -398,14 +429,15 @@ void MonoAgc::UpdateCompressor() { int AgcManagerDirect::instance_counter_ = 0; -AgcManagerDirect::AgcManagerDirect(Agc* agc, - int startup_min_level, - int clipped_level_min, - int sample_rate_hz, - int clipped_level_step, - float clipped_ratio_threshold, - int clipped_wait_frames, - const ClippingPredictorConfig& clipping_cfg) +AgcManagerDirect::AgcManagerDirect( + Agc* agc, + int startup_min_level, + int clipped_level_min, + int sample_rate_hz, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const ClippingPredictorConfig& clipping_config) : AgcManagerDirect(/*num_capture_channels*/ 1, startup_min_level, clipped_level_min, @@ -414,21 +446,22 @@ AgcManagerDirect::AgcManagerDirect(Agc* agc, clipped_level_step, clipped_ratio_threshold, clipped_wait_frames, - clipping_cfg) { + clipping_config) { RTC_DCHECK(channel_agcs_[0]); RTC_DCHECK(agc); channel_agcs_[0]->set_agc(agc); } -AgcManagerDirect::AgcManagerDirect(int num_capture_channels, - int startup_min_level, - int clipped_level_min, - bool disable_digital_adaptive, - int sample_rate_hz, - int clipped_level_step, - float clipped_ratio_threshold, - int clipped_wait_frames, - const ClippingPredictorConfig& clipping_cfg) +AgcManagerDirect::AgcManagerDirect( + int num_capture_channels, + int startup_min_level, + int clipped_level_min, + bool disable_digital_adaptive, + int sample_rate_hz, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const ClippingPredictorConfig& clipping_config) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_counter_))), use_min_channel_level_(!UseMaxAnalogChannelLevel()), @@ -443,7 +476,11 @@ AgcManagerDirect::AgcManagerDirect(int num_capture_channels, channel_agcs_(num_capture_channels), new_compressions_to_set_(num_capture_channels), clipping_predictor_( - CreateClippingPredictor(num_capture_channels, clipping_cfg)) { + CreateClippingPredictor(num_capture_channels, clipping_config)), + use_clipping_predictor_step_(!!clipping_predictor_ && + clipping_config.use_predicted_step), + clipping_predictor_evaluator_(kClippingPredictorEvaluatorHistorySize), + clipping_predictor_log_counter_(0) { const int min_mic_level = GetMinMicLevel(); for (size_t ch = 0; ch < channel_agcs_.size(); ++ch) { ApmDataDumper* data_dumper_ch = ch == 0 ? data_dumper_.get() : nullptr; @@ -472,6 +509,8 @@ void AgcManagerDirect::Initialize() { capture_output_used_ = true; AggregateChannelLevels(); + clipping_predictor_evaluator_.Reset(); + clipping_predictor_log_counter_ = 0; } void AgcManagerDirect::SetupDigitalGainControl( @@ -538,11 +577,27 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, const auto step = clipping_predictor_->EstimateClippedLevelStep( channel, stream_analog_level_, clipped_level_step_, channel_agcs_[channel]->min_mic_level(), kMaxMicLevel); - if (step.has_value()) { + if (use_clipping_predictor_step_ && step.has_value()) { predicted_step = std::max(predicted_step, step.value()); clipping_predicted = true; } } + // Clipping prediction evaluation. + absl::optional prediction_interval = + clipping_predictor_evaluator_.Observe(clipping_detected, + clipping_predicted); + if (prediction_interval.has_value()) { + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.Agc.ClippingPredictor.PredictionInterval", + prediction_interval.value(), /*min=*/0, + /*max=*/49, /*bucket_count=*/50); + } + constexpr int kNumFramesIn30Seconds = 3000; + clipping_predictor_log_counter_++; + if (clipping_predictor_log_counter_ == kNumFramesIn30Seconds) { + LogClippingPredictorMetrics(clipping_predictor_evaluator_); + clipping_predictor_log_counter_ = 0; + } } if (clipping_detected || clipping_predicted) { int step = clipped_level_step_; @@ -560,6 +615,7 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, frames_since_clipped_ = 0; if (!!clipping_predictor_) { clipping_predictor_->Reset(); + clipping_predictor_evaluator_.Reset(); } } AggregateChannelLevels(); @@ -643,8 +699,4 @@ void AgcManagerDirect::AggregateChannelLevels() { } } -bool AgcManagerDirect::clipping_predictor_enabled() const { - return !!clipping_predictor_; -} - } // namespace webrtc diff --git a/modules/audio_processing/agc/agc_manager_direct.h b/modules/audio_processing/agc/agc_manager_direct.h index 55a7ffa2eb..7ac96a661c 100644 --- a/modules/audio_processing/agc/agc_manager_direct.h +++ b/modules/audio_processing/agc/agc_manager_direct.h @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "modules/audio_processing/agc/agc.h" #include "modules/audio_processing/agc/clipping_predictor.h" +#include "modules/audio_processing/agc/clipping_predictor_evaluator.h" #include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/gtest_prod_util.h" @@ -41,16 +42,17 @@ class AgcManagerDirect final { // samples required to declare a clipping event, limited to (0.f, 1.f). // `clipped_wait_frames` is the time in frames to wait after a clipping event // before checking again, limited to values higher than 0. - AgcManagerDirect(int num_capture_channels, - int startup_min_level, - int clipped_level_min, - bool disable_digital_adaptive, - int sample_rate_hz, - int clipped_level_step, - float clipped_ratio_threshold, - int clipped_wait_frames, - const AudioProcessing::Config::GainController1:: - AnalogGainController::ClippingPredictor& clipping_cfg); + AgcManagerDirect( + int num_capture_channels, + int startup_min_level, + int clipped_level_min, + bool disable_digital_adaptive, + int sample_rate_hz, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const AudioProcessing::Config::GainController1::AnalogGainController:: + ClippingPredictor& clipping_config); ~AgcManagerDirect(); AgcManagerDirect(const AgcManagerDirect&) = delete; @@ -72,12 +74,17 @@ class AgcManagerDirect final { int num_channels() const { return num_capture_channels_; } int sample_rate_hz() const { return sample_rate_hz_; } - // Returns true if clipping prediction was set to be used in ctor. - bool clipping_predictor_enabled() const; - // If available, returns a new compression gain for the digital gain control. absl::optional GetDigitalComressionGain(); + // Returns true if clipping prediction is enabled. + bool clipping_predictor_enabled() const { return !!clipping_predictor_; } + + // Returns true if clipping prediction is used to adjust the analog gain. + bool use_clipping_predictor_step() const { + return use_clipping_predictor_step_; + } + private: friend class AgcManagerDirectTest; @@ -99,20 +106,24 @@ class AgcManagerDirect final { ClippingParametersVerified); FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, DisableClippingPredictorDoesNotLowerVolume); + FRIEND_TEST_ALL_PREFIXES( + AgcManagerDirectStandaloneTest, + EnableClippingPredictorWithUnusedPredictedStepDoesNotLowerVolume); FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, EnableClippingPredictorLowersVolume); // Dependency injection for testing. Don't delete |agc| as the memory is owned // by the manager. - AgcManagerDirect(Agc* agc, - int startup_min_level, - int clipped_level_min, - int sample_rate_hz, - int clipped_level_step, - float clipped_ratio_threshold, - int clipped_wait_frames, - const AudioProcessing::Config::GainController1:: - AnalogGainController::ClippingPredictor& clipping_cfg); + AgcManagerDirect( + Agc* agc, + int startup_min_level, + int clipped_level_min, + int sample_rate_hz, + int clipped_level_step, + float clipped_ratio_threshold, + int clipped_wait_frames, + const AudioProcessing::Config::GainController1::AnalogGainController:: + ClippingPredictor& clipping_config); void AnalyzePreProcess(const float* const* audio, size_t samples_per_channel); @@ -138,6 +149,9 @@ class AgcManagerDirect final { std::vector> new_compressions_to_set_; const std::unique_ptr clipping_predictor_; + const bool use_clipping_predictor_step_; + ClippingPredictorEvaluator clipping_predictor_evaluator_; + int clipping_predictor_log_counter_; }; class MonoAgc { diff --git a/modules/audio_processing/agc/agc_manager_direct_unittest.cc b/modules/audio_processing/agc/agc_manager_direct_unittest.cc index 07bb04022b..bb284f9abc 100644 --- a/modules/audio_processing/agc/agc_manager_direct_unittest.cc +++ b/modules/audio_processing/agc/agc_manager_direct_unittest.cc @@ -907,33 +907,62 @@ TEST(AgcManagerDirectStandaloneTest, kClippedWaitFrames, default_config); manager->Initialize(); EXPECT_FALSE(manager->clipping_predictor_enabled()); + EXPECT_FALSE(manager->use_clipping_predictor_step()); +} + +TEST(AgcManagerDirectStandaloneTest, ClippingPredictorDisabledByDefault) { + constexpr ClippingPredictorConfig kDefaultConfig; + EXPECT_FALSE(kDefaultConfig.enabled); } TEST(AgcManagerDirectStandaloneTest, EnableClippingPredictorEnablesClippingPredictor) { - const ClippingPredictorConfig config( - {/*enabled=*/true, ClippingPredictorConfig::kClippingEventPrediction, - /*window_length=*/5, /*reference_window_length=*/5, - /*reference_window_delay=*/5, /*clipping_threshold=*/-1.0f, - /*crest_factor_margin=*/3.0f}); + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + ClippingPredictorConfig config; + config.enabled = true; + config.use_predicted_step = true; std::unique_ptr manager = CreateAgcManagerDirect( kInitialVolume, kClippedLevelStep, kClippedRatioThreshold, kClippedWaitFrames, config); manager->Initialize(); EXPECT_TRUE(manager->clipping_predictor_enabled()); + EXPECT_TRUE(manager->use_clipping_predictor_step()); } TEST(AgcManagerDirectStandaloneTest, DisableClippingPredictorDoesNotLowerVolume) { - const ClippingPredictorConfig default_config; - EXPECT_FALSE(default_config.enabled); + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + constexpr ClippingPredictorConfig kConfig{/*enabled=*/false}; AgcManagerDirect manager(new ::testing::NiceMock(), kInitialVolume, kClippedMin, kSampleRateHz, kClippedLevelStep, - kClippedRatioThreshold, kClippedWaitFrames, - default_config); + kClippedRatioThreshold, kClippedWaitFrames, kConfig); manager.Initialize(); manager.set_stream_analog_level(/*level=*/255); EXPECT_FALSE(manager.clipping_predictor_enabled()); + EXPECT_FALSE(manager.use_clipping_predictor_step()); + EXPECT_EQ(manager.stream_analog_level(), 255); + manager.Process(nullptr); + CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); + CallPreProcessAudioBuffer(/*num_calls=*/300, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); + CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); + EXPECT_EQ(manager.stream_analog_level(), 255); +} + +TEST(AgcManagerDirectStandaloneTest, + EnableClippingPredictorWithUnusedPredictedStepDoesNotLowerVolume) { + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + ClippingPredictorConfig config; + config.enabled = true; + config.use_predicted_step = false; + AgcManagerDirect manager(new ::testing::NiceMock(), kInitialVolume, + kClippedMin, kSampleRateHz, kClippedLevelStep, + kClippedRatioThreshold, kClippedWaitFrames, config); + manager.Initialize(); + manager.set_stream_analog_level(/*level=*/255); + EXPECT_TRUE(manager.clipping_predictor_enabled()); + EXPECT_FALSE(manager.use_clipping_predictor_step()); EXPECT_EQ(manager.stream_analog_level(), 255); manager.Process(nullptr); CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); @@ -945,17 +974,17 @@ TEST(AgcManagerDirectStandaloneTest, } TEST(AgcManagerDirectStandaloneTest, EnableClippingPredictorLowersVolume) { - const ClippingPredictorConfig config( - {/*enabled=*/true, ClippingPredictorConfig::kClippingEventPrediction, - /*window_length=*/5, /*reference_window_length=*/5, - /*reference_window_delay=*/5, /*clipping_threshold=*/-1.0f, - /*crest_factor_margin=*/3.0f}); + // TODO(bugs.webrtc.org/12874): Use designated initializers one fixed. + ClippingPredictorConfig config; + config.enabled = true; + config.use_predicted_step = true; AgcManagerDirect manager(new ::testing::NiceMock(), kInitialVolume, kClippedMin, kSampleRateHz, kClippedLevelStep, kClippedRatioThreshold, kClippedWaitFrames, config); manager.Initialize(); manager.set_stream_analog_level(/*level=*/255); EXPECT_TRUE(manager.clipping_predictor_enabled()); + EXPECT_TRUE(manager.use_clipping_predictor_step()); EXPECT_EQ(manager.stream_analog_level(), 255); manager.Process(nullptr); CallPreProcessAudioBuffer(/*num_calls=*/10, /*peak_ratio=*/0.99f, manager); diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index b03d6d864f..64b1b5d107 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -366,6 +366,10 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { float clipping_threshold = -1.0f; // Crest factor drop threshold (dB). float crest_factor_margin = 3.0f; + // If true, the recommended clipped level step is used to modify the + // analog gain. Otherwise, the predictor runs without affecting the + // analog gain. + bool use_predicted_step = true; } clipping_predictor; } analog_gain_controller; } gain_controller1;