From d7cfbe3843a046293850acdc74aaaacb90253548 Mon Sep 17 00:00:00 2001 From: Hanna Silen Date: Wed, 2 Nov 2022 19:12:20 +0100 Subject: [PATCH] Add support for InputVolumeController in GainController2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add InputVolumeController as a member in GainController2 (not created by default). Add a method GainController2::Analyze() to update the applied input volume and run the pre-processing steps in InputVolumeController. Add a call InputVolumeController::Process() in GainController2::Process(). Bug: webrtc:7494 Change-Id: Idf4111ac5e19a620b6421c7f23fd642f169c7b5a Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/279822 Reviewed-by: Per Åhgren Reviewed-by: Alessio Bazzica Commit-Queue: Hanna Silen Cr-Commit-Position: refs/heads/main@{#38548} --- modules/audio_processing/BUILD.gn | 1 + .../agc2/input_volume_controller.h | 2 + modules/audio_processing/gain_controller2.cc | 53 +++++++++++++ modules/audio_processing/gain_controller2.h | 17 +++++ .../gain_controller2_unittest.cc | 75 ++++++++++++++++++- 5 files changed, 146 insertions(+), 2 deletions(-) diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn index 13fb9025d8..8edf6fe3c5 100644 --- a/modules/audio_processing/BUILD.gn +++ b/modules/audio_processing/BUILD.gn @@ -142,6 +142,7 @@ rtc_library("gain_controller2") { "agc2:cpu_features", "agc2:fixed_digital", "agc2:gain_applier", + "agc2:input_volume_controller", "agc2:vad_wrapper", ] } diff --git a/modules/audio_processing/agc2/input_volume_controller.h b/modules/audio_processing/agc2/input_volume_controller.h index 423c4435ff..d77fc8813a 100644 --- a/modules/audio_processing/agc2/input_volume_controller.h +++ b/modules/audio_processing/agc2/input_volume_controller.h @@ -71,6 +71,8 @@ class InputVolumeController final { InputVolumeController(const InputVolumeController&) = delete; InputVolumeController& operator=(const InputVolumeController&) = delete; + // TODO(webrtc:7494): Integrate initialization into ctor and remove this + // method. void Initialize(); // Sets the applied input volume. diff --git a/modules/audio_processing/gain_controller2.cc b/modules/audio_processing/gain_controller2.cc index ecc286e0c0..8b8231e59c 100644 --- a/modules/audio_processing/gain_controller2.cc +++ b/modules/audio_processing/gain_controller2.cc @@ -61,6 +61,17 @@ std::unique_ptr CreateAdaptiveDigitalController( return nullptr; } +// Creates an input volume controller if `enabled` is true. +std::unique_ptr CreateInputVolumeController( + bool enabled, + int num_channels) { + if (enabled) { + return std::make_unique( + num_channels, InputVolumeController::Config{.enabled = enabled}); + } + return nullptr; +} + } // namespace std::atomic GainController2::instance_count_(0); @@ -79,6 +90,9 @@ GainController2::GainController2(const Agc2Config& config, sample_rate_hz, num_channels, &data_dumper_)), + input_volume_controller_( + CreateInputVolumeController(config.input_volume_controller.enabled, + num_channels)), limiter_(sample_rate_hz, &data_dumper_, /*histogram_name_prefix=*/"Agc2"), calls_since_last_limiter_log_(0) { RTC_DCHECK(Validate(config)); @@ -91,10 +105,21 @@ GainController2::GainController2(const Agc2Config& config, config.adaptive_digital.vad_reset_period_ms, cpu_features_, sample_rate_hz); } + if (input_volume_controller_) { + input_volume_controller_->Initialize(); + } } GainController2::~GainController2() = default; +// TODO(webrtc:7494): Pass the flag also to the other components. +void GainController2::SetCaptureOutputUsed(bool capture_output_used) { + if (input_volume_controller_) { + input_volume_controller_->HandleCaptureOutputUsedChange( + capture_output_used); + } +} + void GainController2::SetFixedGainDb(float gain_db) { const float gain_factor = DbToRatio(gain_db); if (fixed_gain_applier_.GetGainFactor() != gain_factor) { @@ -105,6 +130,24 @@ void GainController2::SetFixedGainDb(float gain_db) { fixed_gain_applier_.SetGainFactor(gain_factor); } +void GainController2::Analyze(int applied_input_volume, + const AudioBuffer& audio_buffer) { + RTC_DCHECK_GE(applied_input_volume, 0); + RTC_DCHECK_LE(applied_input_volume, 255); + + if (input_volume_controller_) { + input_volume_controller_->set_stream_analog_level(applied_input_volume); + input_volume_controller_->AnalyzePreProcess(audio_buffer); + } +} + +absl::optional GainController2::GetRecommendedInputVolume() const { + return input_volume_controller_ + ? absl::optional( + input_volume_controller_->recommended_analog_level()) + : absl::nullopt; +} + void GainController2::Process(absl::optional speech_probability, bool input_volume_changed, AudioBuffer* audio) { @@ -125,6 +168,16 @@ void GainController2::Process(absl::optional speech_probability, if (speech_probability.has_value()) { data_dumper_.DumpRaw("agc2_speech_probability", speech_probability.value()); } + + if (input_volume_controller_) { + absl::optional speech_level; + if (adaptive_digital_controller_) { + speech_level = + adaptive_digital_controller_->GetSpeechLevelDbfsIfConfident(); + } + input_volume_controller_->Process(speech_probability, speech_level); + } + fixed_gain_applier_.ApplyGain(float_frame); if (adaptive_digital_controller_) { RTC_DCHECK(speech_probability.has_value()); diff --git a/modules/audio_processing/gain_controller2.h b/modules/audio_processing/gain_controller2.h index 843917a802..3341cd22d0 100644 --- a/modules/audio_processing/gain_controller2.h +++ b/modules/audio_processing/gain_controller2.h @@ -18,6 +18,7 @@ #include "modules/audio_processing/agc2/adaptive_digital_gain_controller.h" #include "modules/audio_processing/agc2/cpu_features.h" #include "modules/audio_processing/agc2/gain_applier.h" +#include "modules/audio_processing/agc2/input_volume_controller.h" #include "modules/audio_processing/agc2/limiter.h" #include "modules/audio_processing/agc2/vad_wrapper.h" #include "modules/audio_processing/include/audio_processing.h" @@ -44,6 +45,17 @@ class GainController2 { // Sets the fixed digital gain. void SetFixedGainDb(float gain_db); + // Updates the input volume controller about whether the capture output is + // used or not. + void SetCaptureOutputUsed(bool capture_output_used); + + // Analyzes `audio_buffer` before `Process()` is called so that the analysis + // can be performed before digital processing operations take place (e.g., + // echo cancellation). The analysis consists of input clipping detection and + // prediction (if enabled). The value of `applied_input_volume` is limited to + // [0, 255]. + void Analyze(int applied_input_volume, const AudioBuffer& audio_buffer); + // Applies fixed and adaptive digital gains to `audio` and runs a limiter. // If the internal VAD is used, `speech_probability` is ignored. Otherwise // `speech_probability` is used for digital adaptive gain if it's available @@ -58,6 +70,10 @@ class GainController2 { AvailableCpuFeatures GetCpuFeatures() const { return cpu_features_; } + // Returns the recommended input volume if input volume controller is enabled + // and if a volume recommendation is available. + absl::optional GetRecommendedInputVolume() const; + private: static std::atomic instance_count_; const AvailableCpuFeatures cpu_features_; @@ -65,6 +81,7 @@ class GainController2 { GainApplier fixed_gain_applier_; std::unique_ptr vad_; std::unique_ptr adaptive_digital_controller_; + std::unique_ptr input_volume_controller_; Limiter limiter_; int calls_since_last_limiter_log_; }; diff --git a/modules/audio_processing/gain_controller2_unittest.cc b/modules/audio_processing/gain_controller2_unittest.cc index 83ea5f1343..eaf0859d3a 100644 --- a/modules/audio_processing/gain_controller2_unittest.cc +++ b/modules/audio_processing/gain_controller2_unittest.cc @@ -22,12 +22,16 @@ #include "modules/audio_processing/test/audio_buffer_tools.h" #include "modules/audio_processing/test/bitexactness_tools.h" #include "rtc_base/checks.h" +#include "test/gmock.h" #include "test/gtest.h" namespace webrtc { namespace test { namespace { +using ::testing::Eq; +using ::testing::Optional; + using Agc2Config = AudioProcessing::Config::GainController2; // Sets all the samples in `ab` to `value`. @@ -40,13 +44,20 @@ void SetAudioBufferSamples(float value, AudioBuffer& ab) { float RunAgc2WithConstantInput(GainController2& agc2, float input_level, int num_frames, - int sample_rate_hz) { + int sample_rate_hz, + int num_channels = 1, + int applied_initial_volume = 0) { const int num_samples = rtc::CheckedDivExact(sample_rate_hz, 100); - AudioBuffer ab(sample_rate_hz, 1, sample_rate_hz, 1, sample_rate_hz, 1); + AudioBuffer ab(sample_rate_hz, num_channels, sample_rate_hz, num_channels, + sample_rate_hz, num_channels); // Give time to the level estimator to converge. for (int i = 0; i < num_frames + 1; ++i) { SetAudioBufferSamples(input_level, ab); + const auto applied_volume = agc2.GetRecommendedInputVolume(); + agc2.Analyze(i > 0 && applied_volume.has_value() ? *applied_volume + : applied_initial_volume, + ab); agc2.Process(/*speech_probability=*/absl::nullopt, /*input_volume_changed=*/false, &ab); } @@ -137,6 +148,66 @@ TEST(GainController2, CheckAdaptiveDigitalMaxOutputNoiseLevelConfig) { EXPECT_TRUE(GainController2::Validate(config)); } +TEST(GainController2, + CheckGetRecommendedInputVolumeWhenInputVolumeControllerNotEnabled) { + constexpr float kHighInputLevel = 32767.0f; + constexpr float kLowInputLevel = 1000.0f; + constexpr int kInitialInputVolume = 100; + constexpr int kNumChannels = 2; + constexpr int kNumFrames = 5; + constexpr int kSampleRateHz = 16000; + + Agc2Config config; + config.input_volume_controller.enabled = false; + auto gain_controller = + std::make_unique(config, kSampleRateHz, kNumChannels, + /*use_internal_vad=*/true); + + EXPECT_FALSE(gain_controller->GetRecommendedInputVolume().has_value()); + + // Run AGC for a signal with no clipping or detected speech. + RunAgc2WithConstantInput(*gain_controller, kLowInputLevel, kNumFrames, + kSampleRateHz, kNumChannels, kInitialInputVolume); + + EXPECT_FALSE(gain_controller->GetRecommendedInputVolume().has_value()); + + // Run AGC for a signal with clipping. + RunAgc2WithConstantInput(*gain_controller, kHighInputLevel, kNumFrames, + kSampleRateHz, kNumChannels, kInitialInputVolume); + + EXPECT_FALSE(gain_controller->GetRecommendedInputVolume().has_value()); +} + +TEST(GainController2, + CheckGetRecommendedInputVolumeWhenInputVolumeControllerEnabled) { + constexpr float kHighInputLevel = 32767.0f; + constexpr float kLowInputLevel = 1000.0f; + constexpr int kInitialInputVolume = 100; + constexpr int kNumChannels = 2; + constexpr int kNumFrames = 5; + constexpr int kSampleRateHz = 16000; + + Agc2Config config; + config.input_volume_controller.enabled = true; + auto gain_controller = + std::make_unique(config, kSampleRateHz, kNumChannels, + /*use_internal_vad=*/true); + + EXPECT_TRUE(gain_controller->GetRecommendedInputVolume().has_value()); + + // Run AGC for a signal with no clipping or detected speech. + RunAgc2WithConstantInput(*gain_controller, kLowInputLevel, kNumFrames, + kSampleRateHz, kNumChannels, kInitialInputVolume); + + EXPECT_TRUE(gain_controller->GetRecommendedInputVolume().has_value()); + + // Run AGC for a signal with clipping. + RunAgc2WithConstantInput(*gain_controller, kHighInputLevel, kNumFrames, + kSampleRateHz, kNumChannels, kInitialInputVolume); + + EXPECT_TRUE(gain_controller->GetRecommendedInputVolume().has_value()); +} + // Checks that the default config is applied. TEST(GainController2, ApplyDefaultConfig) { auto gain_controller2 = std::make_unique(