admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding CustomAudioAnalyzer interface in APM.

Browse Source 
CustomAudioAnalyzer is an interface of a component into APM that
reads AudioBuffer without changing it.
The APM sub-module is optional. It operates in full band.
As described in the comments, it is an experimental interface which
may be changed in the nearest future.

Change-Id: I21edf729d97947529256407b10fa4b5219bb2bf5
Bug: webrtc:9678
Reviewed-on: https://webrtc-review.googlesource.com/96560
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
Commit-Queue: Valeriia Nemychnikova <valeriian@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24481}
This commit is contained in:
Valeriia Nemychnikova 2018-08-29 10:37:09 +02:00 committed by Commit Bot
parent 7015bb410d
commit f06eb57a2f
5 changed files with 97 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
modules/audio_processing/audio_processing_impl.cc
View File

@ -158,9 +158,11 @@ static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
AudioProcessingImpl::ApmSubmoduleStates::ApmSubmoduleStates(
bool capture_post_processor_enabled,
bool render_pre_processor_enabled)
bool render_pre_processor_enabled,
bool capture_analyzer_enabled)
: capture_post_processor_enabled_(capture_post_processor_enabled),
render_pre_processor_enabled_(render_pre_processor_enabled) {}
render_pre_processor_enabled_(render_pre_processor_enabled),
capture_analyzer_enabled_(capture_analyzer_enabled) {}
bool AudioProcessingImpl::ApmSubmoduleStates::Update(
bool low_cut_filter_enabled,
@ -240,6 +242,10 @@ bool AudioProcessingImpl::ApmSubmoduleStates::CaptureFullBandProcessingActive()
pre_amplifier_enabled_;
}
bool AudioProcessingImpl::ApmSubmoduleStates::CaptureAnalyzerActive() const {
return capture_analyzer_enabled_;
}
bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandSubModulesActive()
const {
return RenderMultiBandProcessingActive() || echo_canceller_enabled_ ||
@ -285,10 +291,12 @@ struct AudioProcessingImpl::ApmPublicSubmodules {
struct AudioProcessingImpl::ApmPrivateSubmodules {
ApmPrivateSubmodules(std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
rtc::scoped_refptr<EchoDetector> echo_detector)
rtc::scoped_refptr<EchoDetector> echo_detector,
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
: echo_detector(std::move(echo_detector)),
capture_post_processor(std::move(capture_post_processor)),
render_pre_processor(std::move(render_pre_processor)) {}
render_pre_processor(std::move(render_pre_processor)),
capture_analyzer(std::move(capture_analyzer)) {}
// Accessed internally from capture or during initialization
std::unique_ptr<AgcManagerDirect> agc_manager;
std::unique_ptr<GainController2> gain_controller2;
@ -298,6 +306,7 @@ struct AudioProcessingImpl::ApmPrivateSubmodules {
std::unique_ptr<CustomProcessing> capture_post_processor;
std::unique_ptr<CustomProcessing> render_pre_processor;
std::unique_ptr<GainApplier> pre_amplifier;
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer;
};
AudioProcessingBuilder::AudioProcessingBuilder() = default;
@ -315,6 +324,12 @@ AudioProcessingBuilder& AudioProcessingBuilder::SetRenderPreProcessing(
return *this;
}
AudioProcessingBuilder& AudioProcessingBuilder::SetCaptureAnalyzer(
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer) {
capture_analyzer_ = std::move(capture_analyzer);
return *this;
}
AudioProcessingBuilder& AudioProcessingBuilder::SetEchoControlFactory(
std::unique_ptr<EchoControlFactory> echo_control_factory) {
echo_control_factory_ = std::move(echo_control_factory);
@ -336,7 +351,7 @@ AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) {
AudioProcessingImpl* apm = new rtc::RefCountedObject<AudioProcessingImpl>(
config, std::move(capture_post_processing_),
std::move(render_pre_processing_), std::move(echo_control_factory_),
std::move(echo_detector_));
std::move(echo_detector_), std::move(capture_analyzer_));
if (apm->Initialize() != AudioProcessing::kNoError) {
delete apm;
apm = nullptr;
@ -345,7 +360,8 @@ AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) {
}
AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
: AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr) {}
: AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr, nullptr) {
}
int AudioProcessingImpl::instance_count_ = 0;
@ -354,7 +370,8 @@ AudioProcessingImpl::AudioProcessingImpl(
std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
rtc::scoped_refptr<EchoDetector> echo_detector)
rtc::scoped_refptr<EchoDetector> echo_detector,
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
capture_runtime_settings_(kRuntimeSettingQueueSize),
@ -363,12 +380,15 @@ AudioProcessingImpl::AudioProcessingImpl(
render_runtime_settings_enqueuer_(&render_runtime_settings_),
high_pass_filter_impl_(new HighPassFilterImpl(this)),
echo_control_factory_(std::move(echo_control_factory)),
submodule_states_(!!capture_post_processor, !!render_pre_processor),
submodule_states_(!!capture_post_processor,
!!render_pre_processor,
!!capture_analyzer),
public_submodules_(new ApmPublicSubmodules()),
private_submodules_(
new ApmPrivateSubmodules(std::move(capture_post_processor),
std::move(render_pre_processor),
std::move(echo_detector))),
std::move(echo_detector),
std::move(capture_analyzer))),
constants_(config.Get<ExperimentalAgc>().startup_min_volume,
config.Get<ExperimentalAgc>().clipped_level_min,
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
@ -425,7 +445,9 @@ AudioProcessingImpl::AudioProcessingImpl(
// implemented.
private_submodules_->gain_controller2.reset(new GainController2());
RTC_LOG(LS_INFO) << "Capture post processor activated: "
RTC_LOG(LS_INFO) << "Capture analyzer activated: "
<< !!private_submodules_->capture_analyzer
<< "\nCapture post processor activated: "
<< !!private_submodules_->capture_post_processor
<< "\nRender pre processor activated: "
<< !!private_submodules_->render_pre_processor;
@ -578,6 +600,7 @@ int AudioProcessingImpl::InitializeLocked() {
InitializeResidualEchoDetector();
InitializeEchoController();
InitializeGainController2();
InitializeAnalyzer();
InitializePostProcessor();
InitializePreProcessor();
@ -1350,6 +1373,11 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
capture_.key_pressed);
}
// Experimental APM sub-module that analyzes |capture_buffer|.
if (private_submodules_->capture_analyzer) {
private_submodules_->capture_analyzer->Analyze(capture_buffer);
}
if (config_.gain_controller2.enabled) {
private_submodules_->gain_controller2->NotifyAnalogLevel(
gain_control()->stream_analog_level());
@ -1854,6 +1882,13 @@ void AudioProcessingImpl::InitializeResidualEchoDetector() {
formats_.render_processing_format.sample_rate_hz(), 1);
}
void AudioProcessingImpl::InitializeAnalyzer() {
if (private_submodules_->capture_analyzer) {
private_submodules_->capture_analyzer->Initialize(proc_sample_rate_hz(),
num_proc_channels());
}
}
void AudioProcessingImpl::InitializePostProcessor() {
if (private_submodules_->capture_post_processor) {
private_submodules_->capture_post_processor->Initialize(

9
modules/audio_processing/audio_processing_impl.h
View File

@ -42,7 +42,8 @@ class AudioProcessingImpl : public AudioProcessing {
std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
rtc::scoped_refptr<EchoDetector> echo_detector);
rtc::scoped_refptr<EchoDetector> echo_detector,
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer);
~AudioProcessingImpl() override;
int Initialize() override;
int Initialize(int capture_input_sample_rate_hz,
@ -174,7 +175,8 @@ class AudioProcessingImpl : public AudioProcessing {
class ApmSubmoduleStates {
public:
ApmSubmoduleStates(bool capture_post_processor_enabled,
bool render_pre_processor_enabled);
bool render_pre_processor_enabled,
bool capture_analyzer_enabled);
// Updates the submodule state and returns true if it has changed.
bool Update(bool low_cut_filter_enabled,
bool echo_canceller_enabled,
@ -192,6 +194,7 @@ class AudioProcessingImpl : public AudioProcessing {
bool CaptureMultiBandSubModulesActive() const;
bool CaptureMultiBandProcessingActive() const;
bool CaptureFullBandProcessingActive() const;
bool CaptureAnalyzerActive() const;
bool RenderMultiBandSubModulesActive() const;
bool RenderFullBandProcessingActive() const;
bool RenderMultiBandProcessingActive() const;
@ -199,6 +202,7 @@ class AudioProcessingImpl : public AudioProcessing {
private:
const bool capture_post_processor_enabled_ = false;
const bool render_pre_processor_enabled_ = false;
const bool capture_analyzer_enabled_ = false;
bool low_cut_filter_enabled_ = false;
bool echo_canceller_enabled_ = false;
bool mobile_echo_controller_enabled_ = false;
@ -252,6 +256,7 @@ class AudioProcessingImpl : public AudioProcessing {
void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializeAnalyzer() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
// Empties and handles the respective RuntimeSetting queues.

19
modules/audio_processing/audio_processing_unittest.cc
View File

@ -2802,6 +2802,25 @@ TEST(ApmConfiguration, EnablePreProcessing) {
apm->ProcessReverseStream(&audio);
}
TEST(ApmConfiguration, EnableCaptureAnalyzer) {
// Verify that apm uses a capture analyzer if one is provided.
auto mock_capture_analyzer_ptr =
new testing::NiceMock<test::MockCustomAudioAnalyzer>();
auto mock_capture_analyzer =
std::unique_ptr<CustomAudioAnalyzer>(mock_capture_analyzer_ptr);
rtc::scoped_refptr<AudioProcessing> apm =
AudioProcessingBuilder()
.SetCaptureAnalyzer(std::move(mock_capture_analyzer))
.Create();
AudioFrame audio;
audio.num_channels_ = 1;
SetFrameSampleRate(&audio, AudioProcessing::NativeRate::kSampleRate16kHz);
EXPECT_CALL(*mock_capture_analyzer_ptr, Analyze(testing::_)).Times(1);
apm->ProcessStream(&audio);
}
TEST(ApmConfiguration, PreProcessingReceivesRuntimeSettings) {
auto mock_pre_processor_ptr =
new testing::NiceMock<test::MockCustomProcessing>();

18
modules/audio_processing/include/audio_processing.h
View File

@ -53,6 +53,7 @@ class GainControl;
class HighPassFilter;
class LevelEstimator;
class NoiseSuppression;
class CustomAudioAnalyzer;
class CustomProcessing;
class VoiceDetection;
@ -675,6 +676,9 @@ class AudioProcessingBuilder {
// The AudioProcessingBuilder takes ownership of the echo_detector.
AudioProcessingBuilder& SetEchoDetector(
rtc::scoped_refptr<EchoDetector> echo_detector);
// The AudioProcessingBuilder takes ownership of the capture_analyzer.
AudioProcessingBuilder& SetCaptureAnalyzer(
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer);
// This creates an APM instance using the previously set components. Calling
// the Create function resets the AudioProcessingBuilder to its initial state.
AudioProcessing* Create();
@ -685,6 +689,7 @@ class AudioProcessingBuilder {
std::unique_ptr<CustomProcessing> capture_post_processing_;
std::unique_ptr<CustomProcessing> render_pre_processing_;
rtc::scoped_refptr<EchoDetector> echo_detector_;
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer_;
RTC_DISALLOW_COPY_AND_ASSIGN(AudioProcessingBuilder);
};
@ -1018,6 +1023,19 @@ class NoiseSuppression {
virtual ~NoiseSuppression() {}
};
// Experimental interface for a custom analysis submodule.
class CustomAudioAnalyzer {
public:
// (Re-) Initializes the submodule.
virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
// Analyzes the given capture or render signal.
virtual void Analyze(const AudioBuffer* audio) = 0;
// Returns a string representation of the module state.
virtual std::string ToString() const = 0;
virtual ~CustomAudioAnalyzer() {}
};
// Interface for a custom processing submodule.
class CustomProcessing {
public:

8
modules/audio_processing/include/mock_audio_processing.h
View File

@ -115,6 +115,14 @@ class MockCustomProcessing : public CustomProcessing {
MOCK_CONST_METHOD0(ToString, std::string());
};
class MockCustomAudioAnalyzer : public CustomAudioAnalyzer {
public:
virtual ~MockCustomAudioAnalyzer() {}
MOCK_METHOD2(Initialize, void(int sample_rate_hz, int num_channels));
MOCK_METHOD1(Analyze, void(const AudioBuffer* audio));
MOCK_CONST_METHOD0(ToString, std::string());
};
class MockEchoControl : public EchoControl {
public:
virtual ~MockEchoControl() {}