From 73ec01977b31273de89202d61df46982bcb12672 Mon Sep 17 00:00:00 2001 From: Alex Loiko Date: Tue, 15 May 2018 10:52:28 +0200 Subject: [PATCH] Add RuntimeSettings to CustomProcessing. CustomProcessing is the interface to injectable audio processing submodules to AudioProcessing. This CL makes it possible to set runtime settings on the injected render processing component. Note that the current runtime setting handling happens on the capture thread. Therefore, we add another SwapQueue to communicate with the render thread. Bug: webrtc:9138, webrtc:9262 Change-Id: I665ce2d83a2b35ca8b25cca813d2cef7bd0ba911 Reviewed-on: https://webrtc-review.googlesource.com/76123 Commit-Queue: Alex Loiko Reviewed-by: Alessio Bazzica Cr-Commit-Position: refs/heads/master@{#23236} --- .../audio_processing/audio_processing_impl.cc | 53 ++++++++++++++++--- .../audio_processing/audio_processing_impl.h | 26 +++++---- .../audio_processing_unittest.cc | 28 ++++++++-- .../include/audio_processing.cc | 3 ++ .../include/audio_processing.h | 13 ++++- .../include/mock_audio_processing.h | 2 + 6 files changed, 103 insertions(+), 22 deletions(-) diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 28c796497d..e3b385729c 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -66,6 +66,7 @@ namespace webrtc { constexpr int AudioProcessing::kNativeSampleRatesHz[]; +constexpr int kRuntimeSettingQueueSize = 100; namespace { @@ -384,8 +385,10 @@ AudioProcessingImpl::AudioProcessingImpl( NonlinearBeamformer* beamformer) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), - runtime_settings_(100), - runtime_settings_enqueuer_(&runtime_settings_), + capture_runtime_settings_(kRuntimeSettingQueueSize), + render_runtime_settings_(kRuntimeSettingQueueSize), + capture_runtime_settings_enqueuer_(&capture_runtime_settings_), + render_runtime_settings_enqueuer_(&render_runtime_settings_), high_pass_filter_impl_(new HighPassFilterImpl(this)), echo_control_factory_(std::move(echo_control_factory)), submodule_states_(!!capture_post_processor, !!render_pre_processor), @@ -806,8 +809,20 @@ void AudioProcessingImpl::set_output_will_be_muted(bool muted) { } void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) { - RTC_DCHECK(setting.type() != RuntimeSetting::Type::kNotSpecified); - runtime_settings_enqueuer_.Enqueue(setting); + switch (setting.type()) { + case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting: + render_runtime_settings_enqueuer_.Enqueue(setting); + return; + case RuntimeSetting::Type::kNotSpecified: + RTC_NOTREACHED(); + return; + case RuntimeSetting::Type::kCapturePreGain: + capture_runtime_settings_enqueuer_.Enqueue(setting); + return; + } + // The language allows the enum to have a non-enumerator + // value. Check that this doesn't happen. + RTC_NOTREACHED(); } AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer( @@ -913,9 +928,9 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, return kNoError; } -void AudioProcessingImpl::HandleRuntimeSettings() { +void AudioProcessingImpl::HandleCaptureRuntimeSettings() { RuntimeSetting setting; - while (runtime_settings_.Remove(&setting)) { + while (capture_runtime_settings_.Remove(&setting)) { switch (setting.type()) { case RuntimeSetting::Type::kCapturePreGain: if (config_.pre_amplifier.enabled) { @@ -925,6 +940,28 @@ void AudioProcessingImpl::HandleRuntimeSettings() { } // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump. break; + case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting: + RTC_NOTREACHED(); + break; + case RuntimeSetting::Type::kNotSpecified: + RTC_NOTREACHED(); + break; + } + } +} + +void AudioProcessingImpl::HandleRenderRuntimeSettings() { + RuntimeSetting setting; + while (render_runtime_settings_.Remove(&setting)) { + switch (setting.type()) { + case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting: + if (private_submodules_->render_pre_processor) { + private_submodules_->render_pre_processor->SetRuntimeSetting(setting); + } + break; + case RuntimeSetting::Type::kCapturePreGain: + RTC_NOTREACHED(); + break; case RuntimeSetting::Type::kNotSpecified: RTC_NOTREACHED(); break; @@ -1186,7 +1223,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { } int AudioProcessingImpl::ProcessCaptureStreamLocked() { - HandleRuntimeSettings(); + HandleCaptureRuntimeSettings(); // Ensure that not both the AEC and AECM are active at the same time. // TODO(peah): Simplify once the public API Enable functions for these @@ -1509,6 +1546,8 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() { QueueNonbandedRenderAudio(render_buffer); + HandleRenderRuntimeSettings(); + if (private_submodules_->render_pre_processor) { private_submodules_->render_pre_processor->Process(render_buffer); } diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h index a49924ddd9..44b086bfd0 100644 --- a/modules/audio_processing/audio_processing_impl.h +++ b/modules/audio_processing/audio_processing_impl.h @@ -143,13 +143,6 @@ class AudioProcessingImpl : public AudioProcessing { FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior); FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior); FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior); - struct ApmPublicSubmodules; - struct ApmPrivateSubmodules; - - std::unique_ptr data_dumper_; - static int instance_count_; - - SwapQueue runtime_settings_; // Class providing thread-safe message pipe functionality for // |runtime_settings_|. @@ -162,7 +155,18 @@ class AudioProcessingImpl : public AudioProcessing { private: SwapQueue& runtime_settings_; - } runtime_settings_enqueuer_; + }; + struct ApmPublicSubmodules; + struct ApmPrivateSubmodules; + + std::unique_ptr data_dumper_; + static int instance_count_; + + SwapQueue capture_runtime_settings_; + SwapQueue render_runtime_settings_; + + RuntimeSettingEnqueuer capture_runtime_settings_enqueuer_; + RuntimeSettingEnqueuer render_runtime_settings_enqueuer_; // Submodule interface implementations. std::unique_ptr high_pass_filter_impl_; @@ -257,8 +261,10 @@ class AudioProcessingImpl : public AudioProcessing { void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); - // Handle all the runtime settings in the queue. - void HandleRuntimeSettings() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + // Empties and handles the respective RuntimeSetting queues. + void HandleCaptureRuntimeSettings() + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void HandleRenderRuntimeSettings() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); void EmptyQueuedRenderAudio(); void AllocateRenderQueue() diff --git a/modules/audio_processing/audio_processing_unittest.cc b/modules/audio_processing/audio_processing_unittest.cc index fbd81a142e..efbe3c81ce 100644 --- a/modules/audio_processing/audio_processing_unittest.cc +++ b/modules/audio_processing/audio_processing_unittest.cc @@ -2850,7 +2850,6 @@ TEST(RuntimeSettingTest, TestUsageWithSwapQueue) { TEST(ApmConfiguration, EnablePostProcessing) { // Verify that apm uses a capture post processing module if one is provided. - webrtc::Config webrtc_config; auto mock_post_processor_ptr = new testing::NiceMock(); auto mock_post_processor = @@ -2858,7 +2857,7 @@ TEST(ApmConfiguration, EnablePostProcessing) { rtc::scoped_refptr apm = AudioProcessingBuilder() .SetCapturePostProcessing(std::move(mock_post_processor)) - .Create(webrtc_config); + .Create(); AudioFrame audio; audio.num_channels_ = 1; @@ -2870,7 +2869,6 @@ TEST(ApmConfiguration, EnablePostProcessing) { TEST(ApmConfiguration, EnablePreProcessing) { // Verify that apm uses a capture post processing module if one is provided. - webrtc::Config webrtc_config; auto mock_pre_processor_ptr = new testing::NiceMock(); auto mock_pre_processor = @@ -2878,7 +2876,7 @@ TEST(ApmConfiguration, EnablePreProcessing) { rtc::scoped_refptr apm = AudioProcessingBuilder() .SetRenderPreProcessing(std::move(mock_pre_processor)) - .Create(webrtc_config); + .Create(); AudioFrame audio; audio.num_channels_ = 1; @@ -2888,6 +2886,28 @@ TEST(ApmConfiguration, EnablePreProcessing) { apm->ProcessReverseStream(&audio); } +TEST(ApmConfiguration, PreProcessingReceivesRuntimeSettings) { + auto mock_pre_processor_ptr = + new testing::NiceMock(); + auto mock_pre_processor = + std::unique_ptr(mock_pre_processor_ptr); + rtc::scoped_refptr apm = + AudioProcessingBuilder() + .SetRenderPreProcessing(std::move(mock_pre_processor)) + .Create(); + apm->SetRuntimeSetting( + AudioProcessing::RuntimeSetting::CreateCustomRenderSetting(0)); + + // RuntimeSettings forwarded during 'Process*Stream' calls. + // Therefore we have to make one such call. + AudioFrame audio; + audio.num_channels_ = 1; + SetFrameSampleRate(&audio, AudioProcessing::NativeRate::kSampleRate16kHz); + + EXPECT_CALL(*mock_pre_processor_ptr, SetRuntimeSetting(testing::_)).Times(1); + apm->ProcessReverseStream(&audio); +} + class MyEchoControlFactory : public EchoControlFactory { public: std::unique_ptr Create(int sample_rate_hz) { diff --git a/modules/audio_processing/include/audio_processing.cc b/modules/audio_processing/include/audio_processing.cc index e9c56e855f..fa025b9280 100644 --- a/modules/audio_processing/include/audio_processing.cc +++ b/modules/audio_processing/include/audio_processing.cc @@ -33,4 +33,7 @@ Beamforming::Beamforming(bool enabled, Beamforming::~Beamforming() {} +void CustomProcessing::SetRuntimeSetting( + AudioProcessing::RuntimeSetting setting) {} + } // namespace webrtc diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index 379a664da5..6162199f04 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -313,7 +313,11 @@ class AudioProcessing : public rtc::RefCountInterface { // runtime. class RuntimeSetting { public: - enum class Type { kNotSpecified, kCapturePreGain }; + enum class Type { + kNotSpecified, + kCapturePreGain, + kCustomRenderProcessingRuntimeSetting + }; RuntimeSetting() : type_(Type::kNotSpecified), value_(0.f) {} ~RuntimeSetting() = default; @@ -323,6 +327,10 @@ class AudioProcessing : public rtc::RefCountInterface { return {Type::kCapturePreGain, gain}; } + static RuntimeSetting CreateCustomRenderSetting(float payload) { + return {Type::kCustomRenderProcessingRuntimeSetting, payload}; + } + Type type() const { return type_; } void GetFloat(float* value) const { RTC_DCHECK(value); @@ -1119,6 +1127,9 @@ class CustomProcessing { virtual void Process(AudioBuffer* audio) = 0; // Returns a string representation of the module state. virtual std::string ToString() const = 0; + // Handles RuntimeSettings. TODO(webrtc:9262): make pure virtual + // after updating dependencies. + virtual void SetRuntimeSetting(AudioProcessing::RuntimeSetting setting); virtual ~CustomProcessing() {} }; diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h index 5fa3b1b03a..b3253e9380 100644 --- a/modules/audio_processing/include/mock_audio_processing.h +++ b/modules/audio_processing/include/mock_audio_processing.h @@ -110,6 +110,8 @@ class MockCustomProcessing : public CustomProcessing { virtual ~MockCustomProcessing() {} MOCK_METHOD2(Initialize, void(int sample_rate_hz, int num_channels)); MOCK_METHOD1(Process, void(AudioBuffer* audio)); + MOCK_METHOD1(SetRuntimeSetting, + void(AudioProcessing::RuntimeSetting setting)); MOCK_CONST_METHOD0(ToString, std::string()); };