From c054e78f4e13223766b4bad56c76ac950879c97c Mon Sep 17 00:00:00 2001 From: Alessio Bazzica Date: Mon, 16 Apr 2018 12:10:09 +0200 Subject: [PATCH] Send runtime settings to the Audio Processing Module (APM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL includes the following changes: - APM runtime setting (ID + float payload) and unit tests - Swap queue of APM runtime settings used in AudioProcessingImpl - runtime settings handler that forwards the settings to APM sub-modules when a message is retrieved from the queue - Unit test placeholder to check that the pre-gain update message is correctly delivered Bug: webrtc:9138 Change-Id: Id22704af15fde2b87a4431f5ce64ad1aeafc5280 Reviewed-on: https://webrtc-review.googlesource.com/69320 Reviewed-by: Per Åhgren Reviewed-by: Alex Loiko Commit-Queue: Alessio Bazzica Cr-Commit-Position: refs/heads/master@{#22873} --- .../audio_processing/audio_processing_impl.cc | 46 +++++++++++++++++++ .../audio_processing/audio_processing_impl.h | 20 ++++++++ .../audio_processing_impl_unittest.cc | 6 +++ .../audio_processing_unittest.cc | 29 ++++++++++++ .../include/audio_processing.h | 29 ++++++++++++ .../include/mock_audio_processing.h | 1 + 6 files changed, 131 insertions(+) diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 2e3689561a..f25c430c6a 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -379,6 +379,8 @@ AudioProcessingImpl::AudioProcessingImpl( NonlinearBeamformer* beamformer) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + runtime_settings_(new SwapQueue(100)), + runtime_settings_enqueuer_(runtime_settings_.get()), high_pass_filter_impl_(new HighPassFilterImpl(this)), echo_control_factory_(std::move(echo_control_factory)), submodule_states_(!!capture_post_processor, !!render_pre_processor), @@ -795,6 +797,32 @@ void AudioProcessingImpl::set_output_will_be_muted(bool muted) { } } +void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) { + RTC_DCHECK(setting.type() != RuntimeSetting::Type::kNotSpecified); + runtime_settings_enqueuer_.Enqueue(setting); +} + +AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer( + SwapQueue* runtime_settings) + : runtime_settings_(runtime_settings) { + RTC_DCHECK(runtime_settings_); +} + +AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() = + default; + +void AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue( + RuntimeSetting setting) { + size_t remaining_attempts = 10; + while (!runtime_settings_->Insert(&setting) && remaining_attempts-- > 0) { + RuntimeSetting setting_to_discard; + if (runtime_settings_->Remove(&setting_to_discard)) + RTC_LOG(LS_ERROR) + << "The runtime settings queue is full. Oldest setting discarded."; + } + if (remaining_attempts == 0) + RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting."; +} int AudioProcessingImpl::ProcessStream(const float* const* src, size_t samples_per_channel, @@ -877,6 +905,22 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, return kNoError; } +void AudioProcessingImpl::HandleRuntimeSettings() { + RuntimeSetting setting; + while (runtime_settings_->Remove(&setting)) { + RTC_DCHECK(setting.type() != RuntimeSetting::Type::kNotSpecified); + switch (setting.type()) { + case RuntimeSetting::Type::kCapturePreGain: + // TODO(bugs.chromium.org/9138): Notify + // pre-gain when the sub-module is implemented. + break; + default: + RTC_NOTREACHED(); + break; + } + } +} + void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) { EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), num_reverse_channels(), @@ -1131,6 +1175,8 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { } int AudioProcessingImpl::ProcessCaptureStreamLocked() { + HandleRuntimeSettings(); + // Ensure that not both the AEC and AECM are active at the same time. // TODO(peah): Simplify once the public API Enable functions for these // are moved to APM. diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h index 37240909d2..7f9cbcdd58 100644 --- a/modules/audio_processing/audio_processing_impl.h +++ b/modules/audio_processing/audio_processing_impl.h @@ -66,6 +66,8 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr audio_generator) override; void DetachPlayoutAudioGenerator() override; + void SetRuntimeSetting(RuntimeSetting setting) override; + // Capture-side exclusive methods possibly running APM in a // multi-threaded manner. Acquire the capture lock. int ProcessStream(AudioFrame* frame) override; @@ -149,6 +151,21 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr data_dumper_; static int instance_count_; + std::unique_ptr> runtime_settings_; + + // Class providing thread-safe message pipe functionality for + // |runtime_settings_|. + class RuntimeSettingEnqueuer { + public: + explicit RuntimeSettingEnqueuer( + SwapQueue* runtime_settings); + ~RuntimeSettingEnqueuer(); + void Enqueue(RuntimeSetting setting); + + private: + SwapQueue* runtime_settings_; + } runtime_settings_enqueuer_; + // Submodule interface implementations. std::unique_ptr high_pass_filter_impl_; @@ -239,6 +256,9 @@ class AudioProcessingImpl : public AudioProcessing { void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + // Handle all the runtime settings in the queue. + void HandleRuntimeSettings() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void EmptyQueuedRenderAudio(); void AllocateRenderQueue() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); diff --git a/modules/audio_processing/audio_processing_impl_unittest.cc b/modules/audio_processing/audio_processing_impl_unittest.cc index 08ac6233d5..cd2bba6298 100644 --- a/modules/audio_processing/audio_processing_impl_unittest.cc +++ b/modules/audio_processing/audio_processing_impl_unittest.cc @@ -74,4 +74,10 @@ TEST(AudioProcessingImplTest, AudioParameterChangeTriggersInit) { EXPECT_NOERR(mock.ProcessReverseStream(&frame)); } +TEST(AudioProcessingImplTest, UpdateCapturePreGainRuntimeSetting) { + // TODO(bugs.chromium.org/9138): Implement this test as soon as the pre-gain + // sub-module is implemented and it is notified by HandleRuntimeSettings() + // when the gain changes. +} + } // namespace webrtc diff --git a/modules/audio_processing/audio_processing_unittest.cc b/modules/audio_processing/audio_processing_unittest.cc index 6993da59b1..fbd81a142e 100644 --- a/modules/audio_processing/audio_processing_unittest.cc +++ b/modules/audio_processing/audio_processing_unittest.cc @@ -35,6 +35,7 @@ #include "rtc_base/numerics/safe_minmax.h" #include "rtc_base/protobuf_utils.h" #include "rtc_base/refcountedobject.h" +#include "rtc_base/swap_queue.h" #include "rtc_base/task_queue.h" #include "rtc_base/thread.h" #include "system_wrappers/include/event_wrapper.h" @@ -2819,6 +2820,34 @@ INSTANTIATE_TEST_CASE_P( } // namespace +TEST(RuntimeSettingTest, TestDefaultCtor) { + auto s = AudioProcessing::RuntimeSetting(); + EXPECT_EQ(AudioProcessing::RuntimeSetting::Type::kNotSpecified, s.type()); +} + +TEST(RuntimeSettingTest, TestCapturePreGain) { + using Type = AudioProcessing::RuntimeSetting::Type; + { + auto s = AudioProcessing::RuntimeSetting::CreateCapturePreGain(1.25f); + EXPECT_EQ(Type::kCapturePreGain, s.type()); + float v; + s.GetFloat(&v); + EXPECT_EQ(1.25f, v); + } + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + EXPECT_DEATH(AudioProcessing::RuntimeSetting::CreateCapturePreGain(0.1f), ""); +#endif +} + +TEST(RuntimeSettingTest, TestUsageWithSwapQueue) { + SwapQueue q(1); + auto s = AudioProcessing::RuntimeSetting(); + ASSERT_TRUE(q.Insert(&s)); + ASSERT_TRUE(q.Remove(&s)); + EXPECT_EQ(AudioProcessing::RuntimeSetting::Type::kNotSpecified, s.type()); +} + TEST(ApmConfiguration, EnablePostProcessing) { // Verify that apm uses a capture post processing module if one is provided. webrtc::Config webrtc_config; diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index 05eef5e093..027010b9cd 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -302,6 +302,32 @@ class AudioProcessing : public rtc::RefCountInterface { kStereoAndKeyboard }; + // Specifies the properties of a setting to be passed to AudioProcessing at + // runtime. + class RuntimeSetting { + public: + enum class Type { kNotSpecified, kCapturePreGain }; + + RuntimeSetting() : type_(Type::kNotSpecified), value_(0.f) {} + ~RuntimeSetting() = default; + + static RuntimeSetting CreateCapturePreGain(float gain) { + RTC_DCHECK_GE(gain, 1.f) << "Attenuation is not allowed."; + return {Type::kCapturePreGain, gain}; + } + + Type type() const { return type_; } + void GetFloat(float* value) const { + RTC_DCHECK(value); + *value = value_; + } + + private: + RuntimeSetting(Type id, float value) : type_(id), value_(value) {} + Type type_; + float value_; + }; + ~AudioProcessing() override {} // Initializes internal states, while retaining all user settings. This @@ -359,6 +385,9 @@ class AudioProcessing : public rtc::RefCountInterface { // Default false. virtual void set_output_will_be_muted(bool muted) = 0; + // Enqueue a runtime setting. + virtual void SetRuntimeSetting(RuntimeSetting setting) = 0; + // Processes a 10 ms |frame| of the primary audio stream. On the client-side, // this is the near-end (or captured) audio. // diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h index 96a04ef470..2d49388425 100644 --- a/modules/audio_processing/include/mock_audio_processing.h +++ b/modules/audio_processing/include/mock_audio_processing.h @@ -167,6 +167,7 @@ class MockAudioProcessing : public testing::NiceMock { MOCK_CONST_METHOD0(num_output_channels, size_t()); MOCK_CONST_METHOD0(num_reverse_channels, size_t()); MOCK_METHOD1(set_output_will_be_muted, void(bool muted)); + MOCK_METHOD1(SetRuntimeSetting, void(RuntimeSetting setting)); MOCK_METHOD1(ProcessStream, int(AudioFrame* frame)); MOCK_METHOD7(ProcessStream, int(const float* const* src, size_t samples_per_channel,