From e36e8bbf6db70fb3805a73b6a3e0ec499a14e784 Mon Sep 17 00:00:00 2001 From: Alex Loiko Date: Fri, 16 Feb 2018 11:54:07 +0100 Subject: [PATCH] Add FixedGainController and move GainController2 in APM. The FixedGainController (FGC) applies a fixed gain. It will also control the limiter. The limiter will be landed over the next several CLs. The GainController2 is a 'private submodule' of APM. It will control the new automatic gain controller (AGC). It controls the AGC through Initialize() and ApplyConfig(). This CL contains * build changes to make modules/audio_processing/agc2 an independent target * a new MutableFloatAudioFrame which is the audio interface between AGC2 and APM * move of the fixed gain application from GainController2 to FixedGainController. If you are a googler, there is more information in this doc: https://docs.google.com/document/d/1RV2Doet3MZtUPAHVva61Vjo20iyd1bmmm3aR8znWpzo/edit# Bug: webrtc:7949 Change-Id: Ief95cbbce83c3aafe54638fd2ab881c9fb8bdc3a Reviewed-on: https://webrtc-review.googlesource.com/50440 Commit-Queue: Alex Loiko Reviewed-by: Oskar Sundbom Cr-Commit-Position: refs/heads/master@{#22046} --- modules/audio_processing/BUILD.gn | 19 ++- .../aec_dump/aec_dump_impl.cc | 9 +- .../audio_processing/aec_dump/aec_dump_impl.h | 7 +- .../aec_dump/capture_stream_info.cc | 4 +- .../aec_dump/capture_stream_info.h | 4 +- .../audio_processing/aec_dump/mock_aec_dump.h | 9 +- modules/audio_processing/agc2/BUILD.gn | 46 ++++++++ modules/audio_processing/agc2/agc2_common.h | 23 ++++ .../agc2/fixed_gain_controller.cc | 90 +++++++++++++++ .../agc2/fixed_gain_controller.h | 37 ++++++ .../agc2/fixed_gain_controller_unittest.cc | 109 ++++++++++++++++++ .../agc2/vector_float_frame.cc | 39 +++++++ .../agc2/vector_float_frame.h | 39 +++++++ .../audio_processing/audio_processing_impl.cc | 10 +- .../{agc2 => }/gain_controller2.cc | 34 +++--- .../{agc2 => }/gain_controller2.h | 13 +-- .../{agc2 => }/gain_controller2_unittest.cc | 9 +- modules/audio_processing/include/aec_dump.h | 41 ++----- .../include/audio_frame_view.h | 54 +++++++++ .../include/audio_processing.h | 1 + 20 files changed, 510 insertions(+), 87 deletions(-) create mode 100644 modules/audio_processing/agc2/BUILD.gn create mode 100644 modules/audio_processing/agc2/agc2_common.h create mode 100644 modules/audio_processing/agc2/fixed_gain_controller.cc create mode 100644 modules/audio_processing/agc2/fixed_gain_controller.h create mode 100644 modules/audio_processing/agc2/fixed_gain_controller_unittest.cc create mode 100644 modules/audio_processing/agc2/vector_float_frame.cc create mode 100644 modules/audio_processing/agc2/vector_float_frame.h rename modules/audio_processing/{agc2 => }/gain_controller2.cc (69%) rename modules/audio_processing/{agc2 => }/gain_controller2.h (80%) rename modules/audio_processing/{agc2 => }/gain_controller2_unittest.cc (92%) create mode 100644 modules/audio_processing/include/audio_frame_view.h diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn index ba0cf9900d..60605e90cc 100644 --- a/modules/audio_processing/BUILD.gn +++ b/modules/audio_processing/BUILD.gn @@ -120,8 +120,6 @@ rtc_static_library("audio_processing") { "agc/loudness_histogram.h", "agc/utility.cc", "agc/utility.h", - "agc2/gain_controller2.cc", - "agc2/gain_controller2.h", "audio_buffer.cc", "audio_buffer.h", "audio_processing_impl.cc", @@ -151,6 +149,8 @@ rtc_static_library("audio_processing") { "gain_control_for_experimental_agc.h", "gain_control_impl.cc", "gain_control_impl.h", + "gain_controller2.cc", + "gain_controller2.h", "include/audio_processing.cc", "include/audio_processing.h", "include/config.cc", @@ -215,6 +215,7 @@ rtc_static_library("audio_processing") { ":aec_core", ":aec_dump_interface", ":apm_logging", + ":audio_frame_view", ":audio_processing_c", ":audio_processing_statistics", "..:module_api", @@ -234,6 +235,7 @@ rtc_static_library("audio_processing") { "../../system_wrappers:cpu_features_api", "../../system_wrappers:field_trial_api", "../../system_wrappers:metrics_api", + "agc2", "vad", ] @@ -287,6 +289,15 @@ rtc_source_set("audio_processing_statistics") { ] } +rtc_source_set("audio_frame_view") { + sources = [ + "include/audio_frame_view.h", + ] + deps = [ + "../../api:array_view", + ] +} + rtc_source_set("aec_dump_interface") { sources = [ "include/aec_dump.cc", @@ -294,6 +305,7 @@ rtc_source_set("aec_dump_interface") { ] deps = [ + ":audio_frame_view", "../../api:array_view", "../../rtc_base:rtc_base_approved", ] @@ -585,6 +597,7 @@ if (rtc_include_tests) { "../../test:test_support", "../audio_coding:neteq_input_audio_tools", "aec_dump:mock_aec_dump_unittests", + "agc2:fixed_digital_unittests", "test/conversational_speech:unittest", "vad:vad_unittests", "//testing/gtest", @@ -653,7 +666,6 @@ if (rtc_include_tests) { "aec3/suppression_filter_unittest.cc", "aec3/suppression_gain_unittest.cc", "aec3/vector_math_unittest.cc", - "agc2/gain_controller2_unittest.cc", "audio_processing_impl_locking_unittest.cc", "audio_processing_impl_unittest.cc", "audio_processing_unittest.cc", @@ -665,6 +677,7 @@ if (rtc_include_tests) { "echo_detector/moving_max_unittest.cc", "echo_detector/normalized_covariance_estimator_unittest.cc", "gain_control_unittest.cc", + "gain_controller2_unittest.cc", "level_controller/level_controller_unittest.cc", "level_estimator_unittest.cc", "low_cut_filter_unittest.cc", diff --git a/modules/audio_processing/aec_dump/aec_dump_impl.cc b/modules/audio_processing/aec_dump/aec_dump_impl.cc index 594bf85cd9..4deb1922e6 100644 --- a/modules/audio_processing/aec_dump/aec_dump_impl.cc +++ b/modules/audio_processing/aec_dump/aec_dump_impl.cc @@ -94,11 +94,13 @@ void AecDumpImpl::WriteInitMessage( worker_queue_->PostTask(std::unique_ptr(std::move(task))); } -void AecDumpImpl::AddCaptureStreamInput(const FloatAudioFrame& src) { +void AecDumpImpl::AddCaptureStreamInput( + const AudioFrameView& src) { capture_stream_info_.AddInput(src); } -void AecDumpImpl::AddCaptureStreamOutput(const FloatAudioFrame& src) { +void AecDumpImpl::AddCaptureStreamOutput( + const AudioFrameView& src) { capture_stream_info_.AddOutput(src); } @@ -134,7 +136,8 @@ void AecDumpImpl::WriteRenderStreamMessage(const AudioFrame& frame) { worker_queue_->PostTask(std::unique_ptr(std::move(task))); } -void AecDumpImpl::WriteRenderStreamMessage(const FloatAudioFrame& src) { +void AecDumpImpl::WriteRenderStreamMessage( + const AudioFrameView& src) { auto task = CreateWriteToFileTask(); auto* event = task->GetEvent(); diff --git a/modules/audio_processing/aec_dump/aec_dump_impl.h b/modules/audio_processing/aec_dump/aec_dump_impl.h index 5be876b6b7..36d72e995c 100644 --- a/modules/audio_processing/aec_dump/aec_dump_impl.h +++ b/modules/audio_processing/aec_dump/aec_dump_impl.h @@ -54,15 +54,16 @@ class AecDumpImpl : public AecDump { void WriteInitMessage(const InternalAPMStreamsConfig& api_format) override; - void AddCaptureStreamInput(const FloatAudioFrame& src) override; - void AddCaptureStreamOutput(const FloatAudioFrame& src) override; + void AddCaptureStreamInput(const AudioFrameView& src) override; + void AddCaptureStreamOutput(const AudioFrameView& src) override; void AddCaptureStreamInput(const AudioFrame& frame) override; void AddCaptureStreamOutput(const AudioFrame& frame) override; void AddAudioProcessingState(const AudioProcessingState& state) override; void WriteCaptureStreamMessage() override; void WriteRenderStreamMessage(const AudioFrame& frame) override; - void WriteRenderStreamMessage(const FloatAudioFrame& src) override; + void WriteRenderStreamMessage( + const AudioFrameView& src) override; void WriteConfig(const InternalAPMConfig& config) override; diff --git a/modules/audio_processing/aec_dump/capture_stream_info.cc b/modules/audio_processing/aec_dump/capture_stream_info.cc index e3284d8822..dd48fd4210 100644 --- a/modules/audio_processing/aec_dump/capture_stream_info.cc +++ b/modules/audio_processing/aec_dump/capture_stream_info.cc @@ -19,7 +19,7 @@ CaptureStreamInfo::CaptureStreamInfo(std::unique_ptr task) CaptureStreamInfo::~CaptureStreamInfo() = default; -void CaptureStreamInfo::AddInput(const FloatAudioFrame& src) { +void CaptureStreamInfo::AddInput(const AudioFrameView& src) { RTC_DCHECK(task_); auto* stream = task_->GetEvent()->mutable_stream(); @@ -30,7 +30,7 @@ void CaptureStreamInfo::AddInput(const FloatAudioFrame& src) { } } -void CaptureStreamInfo::AddOutput(const FloatAudioFrame& src) { +void CaptureStreamInfo::AddOutput(const AudioFrameView& src) { RTC_DCHECK(task_); auto* stream = task_->GetEvent()->mutable_stream(); diff --git a/modules/audio_processing/aec_dump/capture_stream_info.h b/modules/audio_processing/aec_dump/capture_stream_info.h index 9999c3fbd0..91bb1faf5b 100644 --- a/modules/audio_processing/aec_dump/capture_stream_info.h +++ b/modules/audio_processing/aec_dump/capture_stream_info.h @@ -37,8 +37,8 @@ class CaptureStreamInfo { public: explicit CaptureStreamInfo(std::unique_ptr task); ~CaptureStreamInfo(); - void AddInput(const FloatAudioFrame& src); - void AddOutput(const FloatAudioFrame& src); + void AddInput(const AudioFrameView& src); + void AddOutput(const AudioFrameView& src); void AddInput(const AudioFrame& frame); void AddOutput(const AudioFrame& frame); diff --git a/modules/audio_processing/aec_dump/mock_aec_dump.h b/modules/audio_processing/aec_dump/mock_aec_dump.h index 6df6f2849c..8cfabdd648 100644 --- a/modules/audio_processing/aec_dump/mock_aec_dump.h +++ b/modules/audio_processing/aec_dump/mock_aec_dump.h @@ -29,8 +29,10 @@ class MockAecDump : public AecDump { MOCK_METHOD1(WriteInitMessage, void(const InternalAPMStreamsConfig& streams_config)); - MOCK_METHOD1(AddCaptureStreamInput, void(const FloatAudioFrame& src)); - MOCK_METHOD1(AddCaptureStreamOutput, void(const FloatAudioFrame& src)); + MOCK_METHOD1(AddCaptureStreamInput, + void(const AudioFrameView& src)); + MOCK_METHOD1(AddCaptureStreamOutput, + void(const AudioFrameView& src)); MOCK_METHOD1(AddCaptureStreamInput, void(const AudioFrame& frame)); MOCK_METHOD1(AddCaptureStreamOutput, void(const AudioFrame& frame)); MOCK_METHOD1(AddAudioProcessingState, @@ -38,7 +40,8 @@ class MockAecDump : public AecDump { MOCK_METHOD0(WriteCaptureStreamMessage, void()); MOCK_METHOD1(WriteRenderStreamMessage, void(const AudioFrame& frame)); - MOCK_METHOD1(WriteRenderStreamMessage, void(const FloatAudioFrame& src)); + MOCK_METHOD1(WriteRenderStreamMessage, + void(const AudioFrameView& src)); MOCK_METHOD1(WriteConfig, void(const InternalAPMConfig& config)); }; diff --git a/modules/audio_processing/agc2/BUILD.gn b/modules/audio_processing/agc2/BUILD.gn new file mode 100644 index 0000000000..8702e0f460 --- /dev/null +++ b/modules/audio_processing/agc2/BUILD.gn @@ -0,0 +1,46 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_source_set("agc2") { + sources = [ + "agc2_common.h", + "fixed_gain_controller.cc", + "fixed_gain_controller.h", + ] + + configs += [ "..:apm_debug_dump" ] + + deps = [ + "..:apm_logging", + "..:audio_frame_view", + "../../../api:array_view", + "../../../common_audio", + "../../../rtc_base:checks", + "../../../rtc_base:rtc_base_approved", + ] +} + +rtc_source_set("fixed_digital_unittests") { + testonly = true + configs += [ "..:apm_debug_dump" ] + + sources = [ + "fixed_gain_controller_unittest.cc", + "vector_float_frame.cc", + "vector_float_frame.h", + ] + deps = [ + ":agc2", + "..:apm_logging", + "..:audio_frame_view", + "../../../api:array_view", + "../../../rtc_base:rtc_base_tests_utils", + ] +} diff --git a/modules/audio_processing/agc2/agc2_common.h b/modules/audio_processing/agc2/agc2_common.h new file mode 100644 index 0000000000..c668d0a693 --- /dev/null +++ b/modules/audio_processing/agc2/agc2_common.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_AGC2_COMMON_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_AGC2_COMMON_H_ + +namespace webrtc { + +constexpr float kMinSampleValue = -32768.f; +constexpr float kMaxSampleValue = 32767.f; + +// TODO(aleloi): add the other constants as more AGC2 components are +// added. +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_AGC2_COMMON_H_ diff --git a/modules/audio_processing/agc2/fixed_gain_controller.cc b/modules/audio_processing/agc2/fixed_gain_controller.cc new file mode 100644 index 0000000000..a332365748 --- /dev/null +++ b/modules/audio_processing/agc2/fixed_gain_controller.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/fixed_gain_controller.h" + +#include +#include + +#include "api/array_view.h" +#include "common_audio/include/audio_util.h" +#include "modules/audio_processing/agc2/agc2_common.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { +namespace { + +// Returns true when the gain factor is so close to 1 that it would +// not affect int16 samples. +bool CloseToOne(float gain_factor) { + return 1.f - 1.f / kMaxSampleValue <= gain_factor && + gain_factor <= 1.f + 1.f / kMaxSampleValue; +} +} // namespace + +FixedGainController::FixedGainController(ApmDataDumper* apm_data_dumper) + : apm_data_dumper_(apm_data_dumper) { + RTC_DCHECK_LT(0.f, gain_to_apply_); + RTC_DLOG(LS_INFO) << "Gain to apply: " << gain_to_apply_; +} + +void FixedGainController::SetGain(float gain_to_apply_db) { + // Changes in gain_to_apply_ cause discontinuities. We assume + // gain_to_apply_ is set in the beginning of the call. If it is + // frequently changed, we should add interpolation between the + // values. + gain_to_apply_ = DbToRatio(gain_to_apply_db); +} + +void FixedGainController::SetSampleRate(size_t sample_rate_hz) { + // TODO(aleloi): propagate the new sample rate to the GainCurveApplier. +} + +void FixedGainController::EnableLimiter(bool enable_limiter) { + enable_limiter_ = enable_limiter; +} + +void FixedGainController::Process(AudioFrameView signal) { + // Apply fixed digital gain; interpolate if necessary. One of the + // planned usages of the FGC is to only use the limiter. In that + // case, the gain would be 1.0. Not doing the multiplications speeds + // it up considerably. Hence the check. + if (!CloseToOne(gain_to_apply_)) { + for (size_t k = 0; k < signal.num_channels(); ++k) { + rtc::ArrayView channel_view = signal.channel(k); + for (auto& sample : channel_view) { + sample *= gain_to_apply_; + } + } + } + + // Use the limiter (if configured to). + if (enable_limiter_) { + // TODO(aleloi): Process the signal with the + // GainCurveApplier. This will be done in the upcoming CLs. + + // Dump data for debug. + const auto channel_view = signal.channel(0); + apm_data_dumper_->DumpRaw("agc2_fixed_digital_gain_curve_applier", + channel_view.size(), channel_view.data()); + } + + // Hard-clipping. + for (size_t k = 0; k < signal.num_channels(); ++k) { + rtc::ArrayView channel_view = signal.channel(k); + for (auto& sample : channel_view) { + sample = rtc::SafeClamp(sample, kMinSampleValue, kMaxSampleValue); + } + } +} +} // namespace webrtc diff --git a/modules/audio_processing/agc2/fixed_gain_controller.h b/modules/audio_processing/agc2/fixed_gain_controller.h new file mode 100644 index 0000000000..decc22e2b3 --- /dev/null +++ b/modules/audio_processing/agc2/fixed_gain_controller.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_FIXED_GAIN_CONTROLLER_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_FIXED_GAIN_CONTROLLER_H_ + +#include "modules/audio_processing/include/audio_frame_view.h" + +namespace webrtc { +class ApmDataDumper; + +class FixedGainController { + public: + explicit FixedGainController(ApmDataDumper* apm_data_dumper); + + void Process(AudioFrameView signal); + + void SetGain(float gain_to_apply_db); + void SetSampleRate(size_t sample_rate_hz); + void EnableLimiter(bool enable_limiter); + + private: + float gain_to_apply_ = 1.f; + ApmDataDumper* apm_data_dumper_; + bool enable_limiter_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_FIXED_GAIN_CONTROLLER_H_ diff --git a/modules/audio_processing/agc2/fixed_gain_controller_unittest.cc b/modules/audio_processing/agc2/fixed_gain_controller_unittest.cc new file mode 100644 index 0000000000..9336fd768f --- /dev/null +++ b/modules/audio_processing/agc2/fixed_gain_controller_unittest.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/fixed_gain_controller.h" + +#include "api/array_view.h" +#include "modules/audio_processing/agc2/vector_float_frame.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/gunit.h" + +namespace webrtc { +namespace { + +constexpr float kInputLevelLinear = 15000.f; + +constexpr float kGainToApplyDb = 15.f; + +float RunFixedGainControllerWithConstantInput(FixedGainController* fixed_gc, + const float input_level, + const size_t num_frames, + const int sample_rate) { + // Give time to the level etimator to converge. + for (size_t i = 0; i < num_frames; ++i) { + VectorFloatFrame vectors_with_float_frame( + 1, rtc::CheckedDivExact(sample_rate, 100), input_level); + fixed_gc->Process(vectors_with_float_frame.float_frame_view()); + } + + // Process the last frame with constant input level. + VectorFloatFrame vectors_with_float_frame_last( + 1, rtc::CheckedDivExact(sample_rate, 100), input_level); + fixed_gc->Process(vectors_with_float_frame_last.float_frame_view()); + + // Return the last sample from the last processed frame. + const auto channel = + vectors_with_float_frame_last.float_frame_view().channel(0); + return channel[channel.size() - 1]; +} +ApmDataDumper test_data_dumper(0); + +FixedGainController CreateFixedGainController(float gain_to_apply, + size_t rate, + bool enable_limiter) { + FixedGainController fgc(&test_data_dumper); + fgc.SetGain(gain_to_apply); + fgc.SetSampleRate(gain_to_apply); + fgc.EnableLimiter(enable_limiter); + return fgc; +} + +} // namespace + +TEST(AutomaticGainController2FixedDigital, CreateUseWithoutLimiter) { + const int kSampleRate = 48000; + FixedGainController fixed_gc = + CreateFixedGainController(kGainToApplyDb, kSampleRate, false); + VectorFloatFrame vectors_with_float_frame( + 1, rtc::CheckedDivExact(kSampleRate, 100), kInputLevelLinear); + auto float_frame = vectors_with_float_frame.float_frame_view(); + fixed_gc.Process(float_frame); + const auto channel = float_frame.channel(0); + EXPECT_LT(kInputLevelLinear, channel[0]); +} + +TEST(AutomaticGainController2FixedDigital, CreateUseWithLimiter) { + const int kSampleRate = 44000; + FixedGainController fixed_gc = + CreateFixedGainController(kGainToApplyDb, kSampleRate, true); + VectorFloatFrame vectors_with_float_frame( + 1, rtc::CheckedDivExact(kSampleRate, 100), kInputLevelLinear); + auto float_frame = vectors_with_float_frame.float_frame_view(); + fixed_gc.Process(float_frame); + const auto channel = float_frame.channel(0); + EXPECT_LT(kInputLevelLinear, channel[0]); +} + +TEST(AutomaticGainController2FixedDigital, GainShouldChangeOnSetGain) { + constexpr float input_level = 1000.f; + constexpr size_t num_frames = 5; + constexpr size_t kSampleRate = 8000; + constexpr float gain_db_no_change = 0.f; + constexpr float gain_db_factor_10 = 20.f; + + FixedGainController fixed_gc_no_saturation = + CreateFixedGainController(gain_db_no_change, kSampleRate, false); + + // Signal level is unchanged with 0 db gain. + EXPECT_FLOAT_EQ( + RunFixedGainControllerWithConstantInput( + &fixed_gc_no_saturation, input_level, num_frames, kSampleRate), + input_level); + + fixed_gc_no_saturation.SetGain(gain_db_factor_10); + + // +20db should increase signal by a factor of 10. + EXPECT_FLOAT_EQ( + RunFixedGainControllerWithConstantInput( + &fixed_gc_no_saturation, input_level, num_frames, kSampleRate), + input_level * 10); +} + +} // namespace webrtc diff --git a/modules/audio_processing/agc2/vector_float_frame.cc b/modules/audio_processing/agc2/vector_float_frame.cc new file mode 100644 index 0000000000..a70d815196 --- /dev/null +++ b/modules/audio_processing/agc2/vector_float_frame.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/vector_float_frame.h" + +namespace webrtc { + +namespace { + +std::vector ConstructChannelPointers( + std::vector>* x) { + std::vector channel_ptrs; + for (auto& v : *x) { + channel_ptrs.push_back(v.data()); + } + return channel_ptrs; +} +} // namespace + +VectorFloatFrame::VectorFloatFrame(int num_channels, + int samples_per_channel, + float start_value) + : channels_(num_channels, + std::vector(samples_per_channel, start_value)), + channel_ptrs_(ConstructChannelPointers(&channels_)), + float_frame_view_(channel_ptrs_.data(), + channels_.size(), + samples_per_channel) {} + +VectorFloatFrame::~VectorFloatFrame() = default; + +} // namespace webrtc diff --git a/modules/audio_processing/agc2/vector_float_frame.h b/modules/audio_processing/agc2/vector_float_frame.h new file mode 100644 index 0000000000..0e86089713 --- /dev/null +++ b/modules/audio_processing/agc2/vector_float_frame.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_VECTOR_FLOAT_FRAME_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_VECTOR_FLOAT_FRAME_H_ + +#include + +#include "modules/audio_processing/include/audio_frame_view.h" + +namespace webrtc { + +// A construct consisting of a multi-channel audio frame, and a FloatFrame view +// of it. +class VectorFloatFrame { + public: + VectorFloatFrame(int num_channels, + int samples_per_channel, + float start_value); + const AudioFrameView& float_frame_view() { return float_frame_view_; } + + ~VectorFloatFrame(); + + private: + std::vector> channels_; + std::vector channel_ptrs_; + AudioFrameView float_frame_view_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_VECTOR_FLOAT_FRAME_H_ diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 6275726cec..a9e6b059f8 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -20,7 +20,6 @@ #include "common_audio/signal_processing/include/signal_processing_library.h" #include "modules/audio_processing/aec/aec_core.h" #include "modules/audio_processing/agc/agc_manager_direct.h" -#include "modules/audio_processing/agc2/gain_controller2.h" #include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/beamformer/nonlinear_beamformer.h" #include "modules/audio_processing/common.h" @@ -28,6 +27,7 @@ #include "modules/audio_processing/echo_control_mobile_impl.h" #include "modules/audio_processing/gain_control_for_experimental_agc.h" #include "modules/audio_processing/gain_control_impl.h" +#include "modules/audio_processing/gain_controller2.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -1425,7 +1425,7 @@ int AudioProcessingImpl::AnalyzeReverseStreamLocked( const size_t num_channels = formats_.api_format.reverse_input_stream().num_channels(); aec_dump_->WriteRenderStreamMessage( - FloatAudioFrame(src, num_channels, channel_size)); + AudioFrameView(src, num_channels, channel_size)); } render_.render_audio->CopyFrom(src, formats_.api_format.reverse_input_stream()); @@ -1993,7 +1993,7 @@ void AudioProcessingImpl::RecordUnprocessedCaptureStream( const size_t channel_size = formats_.api_format.input_stream().num_frames(); const size_t num_channels = formats_.api_format.input_stream().num_channels(); aec_dump_->AddCaptureStreamInput( - FloatAudioFrame(src, num_channels, channel_size)); + AudioFrameView(src, num_channels, channel_size)); RecordAudioProcessingState(); } @@ -2013,8 +2013,8 @@ void AudioProcessingImpl::RecordProcessedCaptureStream( const size_t channel_size = formats_.api_format.output_stream().num_frames(); const size_t num_channels = formats_.api_format.output_stream().num_channels(); - aec_dump_->AddCaptureStreamOutput( - FloatAudioFrame(processed_capture_stream, num_channels, channel_size)); + aec_dump_->AddCaptureStreamOutput(AudioFrameView( + processed_capture_stream, num_channels, channel_size)); aec_dump_->WriteCaptureStreamMessage(); } diff --git a/modules/audio_processing/agc2/gain_controller2.cc b/modules/audio_processing/gain_controller2.cc similarity index 69% rename from modules/audio_processing/agc2/gain_controller2.cc rename to modules/audio_processing/gain_controller2.cc index b02aa59bc8..aa866c12cc 100644 --- a/modules/audio_processing/agc2/gain_controller2.cc +++ b/modules/audio_processing/gain_controller2.cc @@ -8,16 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/audio_processing/agc2/gain_controller2.h" +#include "modules/audio_processing/gain_controller2.h" -#include - -#include "common_audio/include/audio_util.h" #include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/audio_frame_view.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/atomicops.h" #include "rtc_base/checks.h" -#include "rtc_base/numerics/safe_minmax.h" namespace webrtc { @@ -26,8 +23,7 @@ int GainController2::instance_count_ = 0; GainController2::GainController2() : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), - sample_rate_hz_(AudioProcessing::kSampleRate48kHz), - fixed_gain_(1.f) {} + gain_controller_(data_dumper_.get()) {} GainController2::~GainController2() = default; @@ -36,28 +32,23 @@ void GainController2::Initialize(int sample_rate_hz) { sample_rate_hz == AudioProcessing::kSampleRate16kHz || sample_rate_hz == AudioProcessing::kSampleRate32kHz || sample_rate_hz == AudioProcessing::kSampleRate48kHz); - sample_rate_hz_ = sample_rate_hz; + gain_controller_.SetSampleRate(sample_rate_hz); data_dumper_->InitiateNewSetOfRecordings(); - data_dumper_->DumpRaw("sample_rate_hz", sample_rate_hz_); - data_dumper_->DumpRaw("fixed_gain_linear", fixed_gain_); + data_dumper_->DumpRaw("sample_rate_hz", sample_rate_hz); } void GainController2::Process(AudioBuffer* audio) { - if (fixed_gain_ == 1.f) - return; - - for (size_t k = 0; k < audio->num_channels(); ++k) { - for (size_t j = 0; j < audio->num_frames(); ++j) { - audio->channels_f()[k][j] = rtc::SafeClamp( - fixed_gain_ * audio->channels_f()[k][j], -32768.f, 32767.f); - } - } + AudioFrameView float_frame(audio->channels_f(), audio->num_channels(), + audio->num_frames()); + gain_controller_.Process(float_frame); } void GainController2::ApplyConfig( const AudioProcessing::Config::GainController2& config) { RTC_DCHECK(Validate(config)); - fixed_gain_ = DbToRatio(config.fixed_gain_db); + config_ = config; + gain_controller_.SetGain(config_.fixed_gain_db); + gain_controller_.EnableLimiter(config_.enable_limiter); } bool GainController2::Validate( @@ -69,7 +60,8 @@ std::string GainController2::ToString( const AudioProcessing::Config::GainController2& config) { std::stringstream ss; ss << "{enabled: " << (config.enabled ? "true" : "false") << ", " - << "fixed_gain_dB: " << config.fixed_gain_db << "}"; + << "fixed_gain_dB: " << config.fixed_gain_db << ", " + << "enable_limiter: " << (config.enable_limiter ? "true" : "false") << "}"; return ss.str(); } diff --git a/modules/audio_processing/agc2/gain_controller2.h b/modules/audio_processing/gain_controller2.h similarity index 80% rename from modules/audio_processing/agc2/gain_controller2.h rename to modules/audio_processing/gain_controller2.h index 11706870f4..6de45642b1 100644 --- a/modules/audio_processing/agc2/gain_controller2.h +++ b/modules/audio_processing/gain_controller2.h @@ -8,12 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef MODULES_AUDIO_PROCESSING_AGC2_GAIN_CONTROLLER2_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_GAIN_CONTROLLER2_H_ +#ifndef MODULES_AUDIO_PROCESSING_GAIN_CONTROLLER2_H_ +#define MODULES_AUDIO_PROCESSING_GAIN_CONTROLLER2_H_ #include #include +#include "modules/audio_processing/agc2/fixed_gain_controller.h" #include "modules/audio_processing/include/audio_processing.h" #include "rtc_base/constructormagic.h" @@ -24,8 +25,6 @@ class AudioBuffer; // Gain Controller 2 aims to automatically adjust levels by acting on the // microphone gain and/or applying digital gain. -// -// Temporarily implements a fixed gain mode with hard-clipping. class GainController2 { public: GainController2(); @@ -42,12 +41,12 @@ class GainController2 { private: static int instance_count_; std::unique_ptr data_dumper_; - int sample_rate_hz_; - float fixed_gain_; + FixedGainController gain_controller_; + AudioProcessing::Config::GainController2 config_; RTC_DISALLOW_COPY_AND_ASSIGN(GainController2); }; } // namespace webrtc -#endif // MODULES_AUDIO_PROCESSING_AGC2_GAIN_CONTROLLER2_H_ +#endif // MODULES_AUDIO_PROCESSING_GAIN_CONTROLLER2_H_ diff --git a/modules/audio_processing/agc2/gain_controller2_unittest.cc b/modules/audio_processing/gain_controller2_unittest.cc similarity index 92% rename from modules/audio_processing/agc2/gain_controller2_unittest.cc rename to modules/audio_processing/gain_controller2_unittest.cc index 46f654db62..0c3b38363a 100644 --- a/modules/audio_processing/agc2/gain_controller2_unittest.cc +++ b/modules/audio_processing/gain_controller2_unittest.cc @@ -11,8 +11,8 @@ #include #include "api/array_view.h" -#include "modules/audio_processing/agc2/gain_controller2.h" #include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/gain_controller2.h" #include "rtc_base/checks.h" #include "test/gtest.h" @@ -61,11 +61,12 @@ TEST(GainController2, ToString) { config.fixed_gain_db = 5.f; config.enabled = false; - EXPECT_EQ("{enabled: false, fixed_gain_dB: 5}", + config.enable_limiter = true; + EXPECT_EQ("{enabled: false, fixed_gain_dB: 5, enable_limiter: true}", GainController2::ToString(config)); config.enabled = true; - EXPECT_EQ("{enabled: true, fixed_gain_dB: 5}", + EXPECT_EQ("{enabled: true, fixed_gain_dB: 5, enable_limiter: true}", GainController2::ToString(config)); } @@ -81,7 +82,7 @@ TEST(GainController2, Usage) { AudioProcessing::Config::GainController2 config; // Check that samples are not modified when the fixed gain is 0 dB. - ASSERT_EQ(config.fixed_gain_db, 0.f); + config.fixed_gain_db = 0.f; gain_controller2->ApplyConfig(config); gain_controller2->Process(&ab); EXPECT_EQ(ab.channels_f()[0][0], sample_value); diff --git a/modules/audio_processing/include/aec_dump.h b/modules/audio_processing/include/aec_dump.h index 0c8d2271cc..2035bf4742 100644 --- a/modules/audio_processing/include/aec_dump.h +++ b/modules/audio_processing/include/aec_dump.h @@ -16,6 +16,7 @@ #include #include "api/array_view.h" +#include "modules/audio_processing/include/audio_frame_view.h" namespace webrtc { @@ -65,37 +66,6 @@ struct InternalAPMStreamsConfig { size_t render_output_num_channels = 0; }; -// Class to pass audio data in float** format. This is to avoid -// dependence on AudioBuffer, and avoid problems associated with -// rtc::ArrayView. -class FloatAudioFrame { - public: - // |num_channels| and |channel_size| describe the float** - // |audio_samples|. |audio_samples| is assumed to point to a - // two-dimensional |num_channels * channel_size| array of floats. - FloatAudioFrame(const float* const* audio_samples, - size_t num_channels, - size_t channel_size) - : audio_samples_(audio_samples), - num_channels_(num_channels), - channel_size_(channel_size) {} - - FloatAudioFrame() = delete; - - size_t num_channels() const { return num_channels_; } - - rtc::ArrayView channel(size_t idx) const { - RTC_DCHECK_LE(0, idx); - RTC_DCHECK_LE(idx, num_channels_); - return rtc::ArrayView(audio_samples_[idx], channel_size_); - } - - private: - const float* const* audio_samples_; - size_t num_channels_; - size_t channel_size_; -}; - // An interface for recording configuration and input/output streams // of the Audio Processing Module. The recordings are called // 'aec-dumps' and are stored in a protobuf format defined in @@ -122,8 +92,10 @@ class AecDump { // Logs Event::Type STREAM message. To log an input/output pair, // call the AddCapture* and AddAudioProcessingState methods followed // by a WriteCaptureStreamMessage call. - virtual void AddCaptureStreamInput(const FloatAudioFrame& src) = 0; - virtual void AddCaptureStreamOutput(const FloatAudioFrame& src) = 0; + virtual void AddCaptureStreamInput( + const AudioFrameView& src) = 0; + virtual void AddCaptureStreamOutput( + const AudioFrameView& src) = 0; virtual void AddCaptureStreamInput(const AudioFrame& frame) = 0; virtual void AddCaptureStreamOutput(const AudioFrame& frame) = 0; virtual void AddAudioProcessingState(const AudioProcessingState& state) = 0; @@ -131,7 +103,8 @@ class AecDump { // Logs Event::Type REVERSE_STREAM message. virtual void WriteRenderStreamMessage(const AudioFrame& frame) = 0; - virtual void WriteRenderStreamMessage(const FloatAudioFrame& src) = 0; + virtual void WriteRenderStreamMessage( + const AudioFrameView& src) = 0; // Logs Event::Type CONFIG message. virtual void WriteConfig(const InternalAPMConfig& config) = 0; diff --git a/modules/audio_processing/include/audio_frame_view.h b/modules/audio_processing/include/audio_frame_view.h new file mode 100644 index 0000000000..86e593a9f0 --- /dev/null +++ b/modules/audio_processing/include/audio_frame_view.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_ + +#include "api/array_view.h" + +// Class to pass audio data in T** format, where T is a numeric type. +template +class AudioFrameView { + public: + // |num_channels| and |channel_size| describe the T** + // |audio_samples|. |audio_samples| is assumed to point to a + // two-dimensional |num_channels * channel_size| array of floats. + AudioFrameView(T* const* audio_samples, + size_t num_channels, + size_t channel_size) + : audio_samples_(audio_samples), + num_channels_(num_channels), + channel_size_(channel_size) {} + + AudioFrameView() = delete; + + size_t num_channels() const { return num_channels_; } + + size_t samples_per_channel() const { return channel_size_; } + + rtc::ArrayView channel(size_t idx) { + RTC_DCHECK_LE(0, idx); + RTC_DCHECK_LE(idx, num_channels_); + return rtc::ArrayView(audio_samples_[idx], channel_size_); + } + + rtc::ArrayView channel(size_t idx) const { + RTC_DCHECK_LE(0, idx); + RTC_DCHECK_LE(idx, num_channels_); + return rtc::ArrayView(audio_samples_[idx], channel_size_); + } + + private: + T* const* audio_samples_; + size_t num_channels_; + size_t channel_size_; +}; + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_ diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index 50ec43020d..06c1f4a3ed 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -284,6 +284,7 @@ class AudioProcessing : public rtc::RefCountInterface { struct GainController2 { bool enabled = false; float fixed_gain_db = 0.f; + bool enable_limiter = true; } gain_controller2; // Explicit copy assignment implementation to avoid issues with memory