From 4d3644979cade2143fe3800a6cca5587c98a4640 Mon Sep 17 00:00:00 2001 From: Sam Zackrisson Date: Fri, 2 Mar 2018 16:03:21 +0100 Subject: [PATCH] Add stub draft of audio generator to APM This provides the empty shell of an AudioGenerator class. It is intended to be used for debugging purposes and can be inserted into the APM much like an AecDump. It allows for playing out diagnostic audio unaffected by codecs and network jitter, while still capturing API interaction like in a normal call. NOTRY=True Bug: webrtc:8882 Change-Id: I8132afc95cdba02ab233f44e22e0a5f530710ef7 Reviewed-on: https://webrtc-review.googlesource.com/53300 Commit-Queue: Sam Zackrisson Reviewed-by: Alex Loiko Cr-Commit-Position: refs/heads/master@{#22282} --- modules/audio_processing/BUILD.gn | 58 +++++++++++++++++++ .../audio_generator/file_audio_generator.cc | 36 ++++++++++++ .../audio_generator/file_audio_generator.h | 48 +++++++++++++++ .../file_audio_generator_unittest.cc | 31 ++++++++++ .../audio_processing/audio_processing_impl.cc | 11 ++++ .../audio_processing/audio_processing_impl.h | 3 + .../include/audio_generator.h | 36 ++++++++++++ .../include/audio_generator_factory.cc | 25 ++++++++ .../include/audio_generator_factory.h | 31 ++++++++++ .../include/audio_processing.h | 11 ++++ .../include/mock_audio_processing.h | 4 ++ 11 files changed, 294 insertions(+) create mode 100644 modules/audio_processing/audio_generator/file_audio_generator.cc create mode 100644 modules/audio_processing/audio_generator/file_audio_generator.h create mode 100644 modules/audio_processing/audio_generator/file_audio_generator_unittest.cc create mode 100644 modules/audio_processing/include/audio_generator.h create mode 100644 modules/audio_processing/include/audio_generator_factory.cc create mode 100644 modules/audio_processing/include/audio_generator_factory.h diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn index 68f0971177..3dcea89d90 100644 --- a/modules/audio_processing/BUILD.gn +++ b/modules/audio_processing/BUILD.gn @@ -140,6 +140,7 @@ rtc_static_library("audio_processing") { ":aec_dump_interface", ":apm_logging", ":audio_frame_view", + ":audio_generator_interface", ":audio_processing_c", ":audio_processing_statistics", "..:module_api", @@ -235,6 +236,46 @@ rtc_source_set("aec_dump_interface") { ] } +rtc_source_set("audio_generator_interface") { + visibility = [ "*" ] + sources = [ + "include/audio_generator.h", + ] + deps = [ + ":audio_frame_view", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers", + ] +} + +rtc_source_set("audio_generator_factory") { + visibility = [ "*" ] + sources = [ + "include/audio_generator_factory.cc", + "include/audio_generator_factory.h", + ] + deps = [ + ":audio_generator_interface", + ":file_audio_generator", + "../../common_audio:common_audio", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers", + ] +} + +rtc_source_set("file_audio_generator") { + sources = [ + "audio_generator/file_audio_generator.cc", + "audio_generator/file_audio_generator.h", + ] + deps = [ + ":audio_generator_interface", + "../../common_audio:common_audio", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers:system_wrappers", + ] +} + rtc_source_set("audio_processing_c") { visibility = [ ":*" ] # Only targets in this file can depend on this. sources = [ @@ -501,6 +542,7 @@ if (rtc_include_tests) { ":audio_frame_view", ":audio_processing", ":audioproc_test_utils", + ":file_audio_generator_unittests", ":mocks", "..:module_api", "../..:typedefs", @@ -617,6 +659,22 @@ if (rtc_include_tests) { } } + rtc_source_set("file_audio_generator_unittests") { + testonly = true + + sources = [ + "audio_generator/file_audio_generator_unittest.cc", + ] + + deps = [ + ":audio_generator_factory", + ":audio_processing", + ":file_audio_generator", + "../../rtc_base:rtc_base_approved", + "../../test:test_support", + ] + } + rtc_source_set("analog_mic_simulation") { sources = [ "test/fake_recording_device.cc", diff --git a/modules/audio_processing/audio_generator/file_audio_generator.cc b/modules/audio_processing/audio_generator/file_audio_generator.cc new file mode 100644 index 0000000000..0d691b8bbf --- /dev/null +++ b/modules/audio_processing/audio_generator/file_audio_generator.cc @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/audio_generator/file_audio_generator.h" + +namespace webrtc { + +FileAudioGenerator::FileAudioGenerator( + std::unique_ptr input_audio_file) { + // TODO(bugs.webrtc.org/8882) Stub. + // Read audio from file into internal buffer. +} + +FileAudioGenerator::~FileAudioGenerator() = default; + +void FileAudioGenerator::FillFrame(AudioFrameView audio) { + // TODO(bugs.webrtc.org/8882) Stub. + // Fill |audio| from internal buffer. +} + +size_t FileAudioGenerator::NumChannels() { + return num_channels_; +} + +size_t FileAudioGenerator::SampleRateHz() { + return sample_rate_hz_; +} + +} // namespace webrtc diff --git a/modules/audio_processing/audio_generator/file_audio_generator.h b/modules/audio_processing/audio_generator/file_audio_generator.h new file mode 100644 index 0000000000..01979a42b3 --- /dev/null +++ b/modules/audio_processing/audio_generator/file_audio_generator.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AUDIO_GENERATOR_FILE_AUDIO_GENERATOR_H_ +#define MODULES_AUDIO_PROCESSING_AUDIO_GENERATOR_FILE_AUDIO_GENERATOR_H_ + +#include + +#include "common_audio/wav_file.h" +#include "modules/audio_processing/include/audio_generator.h" +#include "rtc_base/constructormagic.h" + +namespace webrtc { + +// Provides looping audio from a file. The file is read in its entirety on +// construction and then closed. This class wraps a webrtc::WavReader, and is +// hence unsuitable for non-diagnostic code. +class FileAudioGenerator : public AudioGenerator { + public: + // Reads the playout audio from a given WAV file. + explicit FileAudioGenerator(std::unique_ptr input_audio_file); + + ~FileAudioGenerator() override; + + // Fill |audio| with audio from a file. + void FillFrame(AudioFrameView audio) override; + + size_t NumChannels() override; + + size_t SampleRateHz() override; + + private: + size_t num_channels_; + size_t sample_rate_hz_; + + RTC_DISALLOW_COPY_AND_ASSIGN(FileAudioGenerator); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AUDIO_GENERATOR_FILE_AUDIO_GENERATOR_H_ diff --git a/modules/audio_processing/audio_generator/file_audio_generator_unittest.cc b/modules/audio_processing/audio_generator/file_audio_generator_unittest.cc new file mode 100644 index 0000000000..6ed3b94667 --- /dev/null +++ b/modules/audio_processing/audio_generator/file_audio_generator_unittest.cc @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/audio_processing/include/audio_generator_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +// TODO(bugs.webrtc.org/8882) Stub. +// Add unit tests for both file audio and generated audio. + +TEST(FileAudioGeneratorTest, CreationDeletion) { + const std::string audio_filename = + test::ResourcePath("voice_engine/audio_tiny48", "wav"); + auto audio_generator = AudioGeneratorFactory::Create(audio_filename); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index a9e6b059f8..f4b8dee221 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -1588,6 +1588,17 @@ void AudioProcessingImpl::DetachAecDump() { } } +void AudioProcessingImpl::AttachPlayoutAudioGenerator( + std::unique_ptr audio_generator) { + // TODO(bugs.webrtc.org/8882) Stub. + // Reset internal audio generator with audio_generator. +} + +void AudioProcessingImpl::DetachPlayoutAudioGenerator() { + // TODO(bugs.webrtc.org/8882) Stub. + // Delete audio generator, if one is attached. +} + AudioProcessing::AudioProcessingStatistics::AudioProcessingStatistics() { residual_echo_return_loss.Set(-100.0f, -100.0f, -100.0f, -100.0f); echo_return_loss.Set(-100.0f, -100.0f, -100.0f, -100.0f); diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h index d0fa8cc240..e7c6621ae6 100644 --- a/modules/audio_processing/audio_processing_impl.h +++ b/modules/audio_processing/audio_processing_impl.h @@ -62,6 +62,9 @@ class AudioProcessingImpl : public AudioProcessing { void UpdateHistogramsOnCallEnd() override; void AttachAecDump(std::unique_ptr aec_dump) override; void DetachAecDump() override; + void AttachPlayoutAudioGenerator( + std::unique_ptr audio_generator) override; + void DetachPlayoutAudioGenerator() override; // Capture-side exclusive methods possibly running APM in a // multi-threaded manner. Acquire the capture lock. diff --git a/modules/audio_processing/include/audio_generator.h b/modules/audio_processing/include/audio_generator.h new file mode 100644 index 0000000000..77e6284969 --- /dev/null +++ b/modules/audio_processing/include/audio_generator.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_GENERATOR_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_GENERATOR_H_ + +#include "modules/audio_processing/include/audio_frame_view.h" + +namespace webrtc { +// This class is used as input sink for the APM, for diagnostic purposes. +// Generates an infinite audio signal, [-1, 1] floating point values, in frames +// of fixed channel count and sample rate. +class AudioGenerator { + public: + virtual ~AudioGenerator() {} + + // Fill |audio| with the next samples of the audio signal. + virtual void FillFrame(AudioFrameView audio) = 0; + + // Return the number of channels output by the AudioGenerator. + virtual size_t NumChannels() = 0; + + // Return the sample rate output by the AudioGenerator. + virtual size_t SampleRateHz() = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_GENERATOR_H_ diff --git a/modules/audio_processing/include/audio_generator_factory.cc b/modules/audio_processing/include/audio_generator_factory.cc new file mode 100644 index 0000000000..9084a1e16f --- /dev/null +++ b/modules/audio_processing/include/audio_generator_factory.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/include/audio_generator_factory.h" + +#include "common_audio/wav_file.h" +#include "modules/audio_processing/audio_generator/file_audio_generator.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { + +std::unique_ptr AudioGeneratorFactory::Create( + const std::string& file_name) { + std::unique_ptr input_audio_file(new WavReader(file_name)); + return rtc::MakeUnique(std::move(input_audio_file)); +} + +} // namespace webrtc diff --git a/modules/audio_processing/include/audio_generator_factory.h b/modules/audio_processing/include/audio_generator_factory.h new file mode 100644 index 0000000000..a9513ef88d --- /dev/null +++ b/modules/audio_processing/include/audio_generator_factory.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_GENERATOR_FACTORY_H_ +#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_GENERATOR_FACTORY_H_ + +#include +#include +#include + +#include "modules/audio_processing/include/audio_generator.h" + +namespace webrtc { + +class AudioGeneratorFactory { + public: + // Creates an AudioGenerator that reads the playout audio from a given 16-bit + // int-encoded WAV file. + static std::unique_ptr Create(const std::string& file_name); +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_GENERATOR_FACTORY_H_ diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index ddbaede13a..7057f2804f 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -26,6 +26,7 @@ #include "api/audio/echo_control.h" #include "api/optional.h" #include "modules/audio_processing/beamformer/array_util.h" +#include "modules/audio_processing/include/audio_generator.h" #include "modules/audio_processing/include/audio_processing_statistics.h" #include "modules/audio_processing/include/config.h" #include "rtc_base/arraysize.h" @@ -480,6 +481,16 @@ class AudioProcessing : public rtc::RefCountInterface { // all pending logging tasks are completed. virtual void DetachAecDump() = 0; + // Attaches provided webrtc::AudioGenerator for modifying playout audio. + // Calling this method when another AudioGenerator is attached replaces the + // active AudioGenerator with a new one. + virtual void AttachPlayoutAudioGenerator( + std::unique_ptr audio_generator) = 0; + + // If no AudioGenerator is attached, this has no effect. If an AecDump is + // attached, its destructor is called. + virtual void DetachPlayoutAudioGenerator() = 0; + // Use to send UMA histograms at end of a call. Note that all histogram // specific member variables are reset. virtual void UpdateHistogramsOnCallEnd() = 0; diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h index e973d19264..96a04ef470 100644 --- a/modules/audio_processing/include/mock_audio_processing.h +++ b/modules/audio_processing/include/mock_audio_processing.h @@ -198,6 +198,10 @@ class MockAudioProcessing : public testing::NiceMock { virtual void AttachAecDump(std::unique_ptr aec_dump) {} MOCK_METHOD0(DetachAecDump, void()); + virtual void AttachPlayoutAudioGenerator( + std::unique_ptr audio_generator) {} + MOCK_METHOD0(DetachPlayoutAudioGenerator, void()); + MOCK_METHOD0(UpdateHistogramsOnCallEnd, void()); MOCK_CONST_METHOD0(GetStatistics, AudioProcessingStatistics()); MOCK_CONST_METHOD1(GetStatistics, AudioProcessingStats(bool));