From dd31071e19bdfe0bedc335f070e1592cfd5d2eb6 Mon Sep 17 00:00:00 2001 From: aleloi Date: Thu, 17 Nov 2016 06:28:59 -0800 Subject: [PATCH] Added an empty AudioTransportProxy to AudioState. All audio in calls is now routed through AudioTransportProxy. The AudioTransport implemented by VoEBaseImpl is disconnected from AudioDevice and replaced by an empty proxy layer that forwards calls to the old Transport. This is a refactoring CL in preparation for landing https://codereview.webrtc.org/2436033002/, which will connect the new AudioMixer. In the planned configuration, the currently empty AudioTransportProxy will query the new mixer for audio instead of polling data from the old Transport. Mixed audio will be passed to an AudioProcessing interface. AudioTransportProxy is initialized with an AudioProcessing*, which is currently unused. No presubmit since we implement an interface with non-const references. NOPRESUBMIT=True BUG=webrtc:6346 Review-Url: https://codereview.webrtc.org/2454373002 Cr-Commit-Position: refs/heads/master@{#15133} --- webrtc/audio/BUILD.gn | 6 ++ webrtc/audio/DEPS | 4 +- webrtc/audio/audio_receive_stream_unittest.cc | 4 + webrtc/audio/audio_send_stream_unittest.cc | 4 + webrtc/audio/audio_state.cc | 18 +++- webrtc/audio/audio_state.h | 7 ++ webrtc/audio/audio_state_unittest.cc | 39 ++++++++ webrtc/audio/audio_transport_proxy.cc | 97 +++++++++++++++++++ webrtc/audio/audio_transport_proxy.h | 72 ++++++++++++++ webrtc/call/BUILD.gn | 6 +- webrtc/call/DEPS | 2 + webrtc/call/call_unittest.cc | 3 + webrtc/test/BUILD.gn | 2 + webrtc/test/DEPS | 1 + webrtc/test/mock_voice_engine.h | 15 +++ 15 files changed, 276 insertions(+), 4 deletions(-) create mode 100644 webrtc/audio/audio_transport_proxy.cc create mode 100644 webrtc/audio/audio_transport_proxy.h diff --git a/webrtc/audio/BUILD.gn b/webrtc/audio/BUILD.gn index 127900544c..e1d583f8e5 100644 --- a/webrtc/audio/BUILD.gn +++ b/webrtc/audio/BUILD.gn @@ -16,6 +16,8 @@ rtc_static_library("audio") { "audio_send_stream.h", "audio_state.cc", "audio_state.h", + "audio_transport_proxy.cc", + "audio_transport_proxy.h", "conversion.h", "scoped_voe_interface.h", ] @@ -29,6 +31,9 @@ rtc_static_library("audio") { "..:webrtc_common", "../api:audio_mixer_api", "../api:call_api", + "../base:rtc_base_approved", + "../modules/audio_device", + "../modules/audio_processing", "../system_wrappers", "../voice_engine", ] @@ -43,6 +48,7 @@ if (rtc_include_tests) { ] deps = [ ":audio", + "../modules/audio_device:mock_audio_device", "//testing/gmock", "//testing/gtest", ] diff --git a/webrtc/audio/DEPS b/webrtc/audio/DEPS index 476d463e6a..e53d28578e 100644 --- a/webrtc/audio/DEPS +++ b/webrtc/audio/DEPS @@ -1,9 +1,9 @@ include_rules = [ "+webrtc/base", - "+webrtc/voice_engine", - "+webrtc/modules/audio_coding/codecs/mock", "+webrtc/call", "+webrtc/logging/rtc_event_log", + "+webrtc/modules/audio_coding/codecs/mock", + "+webrtc/modules/audio_device", "+webrtc/modules/audio_processing/include", "+webrtc/modules/bitrate_controller", "+webrtc/modules/congestion_controller", diff --git a/webrtc/audio/audio_receive_stream_unittest.cc b/webrtc/audio/audio_receive_stream_unittest.cc index 0331d9b7f6..0fc93b8b34 100644 --- a/webrtc/audio/audio_receive_stream_unittest.cc +++ b/webrtc/audio/audio_receive_stream_unittest.cc @@ -78,6 +78,10 @@ struct ConfigHelper { RegisterVoiceEngineObserver(_)).WillOnce(Return(0)); EXPECT_CALL(voice_engine_, DeRegisterVoiceEngineObserver()).WillOnce(Return(0)); + EXPECT_CALL(voice_engine_, audio_processing()); + EXPECT_CALL(voice_engine_, audio_device_module()); + EXPECT_CALL(voice_engine_, audio_transport()); + AudioState::Config config; config.voice_engine = &voice_engine_; audio_state_ = AudioState::Create(config); diff --git a/webrtc/audio/audio_send_stream_unittest.cc b/webrtc/audio/audio_send_stream_unittest.cc index 427cda31a0..7e0ea45bb2 100644 --- a/webrtc/audio/audio_send_stream_unittest.cc +++ b/webrtc/audio/audio_send_stream_unittest.cc @@ -75,6 +75,10 @@ struct ConfigHelper { RegisterVoiceEngineObserver(_)).WillOnce(Return(0)); EXPECT_CALL(voice_engine_, DeRegisterVoiceEngineObserver()).WillOnce(Return(0)); + EXPECT_CALL(voice_engine_, audio_device_module()); + EXPECT_CALL(voice_engine_, audio_processing()); + EXPECT_CALL(voice_engine_, audio_transport()); + AudioState::Config config; config.voice_engine = &voice_engine_; audio_state_ = AudioState::Create(config); diff --git a/webrtc/audio/audio_state.cc b/webrtc/audio/audio_state.cc index e63f97af2d..95a90a5bf1 100644 --- a/webrtc/audio/audio_state.cc +++ b/webrtc/audio/audio_state.cc @@ -13,16 +13,28 @@ #include "webrtc/base/atomicops.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/modules/audio_device/include/audio_device.h" #include "webrtc/voice_engine/include/voe_errors.h" namespace webrtc { namespace internal { AudioState::AudioState(const AudioState::Config& config) - : config_(config), voe_base_(config.voice_engine) { + : config_(config), + voe_base_(config.voice_engine), + audio_transport_proxy_(voe_base_->audio_transport(), + voe_base_->audio_processing(), + config_.audio_mixer) { process_thread_checker_.DetachFromThread(); // Only one AudioState should be created per VoiceEngine. RTC_CHECK(voe_base_->RegisterVoiceEngineObserver(*this) != -1); + + auto* const device = voe_base_->audio_device_module(); + RTC_DCHECK(device); + + // This is needed for the Chrome implementation of RegisterAudioCallback. + device->RegisterAudioCallback(nullptr); + device->RegisterAudioCallback(&audio_transport_proxy_); } AudioState::~AudioState() { @@ -35,6 +47,10 @@ VoiceEngine* AudioState::voice_engine() { return config_.voice_engine; } +rtc::scoped_refptr AudioState::mixer() { + return config_.audio_mixer; +} + bool AudioState::typing_noise_detected() const { RTC_DCHECK(thread_checker_.CalledOnValidThread()); rtc::CritScope lock(&crit_sect_); diff --git a/webrtc/audio/audio_state.h b/webrtc/audio/audio_state.h index 31892d04fa..307f5cec4b 100644 --- a/webrtc/audio/audio_state.h +++ b/webrtc/audio/audio_state.h @@ -12,6 +12,7 @@ #define WEBRTC_AUDIO_AUDIO_STATE_H_ #include "webrtc/api/call/audio_state.h" +#include "webrtc/audio/audio_transport_proxy.h" #include "webrtc/audio/scoped_voe_interface.h" #include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" @@ -28,6 +29,8 @@ class AudioState final : public webrtc::AudioState, ~AudioState() override; VoiceEngine* voice_engine(); + + rtc::scoped_refptr mixer(); bool typing_noise_detected() const; private: @@ -53,6 +56,10 @@ class AudioState final : public webrtc::AudioState, // Reference count; implementation copied from rtc::RefCountedObject. mutable volatile int ref_count_ = 0; + // Transports mixed audio from the mixer to the audio device and + // recorded audio to the VoE AudioTransport. + AudioTransportProxy audio_transport_proxy_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioState); }; } // namespace internal diff --git a/webrtc/audio/audio_state_unittest.cc b/webrtc/audio/audio_state_unittest.cc index 86a9176db3..bd39baac1b 100644 --- a/webrtc/audio/audio_state_unittest.cc +++ b/webrtc/audio/audio_state_unittest.cc @@ -24,6 +24,10 @@ struct ConfigHelper { RegisterVoiceEngineObserver(testing::_)).WillOnce(testing::Return(0)); EXPECT_CALL(voice_engine_, DeRegisterVoiceEngineObserver()).WillOnce(testing::Return(0)); + EXPECT_CALL(voice_engine_, audio_device_module()); + EXPECT_CALL(voice_engine_, audio_processing()); + EXPECT_CALL(voice_engine_, audio_transport()); + config_.voice_engine = &voice_engine_; } AudioState::Config& config() { return config_; } @@ -76,5 +80,40 @@ TEST(AudioStateTest, TypingNoiseDetected) { voe_observer->CallbackOnError(-1, VE_NOT_INITED); EXPECT_FALSE(audio_state->typing_noise_detected()); } + +// Test that RecordedDataIsAvailable calls get to the original transport. +TEST(AudioStateTest, RecordedAudioArrivesAtOriginalTransport) { + using testing::_; + + ConfigHelper helper; + auto& voice_engine = helper.voice_engine(); + auto device = + static_cast(voice_engine.audio_device_module()); + + AudioTransport* audio_transport_proxy = nullptr; + ON_CALL(*device, RegisterAudioCallback(_)) + .WillByDefault( + testing::Invoke([&audio_transport_proxy](AudioTransport* transport) { + audio_transport_proxy = transport; + return 0; + })); + + MockAudioTransport original_audio_transport; + ON_CALL(voice_engine, audio_transport()) + .WillByDefault(testing::Return(&original_audio_transport)); + + EXPECT_CALL(voice_engine, audio_device_module()); + std::unique_ptr audio_state( + new internal::AudioState(helper.config())); + + // Setup completed. Ensure call of old transport is forwarded to new. + uint32_t new_mic_level; + EXPECT_CALL(original_audio_transport, + RecordedDataIsAvailable(nullptr, 80, 2, 1, 8000, 0, 0, 0, false, + testing::Ref(new_mic_level))); + + audio_transport_proxy->RecordedDataIsAvailable(nullptr, 80, 2, 1, 8000, 0, 0, + 0, false, new_mic_level); +} } // namespace test } // namespace webrtc diff --git a/webrtc/audio/audio_transport_proxy.cc b/webrtc/audio/audio_transport_proxy.cc new file mode 100644 index 0000000000..ed72200379 --- /dev/null +++ b/webrtc/audio/audio_transport_proxy.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/audio/audio_transport_proxy.h" + +namespace webrtc { + +AudioTransportProxy::AudioTransportProxy(AudioTransport* voe_audio_transport, + AudioProcessing* apm, + AudioMixer* mixer) + : voe_audio_transport_(voe_audio_transport) { + RTC_DCHECK(voe_audio_transport); + RTC_DCHECK(apm); +} + +AudioTransportProxy::~AudioTransportProxy() {} + +int32_t AudioTransportProxy::RecordedDataIsAvailable( + const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) { + // Pass call through to original audio transport instance. + return voe_audio_transport_->RecordedDataIsAvailable( + audioSamples, nSamples, nBytesPerSample, nChannels, samplesPerSec, + totalDelayMS, clockDrift, currentMicLevel, keyPressed, newMicLevel); +} + +int32_t AudioTransportProxy::NeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + RTC_DCHECK_EQ(sizeof(int16_t) * nChannels, nBytesPerSample); + RTC_DCHECK_GE(nChannels, 1u); + RTC_DCHECK_LE(nChannels, 2u); + RTC_DCHECK_GE( + samplesPerSec, + static_cast(AudioProcessing::NativeRate::kSampleRate8kHz)); + RTC_DCHECK_EQ(nSamples * 100, samplesPerSec); + RTC_DCHECK_LE(nBytesPerSample * nSamples * nChannels, + sizeof(AudioFrame::data_)); + + // Pass call through to original audio transport instance. + return voe_audio_transport_->NeedMorePlayData( + nSamples, nBytesPerSample, nChannels, samplesPerSec, audioSamples, + nSamplesOut, elapsed_time_ms, ntp_time_ms); +} + +void AudioTransportProxy::PushCaptureData(int voe_channel, + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) { + // This is part of deprecated VoE interface operating on specific + // VoE channels. It should not be used. + RTC_NOTREACHED(); +} + +void AudioTransportProxy::PullRenderData(int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + RTC_DCHECK_EQ(static_cast(bits_per_sample), 8 * sizeof(int16_t)); + RTC_DCHECK_GE(number_of_channels, 1u); + RTC_DCHECK_LE(number_of_channels, 2u); + RTC_DCHECK_GE(static_cast(sample_rate), + AudioProcessing::NativeRate::kSampleRate8kHz); + RTC_DCHECK_EQ(static_cast(number_of_frames * 100), sample_rate); + RTC_DCHECK_LE(bits_per_sample / 8 * number_of_frames * number_of_channels, + sizeof(AudioFrame::data_)); + voe_audio_transport_->PullRenderData( + bits_per_sample, sample_rate, number_of_channels, number_of_frames, + audio_data, elapsed_time_ms, ntp_time_ms); +} + +} // namespace webrtc diff --git a/webrtc/audio/audio_transport_proxy.h b/webrtc/audio/audio_transport_proxy.h new file mode 100644 index 0000000000..05e52fce85 --- /dev/null +++ b/webrtc/audio/audio_transport_proxy.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_ +#define WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_ + +#include "webrtc/api/audio/audio_mixer.h" +#include "webrtc/base/constructormagic.h" +#include "webrtc/modules/audio_device/include/audio_device_defines.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" + +namespace webrtc { + +class AudioTransportProxy : public AudioTransport { + public: + AudioTransportProxy(AudioTransport* voe_audio_transport, + AudioProcessing* apm, + AudioMixer* mixer); + + ~AudioTransportProxy() override; + + int32_t RecordedDataIsAvailable(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) override; + + int32_t NeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override; + + void PushCaptureData(int voe_channel, + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) override; + + void PullRenderData(int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override; + + private: + AudioTransport* voe_audio_transport_; + AudioFrame frame_for_mixing_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioTransportProxy); +}; +} // namespace webrtc + +#endif // WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_ diff --git a/webrtc/call/BUILD.gn b/webrtc/call/BUILD.gn index 98ea9bcae8..ad20f340fc 100644 --- a/webrtc/call/BUILD.gn +++ b/webrtc/call/BUILD.gn @@ -21,9 +21,12 @@ rtc_static_library("call") { suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] } + public_deps = [ + "../api:call_api", + ] + deps = [ "..:webrtc_common", - "../api:call_api", "../audio", "../base:rtc_task_queue", "../logging:rtc_event_log_impl", @@ -46,6 +49,7 @@ if (rtc_include_tests) { ] deps = [ ":call", + "../modules/audio_device:mock_audio_device", "//testing/gmock", "//testing/gtest", ] diff --git a/webrtc/call/DEPS b/webrtc/call/DEPS index 4256b589ff..01ad1d69cb 100644 --- a/webrtc/call/DEPS +++ b/webrtc/call/DEPS @@ -3,6 +3,8 @@ include_rules = [ "+webrtc/base", "+webrtc/logging/rtc_event_log", "+webrtc/modules/audio_coding", + "+webrtc/modules/audio_device", + "+webrtc/modules/audio_processing", "+webrtc/modules/bitrate_controller", "+webrtc/modules/congestion_controller", "+webrtc/modules/pacing", diff --git a/webrtc/call/call_unittest.cc b/webrtc/call/call_unittest.cc index 2b93deb857..d981b0b013 100644 --- a/webrtc/call/call_unittest.cc +++ b/webrtc/call/call_unittest.cc @@ -26,6 +26,9 @@ struct CallHelper { : voice_engine_(decoder_factory) { webrtc::AudioState::Config audio_state_config; audio_state_config.voice_engine = &voice_engine_; + EXPECT_CALL(voice_engine_, audio_device_module()); + EXPECT_CALL(voice_engine_, audio_processing()); + EXPECT_CALL(voice_engine_, audio_transport()); webrtc::Call::Config config(&event_log_); config.audio_state = webrtc::AudioState::Create(audio_state_config); call_.reset(webrtc::Call::Create(config)); diff --git a/webrtc/test/BUILD.gn b/webrtc/test/BUILD.gn index 326a3b1e95..28251e6cfb 100644 --- a/webrtc/test/BUILD.gn +++ b/webrtc/test/BUILD.gn @@ -333,6 +333,8 @@ rtc_source_set("test_common") { "../audio", "../base:rtc_base_approved", "../call", + "../modules/audio_device:mock_audio_device", + "../modules/audio_processing", "../modules/media_file", "../modules/video_capture:video_capture_module", "../video", diff --git a/webrtc/test/DEPS b/webrtc/test/DEPS index ccf57d064a..9c41bd4ce8 100644 --- a/webrtc/test/DEPS +++ b/webrtc/test/DEPS @@ -6,6 +6,7 @@ include_rules = [ "+webrtc/media/base", "+webrtc/modules/audio_coding", "+webrtc/modules/audio_device", + "+webrtc/modules/audio_processing", "+webrtc/modules/media_file", "+webrtc/modules/rtp_rtcp", "+webrtc/modules/video_capture", diff --git a/webrtc/test/mock_voice_engine.h b/webrtc/test/mock_voice_engine.h index 84c1f5a856..ced0751204 100644 --- a/webrtc/test/mock_voice_engine.h +++ b/webrtc/test/mock_voice_engine.h @@ -13,6 +13,9 @@ #include +#include "webrtc/modules/audio_device/include/mock_audio_device.h" +#include "webrtc/modules/audio_device/include/mock_audio_transport.h" +#include "webrtc/modules/audio_processing/include/mock_audio_processing.h" #include "webrtc/test/gmock.h" #include "webrtc/test/mock_voe_channel_proxy.h" #include "webrtc/voice_engine/voice_engine_impl.h" @@ -47,6 +50,13 @@ class MockVoiceEngine : public VoiceEngineImpl { .WillRepeatedly(testing::ReturnRef(decoder_factory_)); return proxy; })); + + ON_CALL(*this, audio_device_module()) + .WillByDefault(testing::Return(&mock_audio_device_)); + ON_CALL(*this, audio_processing()) + .WillByDefault(testing::Return(&mock_audio_processing_)); + ON_CALL(*this, audio_transport()) + .WillByDefault(testing::Return(&mock_audio_transport_)); } virtual ~MockVoiceEngine() /* override */ { // Decrease ref count before base class d-tor is called; otherwise it will @@ -111,6 +121,7 @@ class MockVoiceEngine : public VoiceEngineImpl { AudioProcessing* audioproc, const rtc::scoped_refptr& decoder_factory)); MOCK_METHOD0(audio_processing, AudioProcessing*()); + MOCK_METHOD0(audio_device_module, AudioDeviceModule*()); MOCK_METHOD0(Terminate, int()); MOCK_METHOD0(CreateChannel, int()); MOCK_METHOD1(CreateChannel, int(const ChannelConfig& config)); @@ -330,6 +341,10 @@ class MockVoiceEngine : public VoiceEngineImpl { // return a dangling reference. Fortunately, this should go away once // voe::Channel does. rtc::scoped_refptr decoder_factory_; + + MockAudioDeviceModule mock_audio_device_; + MockAudioProcessing mock_audio_processing_; + MockAudioTransport mock_audio_transport_; }; } // namespace test } // namespace webrtc