diff --git a/webrtc/audio/BUILD.gn b/webrtc/audio/BUILD.gn index 127900544c..e1d583f8e5 100644 --- a/webrtc/audio/BUILD.gn +++ b/webrtc/audio/BUILD.gn @@ -16,6 +16,8 @@ rtc_static_library("audio") { "audio_send_stream.h", "audio_state.cc", "audio_state.h", + "audio_transport_proxy.cc", + "audio_transport_proxy.h", "conversion.h", "scoped_voe_interface.h", ] @@ -29,6 +31,9 @@ rtc_static_library("audio") { "..:webrtc_common", "../api:audio_mixer_api", "../api:call_api", + "../base:rtc_base_approved", + "../modules/audio_device", + "../modules/audio_processing", "../system_wrappers", "../voice_engine", ] @@ -43,6 +48,7 @@ if (rtc_include_tests) { ] deps = [ ":audio", + "../modules/audio_device:mock_audio_device", "//testing/gmock", "//testing/gtest", ] diff --git a/webrtc/audio/DEPS b/webrtc/audio/DEPS index 476d463e6a..e53d28578e 100644 --- a/webrtc/audio/DEPS +++ b/webrtc/audio/DEPS @@ -1,9 +1,9 @@ include_rules = [ "+webrtc/base", - "+webrtc/voice_engine", - "+webrtc/modules/audio_coding/codecs/mock", "+webrtc/call", "+webrtc/logging/rtc_event_log", + "+webrtc/modules/audio_coding/codecs/mock", + "+webrtc/modules/audio_device", "+webrtc/modules/audio_processing/include", "+webrtc/modules/bitrate_controller", "+webrtc/modules/congestion_controller", diff --git a/webrtc/audio/audio_receive_stream_unittest.cc b/webrtc/audio/audio_receive_stream_unittest.cc index 0331d9b7f6..0fc93b8b34 100644 --- a/webrtc/audio/audio_receive_stream_unittest.cc +++ b/webrtc/audio/audio_receive_stream_unittest.cc @@ -78,6 +78,10 @@ struct ConfigHelper { RegisterVoiceEngineObserver(_)).WillOnce(Return(0)); EXPECT_CALL(voice_engine_, DeRegisterVoiceEngineObserver()).WillOnce(Return(0)); + EXPECT_CALL(voice_engine_, audio_processing()); + EXPECT_CALL(voice_engine_, audio_device_module()); + EXPECT_CALL(voice_engine_, audio_transport()); + AudioState::Config config; config.voice_engine = &voice_engine_; audio_state_ = AudioState::Create(config); diff --git a/webrtc/audio/audio_send_stream_unittest.cc b/webrtc/audio/audio_send_stream_unittest.cc index 427cda31a0..7e0ea45bb2 100644 --- a/webrtc/audio/audio_send_stream_unittest.cc +++ b/webrtc/audio/audio_send_stream_unittest.cc @@ -75,6 +75,10 @@ struct ConfigHelper { RegisterVoiceEngineObserver(_)).WillOnce(Return(0)); EXPECT_CALL(voice_engine_, DeRegisterVoiceEngineObserver()).WillOnce(Return(0)); + EXPECT_CALL(voice_engine_, audio_device_module()); + EXPECT_CALL(voice_engine_, audio_processing()); + EXPECT_CALL(voice_engine_, audio_transport()); + AudioState::Config config; config.voice_engine = &voice_engine_; audio_state_ = AudioState::Create(config); diff --git a/webrtc/audio/audio_state.cc b/webrtc/audio/audio_state.cc index e63f97af2d..95a90a5bf1 100644 --- a/webrtc/audio/audio_state.cc +++ b/webrtc/audio/audio_state.cc @@ -13,16 +13,28 @@ #include "webrtc/base/atomicops.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/modules/audio_device/include/audio_device.h" #include "webrtc/voice_engine/include/voe_errors.h" namespace webrtc { namespace internal { AudioState::AudioState(const AudioState::Config& config) - : config_(config), voe_base_(config.voice_engine) { + : config_(config), + voe_base_(config.voice_engine), + audio_transport_proxy_(voe_base_->audio_transport(), + voe_base_->audio_processing(), + config_.audio_mixer) { process_thread_checker_.DetachFromThread(); // Only one AudioState should be created per VoiceEngine. RTC_CHECK(voe_base_->RegisterVoiceEngineObserver(*this) != -1); + + auto* const device = voe_base_->audio_device_module(); + RTC_DCHECK(device); + + // This is needed for the Chrome implementation of RegisterAudioCallback. + device->RegisterAudioCallback(nullptr); + device->RegisterAudioCallback(&audio_transport_proxy_); } AudioState::~AudioState() { @@ -35,6 +47,10 @@ VoiceEngine* AudioState::voice_engine() { return config_.voice_engine; } +rtc::scoped_refptr AudioState::mixer() { + return config_.audio_mixer; +} + bool AudioState::typing_noise_detected() const { RTC_DCHECK(thread_checker_.CalledOnValidThread()); rtc::CritScope lock(&crit_sect_); diff --git a/webrtc/audio/audio_state.h b/webrtc/audio/audio_state.h index 31892d04fa..307f5cec4b 100644 --- a/webrtc/audio/audio_state.h +++ b/webrtc/audio/audio_state.h @@ -12,6 +12,7 @@ #define WEBRTC_AUDIO_AUDIO_STATE_H_ #include "webrtc/api/call/audio_state.h" +#include "webrtc/audio/audio_transport_proxy.h" #include "webrtc/audio/scoped_voe_interface.h" #include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" @@ -28,6 +29,8 @@ class AudioState final : public webrtc::AudioState, ~AudioState() override; VoiceEngine* voice_engine(); + + rtc::scoped_refptr mixer(); bool typing_noise_detected() const; private: @@ -53,6 +56,10 @@ class AudioState final : public webrtc::AudioState, // Reference count; implementation copied from rtc::RefCountedObject. mutable volatile int ref_count_ = 0; + // Transports mixed audio from the mixer to the audio device and + // recorded audio to the VoE AudioTransport. + AudioTransportProxy audio_transport_proxy_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioState); }; } // namespace internal diff --git a/webrtc/audio/audio_state_unittest.cc b/webrtc/audio/audio_state_unittest.cc index 86a9176db3..bd39baac1b 100644 --- a/webrtc/audio/audio_state_unittest.cc +++ b/webrtc/audio/audio_state_unittest.cc @@ -24,6 +24,10 @@ struct ConfigHelper { RegisterVoiceEngineObserver(testing::_)).WillOnce(testing::Return(0)); EXPECT_CALL(voice_engine_, DeRegisterVoiceEngineObserver()).WillOnce(testing::Return(0)); + EXPECT_CALL(voice_engine_, audio_device_module()); + EXPECT_CALL(voice_engine_, audio_processing()); + EXPECT_CALL(voice_engine_, audio_transport()); + config_.voice_engine = &voice_engine_; } AudioState::Config& config() { return config_; } @@ -76,5 +80,40 @@ TEST(AudioStateTest, TypingNoiseDetected) { voe_observer->CallbackOnError(-1, VE_NOT_INITED); EXPECT_FALSE(audio_state->typing_noise_detected()); } + +// Test that RecordedDataIsAvailable calls get to the original transport. +TEST(AudioStateTest, RecordedAudioArrivesAtOriginalTransport) { + using testing::_; + + ConfigHelper helper; + auto& voice_engine = helper.voice_engine(); + auto device = + static_cast(voice_engine.audio_device_module()); + + AudioTransport* audio_transport_proxy = nullptr; + ON_CALL(*device, RegisterAudioCallback(_)) + .WillByDefault( + testing::Invoke([&audio_transport_proxy](AudioTransport* transport) { + audio_transport_proxy = transport; + return 0; + })); + + MockAudioTransport original_audio_transport; + ON_CALL(voice_engine, audio_transport()) + .WillByDefault(testing::Return(&original_audio_transport)); + + EXPECT_CALL(voice_engine, audio_device_module()); + std::unique_ptr audio_state( + new internal::AudioState(helper.config())); + + // Setup completed. Ensure call of old transport is forwarded to new. + uint32_t new_mic_level; + EXPECT_CALL(original_audio_transport, + RecordedDataIsAvailable(nullptr, 80, 2, 1, 8000, 0, 0, 0, false, + testing::Ref(new_mic_level))); + + audio_transport_proxy->RecordedDataIsAvailable(nullptr, 80, 2, 1, 8000, 0, 0, + 0, false, new_mic_level); +} } // namespace test } // namespace webrtc diff --git a/webrtc/audio/audio_transport_proxy.cc b/webrtc/audio/audio_transport_proxy.cc new file mode 100644 index 0000000000..ed72200379 --- /dev/null +++ b/webrtc/audio/audio_transport_proxy.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/audio/audio_transport_proxy.h" + +namespace webrtc { + +AudioTransportProxy::AudioTransportProxy(AudioTransport* voe_audio_transport, + AudioProcessing* apm, + AudioMixer* mixer) + : voe_audio_transport_(voe_audio_transport) { + RTC_DCHECK(voe_audio_transport); + RTC_DCHECK(apm); +} + +AudioTransportProxy::~AudioTransportProxy() {} + +int32_t AudioTransportProxy::RecordedDataIsAvailable( + const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) { + // Pass call through to original audio transport instance. + return voe_audio_transport_->RecordedDataIsAvailable( + audioSamples, nSamples, nBytesPerSample, nChannels, samplesPerSec, + totalDelayMS, clockDrift, currentMicLevel, keyPressed, newMicLevel); +} + +int32_t AudioTransportProxy::NeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + RTC_DCHECK_EQ(sizeof(int16_t) * nChannels, nBytesPerSample); + RTC_DCHECK_GE(nChannels, 1u); + RTC_DCHECK_LE(nChannels, 2u); + RTC_DCHECK_GE( + samplesPerSec, + static_cast(AudioProcessing::NativeRate::kSampleRate8kHz)); + RTC_DCHECK_EQ(nSamples * 100, samplesPerSec); + RTC_DCHECK_LE(nBytesPerSample * nSamples * nChannels, + sizeof(AudioFrame::data_)); + + // Pass call through to original audio transport instance. + return voe_audio_transport_->NeedMorePlayData( + nSamples, nBytesPerSample, nChannels, samplesPerSec, audioSamples, + nSamplesOut, elapsed_time_ms, ntp_time_ms); +} + +void AudioTransportProxy::PushCaptureData(int voe_channel, + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) { + // This is part of deprecated VoE interface operating on specific + // VoE channels. It should not be used. + RTC_NOTREACHED(); +} + +void AudioTransportProxy::PullRenderData(int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + RTC_DCHECK_EQ(static_cast(bits_per_sample), 8 * sizeof(int16_t)); + RTC_DCHECK_GE(number_of_channels, 1u); + RTC_DCHECK_LE(number_of_channels, 2u); + RTC_DCHECK_GE(static_cast(sample_rate), + AudioProcessing::NativeRate::kSampleRate8kHz); + RTC_DCHECK_EQ(static_cast(number_of_frames * 100), sample_rate); + RTC_DCHECK_LE(bits_per_sample / 8 * number_of_frames * number_of_channels, + sizeof(AudioFrame::data_)); + voe_audio_transport_->PullRenderData( + bits_per_sample, sample_rate, number_of_channels, number_of_frames, + audio_data, elapsed_time_ms, ntp_time_ms); +} + +} // namespace webrtc diff --git a/webrtc/audio/audio_transport_proxy.h b/webrtc/audio/audio_transport_proxy.h new file mode 100644 index 0000000000..05e52fce85 --- /dev/null +++ b/webrtc/audio/audio_transport_proxy.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_ +#define WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_ + +#include "webrtc/api/audio/audio_mixer.h" +#include "webrtc/base/constructormagic.h" +#include "webrtc/modules/audio_device/include/audio_device_defines.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" + +namespace webrtc { + +class AudioTransportProxy : public AudioTransport { + public: + AudioTransportProxy(AudioTransport* voe_audio_transport, + AudioProcessing* apm, + AudioMixer* mixer); + + ~AudioTransportProxy() override; + + int32_t RecordedDataIsAvailable(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) override; + + int32_t NeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override; + + void PushCaptureData(int voe_channel, + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) override; + + void PullRenderData(int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override; + + private: + AudioTransport* voe_audio_transport_; + AudioFrame frame_for_mixing_; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioTransportProxy); +}; +} // namespace webrtc + +#endif // WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_ diff --git a/webrtc/call/BUILD.gn b/webrtc/call/BUILD.gn index 98ea9bcae8..ad20f340fc 100644 --- a/webrtc/call/BUILD.gn +++ b/webrtc/call/BUILD.gn @@ -21,9 +21,12 @@ rtc_static_library("call") { suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] } + public_deps = [ + "../api:call_api", + ] + deps = [ "..:webrtc_common", - "../api:call_api", "../audio", "../base:rtc_task_queue", "../logging:rtc_event_log_impl", @@ -46,6 +49,7 @@ if (rtc_include_tests) { ] deps = [ ":call", + "../modules/audio_device:mock_audio_device", "//testing/gmock", "//testing/gtest", ] diff --git a/webrtc/call/DEPS b/webrtc/call/DEPS index 4256b589ff..01ad1d69cb 100644 --- a/webrtc/call/DEPS +++ b/webrtc/call/DEPS @@ -3,6 +3,8 @@ include_rules = [ "+webrtc/base", "+webrtc/logging/rtc_event_log", "+webrtc/modules/audio_coding", + "+webrtc/modules/audio_device", + "+webrtc/modules/audio_processing", "+webrtc/modules/bitrate_controller", "+webrtc/modules/congestion_controller", "+webrtc/modules/pacing", diff --git a/webrtc/call/call_unittest.cc b/webrtc/call/call_unittest.cc index 2b93deb857..d981b0b013 100644 --- a/webrtc/call/call_unittest.cc +++ b/webrtc/call/call_unittest.cc @@ -26,6 +26,9 @@ struct CallHelper { : voice_engine_(decoder_factory) { webrtc::AudioState::Config audio_state_config; audio_state_config.voice_engine = &voice_engine_; + EXPECT_CALL(voice_engine_, audio_device_module()); + EXPECT_CALL(voice_engine_, audio_processing()); + EXPECT_CALL(voice_engine_, audio_transport()); webrtc::Call::Config config(&event_log_); config.audio_state = webrtc::AudioState::Create(audio_state_config); call_.reset(webrtc::Call::Create(config)); diff --git a/webrtc/test/BUILD.gn b/webrtc/test/BUILD.gn index 326a3b1e95..28251e6cfb 100644 --- a/webrtc/test/BUILD.gn +++ b/webrtc/test/BUILD.gn @@ -333,6 +333,8 @@ rtc_source_set("test_common") { "../audio", "../base:rtc_base_approved", "../call", + "../modules/audio_device:mock_audio_device", + "../modules/audio_processing", "../modules/media_file", "../modules/video_capture:video_capture_module", "../video", diff --git a/webrtc/test/DEPS b/webrtc/test/DEPS index ccf57d064a..9c41bd4ce8 100644 --- a/webrtc/test/DEPS +++ b/webrtc/test/DEPS @@ -6,6 +6,7 @@ include_rules = [ "+webrtc/media/base", "+webrtc/modules/audio_coding", "+webrtc/modules/audio_device", + "+webrtc/modules/audio_processing", "+webrtc/modules/media_file", "+webrtc/modules/rtp_rtcp", "+webrtc/modules/video_capture", diff --git a/webrtc/test/mock_voice_engine.h b/webrtc/test/mock_voice_engine.h index 84c1f5a856..ced0751204 100644 --- a/webrtc/test/mock_voice_engine.h +++ b/webrtc/test/mock_voice_engine.h @@ -13,6 +13,9 @@ #include +#include "webrtc/modules/audio_device/include/mock_audio_device.h" +#include "webrtc/modules/audio_device/include/mock_audio_transport.h" +#include "webrtc/modules/audio_processing/include/mock_audio_processing.h" #include "webrtc/test/gmock.h" #include "webrtc/test/mock_voe_channel_proxy.h" #include "webrtc/voice_engine/voice_engine_impl.h" @@ -47,6 +50,13 @@ class MockVoiceEngine : public VoiceEngineImpl { .WillRepeatedly(testing::ReturnRef(decoder_factory_)); return proxy; })); + + ON_CALL(*this, audio_device_module()) + .WillByDefault(testing::Return(&mock_audio_device_)); + ON_CALL(*this, audio_processing()) + .WillByDefault(testing::Return(&mock_audio_processing_)); + ON_CALL(*this, audio_transport()) + .WillByDefault(testing::Return(&mock_audio_transport_)); } virtual ~MockVoiceEngine() /* override */ { // Decrease ref count before base class d-tor is called; otherwise it will @@ -111,6 +121,7 @@ class MockVoiceEngine : public VoiceEngineImpl { AudioProcessing* audioproc, const rtc::scoped_refptr& decoder_factory)); MOCK_METHOD0(audio_processing, AudioProcessing*()); + MOCK_METHOD0(audio_device_module, AudioDeviceModule*()); MOCK_METHOD0(Terminate, int()); MOCK_METHOD0(CreateChannel, int()); MOCK_METHOD1(CreateChannel, int(const ChannelConfig& config)); @@ -330,6 +341,10 @@ class MockVoiceEngine : public VoiceEngineImpl { // return a dangling reference. Fortunately, this should go away once // voe::Channel does. rtc::scoped_refptr decoder_factory_; + + MockAudioDeviceModule mock_audio_device_; + MockAudioProcessing mock_audio_processing_; + MockAudioTransport mock_audio_transport_; }; } // namespace test } // namespace webrtc