From a9cc40b7d263cfce28bd3f126481e99283623d75 Mon Sep 17 00:00:00 2001 From: peah Date: Thu, 29 Jun 2017 08:32:09 -0700 Subject: [PATCH] Allow an external audio processing module to be used in WebRTC [This CL is a rebase of an original CL by solenberg@: https://codereview.webrtc.org/2948763002/ which in turn was a rebase of an original CL by peah@: https://chromium-review.googlesource.com/c/527032/] Allow an external audio processing module to be used in WebRTC This CL adds support for optionally using an externally created audio processing module in a peerconnection. The ownership is shared between the peerconnection and the external creator of the module. As part of this the internal ownership of the audio processing module is moved from VoiceEngine to WebRtcVoiceEngine. BUG=webrtc:7775 Review-Url: https://codereview.webrtc.org/2961723004 Cr-Commit-Position: refs/heads/master@{#18837} --- webrtc/api/peerconnectioninterface.h | 2 + webrtc/audio/audio_receive_stream_unittest.cc | 3 +- webrtc/audio/audio_send_stream.cc | 5 +- webrtc/audio/audio_send_stream_unittest.cc | 9 +- webrtc/audio/audio_state.cc | 6 +- webrtc/audio/audio_state.h | 8 +- webrtc/audio/audio_state_unittest.cc | 5 +- webrtc/audio/audio_transport_proxy.cc | 10 +- webrtc/audio/audio_transport_proxy.h | 4 +- webrtc/call/audio_state.h | 6 ++ webrtc/call/call_perf_tests.cc | 7 +- webrtc/call/call_unittest.cc | 6 +- webrtc/media/base/fakemediaengine.h | 5 +- webrtc/media/base/mediaengine.h | 22 +++-- webrtc/media/engine/apm_helpers_unittest.cc | 15 ++- webrtc/media/engine/fakewebrtcvoiceengine.h | 14 +-- .../engine/nullwebrtcvideoengine_unittest.cc | 4 +- webrtc/media/engine/webrtcmediaengine.cc | 55 ++++++----- webrtc/media/engine/webrtcmediaengine.h | 7 +- webrtc/media/engine/webrtcvoiceengine.cc | 29 +++--- webrtc/media/engine/webrtcvoiceengine.h | 6 +- .../engine/webrtcvoiceengine_unittest.cc | 92 +++++++++++-------- .../audio_processing/audio_processing_impl.cc | 3 +- .../audio_processing_impl_unittest.cc | 6 ++ .../audio_processing_unittest.cc | 9 +- .../include/audio_processing.h | 5 +- webrtc/ortc/BUILD.gn | 1 + webrtc/ortc/DEPS | 1 + webrtc/ortc/ortcfactory.cc | 7 +- webrtc/pc/BUILD.gn | 1 + webrtc/pc/DEPS | 1 + webrtc/pc/createpeerconnectionfactory.cc | 4 +- webrtc/pc/peerconnectioninterface_unittest.cc | 3 +- webrtc/sdk/android/BUILD.gn | 5 +- webrtc/sdk/android/src/jni/DEPS | 1 + webrtc/sdk/android/src/jni/media_jni.cc | 3 +- webrtc/test/call_test.cc | 12 ++- webrtc/test/call_test.h | 2 + webrtc/test/mock_voice_engine.h | 7 +- webrtc/video/video_quality_test.cc | 15 ++- webrtc/voice_engine/include/voe_base.h | 13 +-- webrtc/voice_engine/shared_data.cc | 1 - webrtc/voice_engine/shared_data.h | 2 - .../auto_test/fakes/conference_transport.cc | 9 +- .../auto_test/fakes/conference_transport.h | 3 +- .../test/auto_test/standard/codec_test.cc | 3 + webrtc/voice_engine/voe_base_impl.cc | 34 ++++--- webrtc/voice_engine/voe_base_impl.h | 17 +++- webrtc/voice_engine/voe_base_unittest.cc | 16 +--- webrtc/voice_engine/voe_codec_unittest.cc | 4 +- webrtc/voice_engine/voe_network_unittest.cc | 16 ++-- webrtc/voice_engine/voice_engine_fixture.cc | 2 + webrtc/voice_engine/voice_engine_fixture.h | 2 + 53 files changed, 320 insertions(+), 208 deletions(-) diff --git a/webrtc/api/peerconnectioninterface.h b/webrtc/api/peerconnectioninterface.h index ee0121246d..1b74564d6e 100644 --- a/webrtc/api/peerconnectioninterface.h +++ b/webrtc/api/peerconnectioninterface.h @@ -1163,6 +1163,8 @@ CreatePeerConnectionFactory( // |video_encoder_factory| and |video_decoder_factory| is transferred to the // returned factory. // +// If |audio_mixer| is null, an internal audio mixer will be created and used. +// // TODO(deadbeef): Use rtc::scoped_refptr<> and std::unique_ptr<> to make this // ownership transfer and ref counting more obvious. // diff --git a/webrtc/audio/audio_receive_stream_unittest.cc b/webrtc/audio/audio_receive_stream_unittest.cc index 84efb20b91..127ea077b0 100644 --- a/webrtc/audio/audio_receive_stream_unittest.cc +++ b/webrtc/audio/audio_receive_stream_unittest.cc @@ -17,6 +17,7 @@ #include "webrtc/audio/conversion.h" #include "webrtc/call/rtp_stream_receiver_controller.h" #include "webrtc/logging/rtc_event_log/mock/mock_rtc_event_log.h" +#include "webrtc/modules/audio_processing/include/mock_audio_processing.h" #include "webrtc/modules/bitrate_controller/include/mock/mock_bitrate_controller.h" #include "webrtc/modules/pacing/packet_router.h" #include "webrtc/modules/rtp_rtcp/source/byte_io.h" @@ -74,13 +75,13 @@ struct ConfigHelper { RegisterVoiceEngineObserver(_)).WillOnce(Return(0)); EXPECT_CALL(voice_engine_, DeRegisterVoiceEngineObserver()).WillOnce(Return(0)); - EXPECT_CALL(voice_engine_, audio_processing()); EXPECT_CALL(voice_engine_, audio_device_module()); EXPECT_CALL(voice_engine_, audio_transport()); AudioState::Config config; config.voice_engine = &voice_engine_; config.audio_mixer = audio_mixer_; + config.audio_processing = new rtc::RefCountedObject(); audio_state_ = AudioState::Create(config); EXPECT_CALL(voice_engine_, ChannelProxyFactory(kChannelId)) diff --git a/webrtc/audio/audio_send_stream.cc b/webrtc/audio/audio_send_stream.cc index f89da981e4..f8ee3ab565 100644 --- a/webrtc/audio/audio_send_stream.cc +++ b/webrtc/audio/audio_send_stream.cc @@ -279,8 +279,9 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { stats.audio_level = base->transmit_mixer()->AudioLevelFullRange(); RTC_DCHECK_LE(0, stats.audio_level); - RTC_DCHECK(base->audio_processing()); - auto audio_processing_stats = base->audio_processing()->GetStatistics(); + RTC_DCHECK(audio_state_->audio_processing()); + auto audio_processing_stats = + audio_state_->audio_processing()->GetStatistics(); stats.echo_delay_median_ms = audio_processing_stats.delay_median; stats.echo_delay_std_ms = audio_processing_stats.delay_standard_deviation; stats.echo_return_loss = audio_processing_stats.echo_return_loss.instant(); diff --git a/webrtc/audio/audio_send_stream_unittest.cc b/webrtc/audio/audio_send_stream_unittest.cc index e6acf92efb..efc18d127d 100644 --- a/webrtc/audio/audio_send_stream_unittest.cc +++ b/webrtc/audio/audio_send_stream_unittest.cc @@ -128,6 +128,7 @@ rtc::scoped_refptr SetupEncoderFactoryMock() { struct ConfigHelper { ConfigHelper(bool audio_bwe_enabled, bool expect_set_encoder_call) : stream_config_(nullptr), + audio_processing_(new rtc::RefCountedObject()), simulated_clock_(123456), send_side_cc_(rtc::MakeUnique( &simulated_clock_, @@ -144,12 +145,12 @@ struct ConfigHelper { EXPECT_CALL(voice_engine_, DeRegisterVoiceEngineObserver()).WillOnce(Return(0)); EXPECT_CALL(voice_engine_, audio_device_module()); - EXPECT_CALL(voice_engine_, audio_processing()); EXPECT_CALL(voice_engine_, audio_transport()); AudioState::Config config; config.voice_engine = &voice_engine_; config.audio_mixer = AudioMixerImpl::Create(); + config.audio_processing = audio_processing_; audio_state_ = AudioState::Create(config); SetupDefaultChannelProxy(audio_bwe_enabled); @@ -278,8 +279,6 @@ struct ConfigHelper { .WillRepeatedly(Return(report_blocks)); EXPECT_CALL(voice_engine_, transmit_mixer()) .WillRepeatedly(Return(&transmit_mixer_)); - EXPECT_CALL(voice_engine_, audio_processing()) - .WillRepeatedly(Return(&audio_processing_)); EXPECT_CALL(transmit_mixer_, AudioLevelFullRange()) .WillRepeatedly(Return(kSpeechInputLevel)); @@ -294,7 +293,7 @@ struct ConfigHelper { audio_processing_stats_.delay_median = kEchoDelayMedian; audio_processing_stats_.delay_standard_deviation = kEchoDelayStdDev; - EXPECT_CALL(audio_processing_, GetStatistics()) + EXPECT_CALL(*audio_processing_, GetStatistics()) .WillRepeatedly(Return(audio_processing_stats_)); } @@ -303,7 +302,7 @@ struct ConfigHelper { rtc::scoped_refptr audio_state_; AudioSendStream::Config stream_config_; testing::StrictMock* channel_proxy_ = nullptr; - MockAudioProcessing audio_processing_; + rtc::scoped_refptr audio_processing_; MockTransmitMixer transmit_mixer_; AudioProcessing::AudioProcessingStatistics audio_processing_stats_; SimulatedClock simulated_clock_; diff --git a/webrtc/audio/audio_state.cc b/webrtc/audio/audio_state.cc index c15ddd7795..961e772a9d 100644 --- a/webrtc/audio/audio_state.cc +++ b/webrtc/audio/audio_state.cc @@ -19,11 +19,15 @@ namespace webrtc { namespace internal { +// TODO(peah): Remove the conditional in the audio_transport_proxy_ constructor +// call when upstream dependencies have properly been resolved. AudioState::AudioState(const AudioState::Config& config) : config_(config), voe_base_(config.voice_engine), audio_transport_proxy_(voe_base_->audio_transport(), - voe_base_->audio_processing(), + config_.audio_processing + ? config_.audio_processing.get() + : voe_base_->audio_processing(), config_.audio_mixer) { process_thread_checker_.DetachFromThread(); RTC_DCHECK(config_.audio_mixer); diff --git a/webrtc/audio/audio_state.h b/webrtc/audio/audio_state.h index 29b5a9edbf..8a98d5739e 100644 --- a/webrtc/audio/audio_state.h +++ b/webrtc/audio/audio_state.h @@ -28,8 +28,14 @@ class AudioState final : public webrtc::AudioState, explicit AudioState(const AudioState::Config& config); ~AudioState() override; - VoiceEngine* voice_engine(); + // TODO(peah): Remove the conditional when upstream dependencies have properly + // been resolved. + AudioProcessing* audio_processing() override { + return config_.audio_processing ? config_.audio_processing.get() + : voe_base_->audio_processing(); + } + VoiceEngine* voice_engine(); rtc::scoped_refptr mixer(); bool typing_noise_detected() const; diff --git a/webrtc/audio/audio_state_unittest.cc b/webrtc/audio/audio_state_unittest.cc index 05c5003459..ffaacb62ca 100644 --- a/webrtc/audio/audio_state_unittest.cc +++ b/webrtc/audio/audio_state_unittest.cc @@ -12,6 +12,7 @@ #include "webrtc/audio/audio_state.h" #include "webrtc/modules/audio_mixer/audio_mixer_impl.h" +#include "webrtc/modules/audio_processing/include/mock_audio_processing.h" #include "webrtc/test/gtest.h" #include "webrtc/test/mock_voice_engine.h" @@ -31,8 +32,6 @@ struct ConfigHelper { .WillOnce(testing::Return(0)); EXPECT_CALL(mock_voice_engine, audio_device_module()) .Times(testing::AtLeast(1)); - EXPECT_CALL(mock_voice_engine, audio_processing()) - .Times(testing::AtLeast(1)); EXPECT_CALL(mock_voice_engine, audio_transport()) .WillRepeatedly(testing::Return(&audio_transport)); @@ -49,6 +48,8 @@ struct ConfigHelper { audio_state_config.voice_engine = &mock_voice_engine; audio_state_config.audio_mixer = audio_mixer; + audio_state_config.audio_processing = + new rtc::RefCountedObject(); } AudioState::Config& config() { return audio_state_config; } MockVoiceEngine& voice_engine() { return mock_voice_engine; } diff --git a/webrtc/audio/audio_transport_proxy.cc b/webrtc/audio/audio_transport_proxy.cc index d6ce9397c7..73b0091540 100644 --- a/webrtc/audio/audio_transport_proxy.cc +++ b/webrtc/audio/audio_transport_proxy.cc @@ -34,11 +34,13 @@ int Resample(const AudioFrame& frame, } // namespace AudioTransportProxy::AudioTransportProxy(AudioTransport* voe_audio_transport, - AudioProcessing* apm, + AudioProcessing* audio_processing, AudioMixer* mixer) - : voe_audio_transport_(voe_audio_transport), apm_(apm), mixer_(mixer) { + : voe_audio_transport_(voe_audio_transport), + audio_processing_(audio_processing), + mixer_(mixer) { RTC_DCHECK(voe_audio_transport); - RTC_DCHECK(apm); + RTC_DCHECK(audio_processing); RTC_DCHECK(mixer); } @@ -85,7 +87,7 @@ int32_t AudioTransportProxy::NeedMorePlayData(const size_t nSamples, *elapsed_time_ms = mixed_frame_.elapsed_time_ms_; *ntp_time_ms = mixed_frame_.ntp_time_ms_; - const auto error = apm_->ProcessReverseStream(&mixed_frame_); + const auto error = audio_processing_->ProcessReverseStream(&mixed_frame_); RTC_DCHECK_EQ(error, AudioProcessing::kNoError); nSamplesOut = Resample(mixed_frame_, samplesPerSec, &resampler_, diff --git a/webrtc/audio/audio_transport_proxy.h b/webrtc/audio/audio_transport_proxy.h index 1d03378402..fda9339171 100644 --- a/webrtc/audio/audio_transport_proxy.h +++ b/webrtc/audio/audio_transport_proxy.h @@ -23,7 +23,7 @@ namespace webrtc { class AudioTransportProxy : public AudioTransport { public: AudioTransportProxy(AudioTransport* voe_audio_transport, - AudioProcessing* apm, + AudioProcessing* audio_processing, AudioMixer* mixer); ~AudioTransportProxy() override; @@ -65,7 +65,7 @@ class AudioTransportProxy : public AudioTransport { private: AudioTransport* voe_audio_transport_; - AudioProcessing* apm_; + AudioProcessing* audio_processing_; rtc::scoped_refptr mixer_; AudioFrame mixed_frame_; // Converts mixed audio to the audio device output rate. diff --git a/webrtc/call/audio_state.h b/webrtc/call/audio_state.h index 2c26a1749b..32b7ada3f8 100644 --- a/webrtc/call/audio_state.h +++ b/webrtc/call/audio_state.h @@ -16,6 +16,7 @@ namespace webrtc { +class AudioProcessing; class VoiceEngine; // WORK IN PROGRESS @@ -36,8 +37,13 @@ class AudioState : public rtc::RefCountInterface { // The audio mixer connected to active receive streams. One per // AudioState. rtc::scoped_refptr audio_mixer; + + // The audio processing module. + rtc::scoped_refptr audio_processing; }; + virtual AudioProcessing* audio_processing() = 0; + // TODO(solenberg): Replace scoped_refptr with shared_ptr once we can use it. static rtc::scoped_refptr Create( const AudioState::Config& config); diff --git a/webrtc/call/call_perf_tests.cc b/webrtc/call/call_perf_tests.cc index 3b1de73e65..0c0a0cfd29 100644 --- a/webrtc/call/call_perf_tests.cc +++ b/webrtc/call/call_perf_tests.cc @@ -145,12 +145,15 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec, const uint32_t kAudioRecvSsrc = 5678; metrics::Reset(); + rtc::scoped_refptr audio_processing = + AudioProcessing::Create(); VoiceEngine* voice_engine = VoiceEngine::Create(); VoEBase* voe_base = VoEBase::GetInterface(voice_engine); FakeAudioDevice fake_audio_device( FakeAudioDevice::CreatePulsedNoiseCapturer(256, 48000), FakeAudioDevice::CreateDiscardRenderer(48000), audio_rtp_speed); - EXPECT_EQ(0, voe_base->Init(&fake_audio_device, nullptr, decoder_factory_)); + EXPECT_EQ(0, voe_base->Init(&fake_audio_device, audio_processing.get(), + decoder_factory_)); VoEBase::ChannelConfig config; config.enable_voice_pacing = true; int send_channel_id = voe_base->CreateChannel(config); @@ -159,7 +162,9 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec, AudioState::Config send_audio_state_config; send_audio_state_config.voice_engine = voice_engine; send_audio_state_config.audio_mixer = AudioMixerImpl::Create(); + send_audio_state_config.audio_processing = audio_processing; Call::Config sender_config(event_log_.get()); + sender_config.audio_state = AudioState::Create(send_audio_state_config); Call::Config receiver_config(event_log_.get()); receiver_config.audio_state = sender_config.audio_state; diff --git a/webrtc/call/call_unittest.cc b/webrtc/call/call_unittest.cc index 8f0a340e72..5267e7a6ee 100644 --- a/webrtc/call/call_unittest.cc +++ b/webrtc/call/call_unittest.cc @@ -37,8 +37,8 @@ struct CallHelper { webrtc::AudioState::Config audio_state_config; audio_state_config.voice_engine = &voice_engine_; audio_state_config.audio_mixer = webrtc::AudioMixerImpl::Create(); + audio_state_config.audio_processing = webrtc::AudioProcessing::Create(); EXPECT_CALL(voice_engine_, audio_device_module()); - EXPECT_CALL(voice_engine_, audio_processing()); EXPECT_CALL(voice_engine_, audio_transport()); webrtc::Call::Config config(&event_log_); config.audio_state = webrtc::AudioState::Create(audio_state_config); @@ -453,11 +453,13 @@ TEST(CallTest, RecreatingAudioStreamWithSameSsrcReusesRtpState) { }; ScopedVoiceEngine voice_engine; - voice_engine.base->Init(&mock_adm); AudioState::Config audio_state_config; audio_state_config.voice_engine = voice_engine.voe; audio_state_config.audio_mixer = mock_mixer; + audio_state_config.audio_processing = AudioProcessing::Create(); + voice_engine.base->Init(&mock_adm, audio_state_config.audio_processing.get()); auto audio_state = AudioState::Create(audio_state_config); + RtcEventLogNullImpl event_log; Call::Config call_config(&event_log); call_config.audio_state = audio_state; diff --git a/webrtc/media/base/fakemediaengine.h b/webrtc/media/base/fakemediaengine.h index 8b0c92031a..04dfae0a59 100644 --- a/webrtc/media/base/fakemediaengine.h +++ b/webrtc/media/base/fakemediaengine.h @@ -27,6 +27,7 @@ #include "webrtc/media/base/mediaengine.h" #include "webrtc/media/base/rtputils.h" #include "webrtc/media/base/streamparams.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/p2p/base/sessiondescription.h" using webrtc::RtpExtension; @@ -774,7 +775,8 @@ class FakeVoiceEngine : public FakeBaseEngine { audio_encoder_factory, const rtc::scoped_refptr& audio_decoder_factory, - rtc::scoped_refptr audio_mixer) { + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr apm) { // Add a fake audio codec. Note that the name must not be "" as there are // sanity checks against that. codecs_.push_back(AudioCodec(101, "fake_audio_codec", 0, 0, 1)); @@ -883,6 +885,7 @@ class FakeMediaEngine : public: FakeMediaEngine() : CompositeMediaEngine(nullptr, + nullptr, nullptr, nullptr, nullptr) {} diff --git a/webrtc/media/base/mediaengine.h b/webrtc/media/base/mediaengine.h index ea9ef9964c..911b971648 100644 --- a/webrtc/media/base/mediaengine.h +++ b/webrtc/media/base/mediaengine.h @@ -34,6 +34,7 @@ namespace webrtc { class AudioDeviceModule; class AudioMixer; +class AudioProcessing; class Call; } @@ -111,14 +112,19 @@ class MediaEngineFactory { template class CompositeMediaEngine : public MediaEngineInterface { public: - CompositeMediaEngine(webrtc::AudioDeviceModule* adm, - const rtc::scoped_refptr& - audio_encoder_factory, - const rtc::scoped_refptr& - audio_decoder_factory, - rtc::scoped_refptr audio_mixer) - : voice_(adm, audio_encoder_factory, audio_decoder_factory, audio_mixer) { - } + CompositeMediaEngine( + webrtc::AudioDeviceModule* adm, + const rtc::scoped_refptr& + audio_encoder_factory, + const rtc::scoped_refptr& + audio_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing) + : voice_(adm, + audio_encoder_factory, + audio_decoder_factory, + audio_mixer, + audio_processing) {} virtual ~CompositeMediaEngine() {} virtual bool Init() { voice_.Init(); diff --git a/webrtc/media/engine/apm_helpers_unittest.cc b/webrtc/media/engine/apm_helpers_unittest.cc index 6845d9e454..a86990301c 100644 --- a/webrtc/media/engine/apm_helpers_unittest.cc +++ b/webrtc/media/engine/apm_helpers_unittest.cc @@ -33,19 +33,15 @@ struct TestHelper { // This replicates the conditions from voe_auto_test. Config config; config.Set(new ExperimentalAgc(false)); + apm_ = rtc::scoped_refptr(AudioProcessing::Create(config)); EXPECT_EQ(0, voe_wrapper_.base()->Init( - &mock_audio_device_, - AudioProcessing::Create(config), - MockAudioDecoderFactory::CreateEmptyFactory())); + &mock_audio_device_, apm_, + MockAudioDecoderFactory::CreateEmptyFactory())); } - AudioProcessing* apm() { - return voe_wrapper_.base()->audio_processing(); - } + AudioProcessing* apm() { return apm_.get(); } - const AudioProcessing* apm() const { - return voe_wrapper_.base()->audio_processing(); - } + const AudioProcessing* apm() const { return apm_.get(); } test::MockAudioDeviceModule* adm() { return &mock_audio_device_; @@ -77,6 +73,7 @@ struct TestHelper { private: testing::NiceMock mock_audio_device_; cricket::VoEWrapper voe_wrapper_; + rtc::scoped_refptr apm_; }; } // namespace diff --git a/webrtc/media/engine/fakewebrtcvoiceengine.h b/webrtc/media/engine/fakewebrtcvoiceengine.h index 13721ea554..f8343f2e49 100644 --- a/webrtc/media/engine/fakewebrtcvoiceengine.h +++ b/webrtc/media/engine/fakewebrtcvoiceengine.h @@ -16,7 +16,6 @@ #include "webrtc/base/checks.h" #include "webrtc/media/engine/webrtcvoe.h" -#include "webrtc/modules/audio_processing/include/audio_processing.h" namespace webrtc { namespace voe { @@ -42,10 +41,8 @@ class FakeWebRtcVoiceEngine : public webrtc::VoEBase { bool neteq_fast_accelerate = false; }; - explicit FakeWebRtcVoiceEngine(webrtc::AudioProcessing* apm, - webrtc::voe::TransmitMixer* transmit_mixer) - : apm_(apm), transmit_mixer_(transmit_mixer) { - } + explicit FakeWebRtcVoiceEngine(webrtc::voe::TransmitMixer* transmit_mixer) + : transmit_mixer_(transmit_mixer) {} ~FakeWebRtcVoiceEngine() override { RTC_CHECK(channels_.empty()); } @@ -75,9 +72,9 @@ class FakeWebRtcVoiceEngine : public webrtc::VoEBase { inited_ = false; return 0; } - webrtc::AudioProcessing* audio_processing() override { - return apm_; - } + // TODO(peah): Remove this when downstream dependencies have properly been + // resolved. + webrtc::AudioProcessing* audio_processing() override { return nullptr; } webrtc::AudioDeviceModule* audio_device_module() override { return nullptr; } @@ -131,7 +128,6 @@ class FakeWebRtcVoiceEngine : public webrtc::VoEBase { int last_channel_ = -1; std::map channels_; bool fail_create_channel_ = false; - webrtc::AudioProcessing* apm_ = nullptr; webrtc::voe::TransmitMixer* transmit_mixer_ = nullptr; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FakeWebRtcVoiceEngine); diff --git a/webrtc/media/engine/nullwebrtcvideoengine_unittest.cc b/webrtc/media/engine/nullwebrtcvideoengine_unittest.cc index 342b5a8936..e1a2964722 100644 --- a/webrtc/media/engine/nullwebrtcvideoengine_unittest.cc +++ b/webrtc/media/engine/nullwebrtcvideoengine_unittest.cc @@ -10,6 +10,7 @@ #include "webrtc/media/engine/nullwebrtcvideoengine.h" #include "webrtc/media/engine/webrtcvoiceengine.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/test/gtest.h" #include "webrtc/test/mock_audio_decoder_factory.h" #include "webrtc/test/mock_audio_encoder_factory.h" @@ -31,7 +32,8 @@ class WebRtcMediaEngineNullVideo adm, audio_encoder_factory, audio_decoder_factory, - nullptr) { + nullptr, + webrtc::AudioProcessing::Create()) { video_.SetExternalDecoderFactory(video_decoder_factory); video_.SetExternalEncoderFactory(video_encoder_factory); } diff --git a/webrtc/media/engine/webrtcmediaengine.cc b/webrtc/media/engine/webrtcmediaengine.cc index 3c937f21ac..9e02f58b84 100644 --- a/webrtc/media/engine/webrtcmediaengine.cc +++ b/webrtc/media/engine/webrtcmediaengine.cc @@ -31,26 +31,30 @@ class WebRtcMediaEngine2 : public CompositeMediaEngine { #endif public: - WebRtcMediaEngine2(webrtc::AudioDeviceModule* adm, - const rtc::scoped_refptr& - audio_encoder_factory, - const rtc::scoped_refptr& - audio_decoder_factory, - WebRtcVideoEncoderFactory* video_encoder_factory, - WebRtcVideoDecoderFactory* video_decoder_factory, - rtc::scoped_refptr audio_mixer) + WebRtcMediaEngine2( + webrtc::AudioDeviceModule* adm, + const rtc::scoped_refptr& + audio_encoder_factory, + const rtc::scoped_refptr& + audio_decoder_factory, + WebRtcVideoEncoderFactory* video_encoder_factory, + WebRtcVideoDecoderFactory* video_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing) #ifdef HAVE_WEBRTC_VIDEO : CompositeMediaEngine( adm, audio_encoder_factory, audio_decoder_factory, - audio_mixer){ + audio_mixer, + audio_processing){ #else : CompositeMediaEngine( adm, audio_encoder_factory, audio_decoder_factory, - audio_mixer) { + audio_mixer, + audio_processing) { #endif video_.SetExternalDecoderFactory(video_decoder_factory); video_.SetExternalEncoderFactory(video_encoder_factory); @@ -67,10 +71,11 @@ cricket::MediaEngineInterface* CreateWebRtcMediaEngine( audio_decoder_factory, cricket::WebRtcVideoEncoderFactory* video_encoder_factory, cricket::WebRtcVideoDecoderFactory* video_decoder_factory, - rtc::scoped_refptr audio_mixer) { + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing) { return new cricket::WebRtcMediaEngine2( adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory, - video_decoder_factory, audio_mixer); + video_decoder_factory, audio_mixer, audio_processing); } void DestroyWebRtcMediaEngine(cricket::MediaEngineInterface* media_engine) { @@ -89,7 +94,7 @@ MediaEngineInterface* WebRtcMediaEngineFactory::Create( return CreateWebRtcMediaEngine( adm, webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), video_encoder_factory, - video_decoder_factory, nullptr); + video_decoder_factory, nullptr, webrtc::AudioProcessing::Create()); } MediaEngineInterface* WebRtcMediaEngineFactory::Create( @@ -100,7 +105,8 @@ MediaEngineInterface* WebRtcMediaEngineFactory::Create( WebRtcVideoDecoderFactory* video_decoder_factory) { return CreateWebRtcMediaEngine( adm, webrtc::CreateBuiltinAudioEncoderFactory(), audio_decoder_factory, - video_encoder_factory, video_decoder_factory, nullptr); + video_encoder_factory, video_decoder_factory, nullptr, + webrtc::AudioProcessing::Create()); } // Used by PeerConnectionFactory to create a media engine passed into @@ -111,10 +117,12 @@ MediaEngineInterface* WebRtcMediaEngineFactory::Create( audio_decoder_factory, WebRtcVideoEncoderFactory* video_encoder_factory, WebRtcVideoDecoderFactory* video_decoder_factory, - rtc::scoped_refptr audio_mixer) { + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing) { return CreateWebRtcMediaEngine( adm, webrtc::CreateBuiltinAudioEncoderFactory(), audio_decoder_factory, - video_encoder_factory, video_decoder_factory, audio_mixer); + video_encoder_factory, video_decoder_factory, audio_mixer, + audio_processing); } MediaEngineInterface* WebRtcMediaEngineFactory::Create( @@ -125,9 +133,9 @@ MediaEngineInterface* WebRtcMediaEngineFactory::Create( audio_decoder_factory, WebRtcVideoEncoderFactory* video_encoder_factory, WebRtcVideoDecoderFactory* video_decoder_factory) { - return CreateWebRtcMediaEngine(adm, audio_encoder_factory, - audio_decoder_factory, video_encoder_factory, - video_decoder_factory, nullptr); + return CreateWebRtcMediaEngine( + adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory, + video_decoder_factory, nullptr, webrtc::AudioProcessing::Create()); } MediaEngineInterface* WebRtcMediaEngineFactory::Create( @@ -138,10 +146,11 @@ MediaEngineInterface* WebRtcMediaEngineFactory::Create( audio_decoder_factory, WebRtcVideoEncoderFactory* video_encoder_factory, WebRtcVideoDecoderFactory* video_decoder_factory, - rtc::scoped_refptr audio_mixer) { - return CreateWebRtcMediaEngine(adm, audio_encoder_factory, - audio_decoder_factory, video_encoder_factory, - video_decoder_factory, audio_mixer); + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing) { + return CreateWebRtcMediaEngine( + adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory, + video_decoder_factory, audio_mixer, audio_processing); } namespace { diff --git a/webrtc/media/engine/webrtcmediaengine.h b/webrtc/media/engine/webrtcmediaengine.h index db955d0a72..edbda66faa 100644 --- a/webrtc/media/engine/webrtcmediaengine.h +++ b/webrtc/media/engine/webrtcmediaengine.h @@ -22,6 +22,7 @@ namespace webrtc { class AudioDecoderFactory; class AudioDeviceModule; class AudioMixer; +class AudioProcessing; } namespace cricket { class WebRtcVideoDecoderFactory; @@ -59,7 +60,8 @@ class WebRtcMediaEngineFactory { audio_decoder_factory, WebRtcVideoEncoderFactory* video_encoder_factory, WebRtcVideoDecoderFactory* video_decoder_factory, - rtc::scoped_refptr audio_mixer); + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr apm); static MediaEngineInterface* Create( webrtc::AudioDeviceModule* adm, @@ -78,7 +80,8 @@ class WebRtcMediaEngineFactory { audio_decoder_factory, WebRtcVideoEncoderFactory* video_encoder_factory, WebRtcVideoDecoderFactory* video_decoder_factory, - rtc::scoped_refptr audio_mixer); + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr apm); }; // Verify that extension IDs are within 1-byte extension range and are not diff --git a/webrtc/media/engine/webrtcvoiceengine.cc b/webrtc/media/engine/webrtcvoiceengine.cc index b39be5b2b2..1f9794944e 100644 --- a/webrtc/media/engine/webrtcvoiceengine.cc +++ b/webrtc/media/engine/webrtcvoiceengine.cc @@ -163,7 +163,8 @@ rtc::Optional GetAudioNetworkAdaptorConfig( webrtc::AudioState::Config MakeAudioStateConfig( VoEWrapper* voe_wrapper, - rtc::scoped_refptr audio_mixer) { + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing) { webrtc::AudioState::Config config; config.voice_engine = voe_wrapper->engine(); if (audio_mixer) { @@ -171,6 +172,7 @@ webrtc::AudioState::Config MakeAudioStateConfig( } else { config.audio_mixer = webrtc::AudioMixerImpl::Create(); } + config.audio_processing = audio_processing; return config; } @@ -214,11 +216,13 @@ WebRtcVoiceEngine::WebRtcVoiceEngine( webrtc::AudioDeviceModule* adm, const rtc::scoped_refptr& encoder_factory, const rtc::scoped_refptr& decoder_factory, - rtc::scoped_refptr audio_mixer) + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing) : WebRtcVoiceEngine(adm, encoder_factory, decoder_factory, audio_mixer, + audio_processing, nullptr) {} WebRtcVoiceEngine::WebRtcVoiceEngine( @@ -226,11 +230,13 @@ WebRtcVoiceEngine::WebRtcVoiceEngine( const rtc::scoped_refptr& encoder_factory, const rtc::scoped_refptr& decoder_factory, rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing, VoEWrapper* voe_wrapper) : adm_(adm), encoder_factory_(encoder_factory), decoder_factory_(decoder_factory), audio_mixer_(audio_mixer), + apm_(audio_processing), voe_wrapper_(voe_wrapper) { // This may be called from any thread, so detach thread checkers. worker_thread_checker_.DetachFromThread(); @@ -238,6 +244,7 @@ WebRtcVoiceEngine::WebRtcVoiceEngine( LOG(LS_INFO) << "WebRtcVoiceEngine::WebRtcVoiceEngine"; RTC_DCHECK(decoder_factory); RTC_DCHECK(encoder_factory); + RTC_DCHECK(audio_processing); // The rest of our initialization will happen in Init. } @@ -284,8 +291,8 @@ void WebRtcVoiceEngine::Init() { webrtc::Trace::SetTraceCallback(this); webrtc::Trace::set_level_filter(kElevatedTraceFilter); LOG(LS_INFO) << webrtc::VoiceEngine::GetVersionString(); - RTC_CHECK_EQ(0, voe_wrapper_->base()->Init(adm_.get(), nullptr, - decoder_factory_)); + RTC_CHECK_EQ(0, + voe_wrapper_->base()->Init(adm_.get(), apm(), decoder_factory_)); webrtc::Trace::set_level_filter(kDefaultTraceFilter); // No ADM supplied? Get the default one from VoE. @@ -294,15 +301,12 @@ void WebRtcVoiceEngine::Init() { } RTC_DCHECK(adm_); - apm_ = voe_wrapper_->base()->audio_processing(); - RTC_DCHECK(apm_); - transmit_mixer_ = voe_wrapper_->base()->transmit_mixer(); RTC_DCHECK(transmit_mixer_); // Save the default AGC configuration settings. This must happen before // calling ApplyOptions or the default will be overwritten. - default_agc_config_ = webrtc::apm_helpers::GetAgcConfig(apm_); + default_agc_config_ = webrtc::apm_helpers::GetAgcConfig(apm()); // Set default engine options. { @@ -336,8 +340,8 @@ void WebRtcVoiceEngine::Init() { // May be null for VoE injected for testing. if (voe()->engine()) { - audio_state_ = - webrtc::AudioState::Create(MakeAudioStateConfig(voe(), audio_mixer_)); + audio_state_ = webrtc::AudioState::Create( + MakeAudioStateConfig(voe(), audio_mixer_, apm_)); } initialized_ = true; @@ -507,7 +511,7 @@ bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) { << default_agc_config_.targetLeveldBOv << "dB to -" << config.targetLeveldBOv << "dB"; } - webrtc::apm_helpers::SetAgcConfig(apm_, config); + webrtc::apm_helpers::SetAgcConfig(apm(), config); } if (options.intelligibility_enhancer) { @@ -750,8 +754,7 @@ webrtc::AudioDeviceModule* WebRtcVoiceEngine::adm() { webrtc::AudioProcessing* WebRtcVoiceEngine::apm() { RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - RTC_DCHECK(apm_); - return apm_; + return apm_.get(); } webrtc::voe::TransmitMixer* WebRtcVoiceEngine::transmit_mixer() { diff --git a/webrtc/media/engine/webrtcvoiceengine.h b/webrtc/media/engine/webrtcvoiceengine.h index cad2006365..d154d0a442 100644 --- a/webrtc/media/engine/webrtcvoiceengine.h +++ b/webrtc/media/engine/webrtcvoiceengine.h @@ -57,13 +57,15 @@ class WebRtcVoiceEngine final : public webrtc::TraceCallback { webrtc::AudioDeviceModule* adm, const rtc::scoped_refptr& encoder_factory, const rtc::scoped_refptr& decoder_factory, - rtc::scoped_refptr audio_mixer); + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing); // Dependency injection for testing. WebRtcVoiceEngine( webrtc::AudioDeviceModule* adm, const rtc::scoped_refptr& encoder_factory, const rtc::scoped_refptr& decoder_factory, rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing, VoEWrapper* voe_wrapper); ~WebRtcVoiceEngine() override; @@ -133,7 +135,7 @@ class WebRtcVoiceEngine final : public webrtc::TraceCallback { rtc::scoped_refptr decoder_factory_; rtc::scoped_refptr audio_mixer_; // Reference to the APM, owned by VoE. - webrtc::AudioProcessing* apm_ = nullptr; + rtc::scoped_refptr apm_; // Reference to the TransmitMixer, owned by VoE. webrtc::voe::TransmitMixer* transmit_mixer_ = nullptr; // The primary instance of WebRtc VoiceEngine. diff --git a/webrtc/media/engine/webrtcvoiceengine_unittest.cc b/webrtc/media/engine/webrtcvoiceengine_unittest.cc index dbee6d318a..85158d83bd 100644 --- a/webrtc/media/engine/webrtcvoiceengine_unittest.cc +++ b/webrtc/media/engine/webrtcvoiceengine_unittest.cc @@ -15,6 +15,7 @@ #include "webrtc/base/arraysize.h" #include "webrtc/base/byteorder.h" #include "webrtc/base/safe_conversions.h" +#include "webrtc/base/scoped_ref_ptr.h" #include "webrtc/call/call.h" #include "webrtc/logging/rtc_event_log/rtc_event_log.h" #include "webrtc/media/base/fakemediaengine.h" @@ -121,19 +122,21 @@ void AdmSetupExpectations(webrtc::test::MockAudioDeviceModule* adm) { TEST(WebRtcVoiceEngineTestStubLibrary, StartupShutdown) { StrictMock adm; AdmSetupExpectations(&adm); - StrictMock apm; - EXPECT_CALL(apm, ApplyConfig(testing::_)); - EXPECT_CALL(apm, SetExtraOptions(testing::_)); - EXPECT_CALL(apm, Initialize()).WillOnce(Return(0)); - EXPECT_CALL(apm, DetachAecDump()); + rtc::scoped_refptr> apm = + new rtc::RefCountedObject< + StrictMock>(); + EXPECT_CALL(*apm, ApplyConfig(testing::_)); + EXPECT_CALL(*apm, SetExtraOptions(testing::_)); + EXPECT_CALL(*apm, Initialize()).WillOnce(Return(0)); + EXPECT_CALL(*apm, DetachAecDump()); StrictMock transmit_mixer; EXPECT_CALL(transmit_mixer, EnableStereoChannelSwapping(false)); - cricket::FakeWebRtcVoiceEngine voe(&apm, &transmit_mixer); + cricket::FakeWebRtcVoiceEngine voe(&transmit_mixer); EXPECT_FALSE(voe.IsInited()); { cricket::WebRtcVoiceEngine engine( &adm, webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), - webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm, new FakeVoEWrapper(&voe)); engine.Init(); EXPECT_TRUE(voe.IsInited()); @@ -155,17 +158,22 @@ class WebRtcVoiceEngineTestFake : public testing::Test { WebRtcVoiceEngineTestFake() : WebRtcVoiceEngineTestFake("") {} explicit WebRtcVoiceEngineTestFake(const char* field_trials) - : apm_gc_(*apm_.gain_control()), apm_ec_(*apm_.echo_cancellation()), - apm_ns_(*apm_.noise_suppression()), apm_vd_(*apm_.voice_detection()), - call_(webrtc::Call::Config(&event_log_)), voe_(&apm_, &transmit_mixer_), + : apm_(new rtc::RefCountedObject< + StrictMock>()), + apm_gc_(*apm_->gain_control()), + apm_ec_(*apm_->echo_cancellation()), + apm_ns_(*apm_->noise_suppression()), + apm_vd_(*apm_->voice_detection()), + call_(webrtc::Call::Config(&event_log_)), + voe_(&transmit_mixer_), override_field_trials_(field_trials) { // AudioDeviceModule. AdmSetupExpectations(&adm_); // AudioProcessing. - EXPECT_CALL(apm_, ApplyConfig(testing::_)); - EXPECT_CALL(apm_, SetExtraOptions(testing::_)); - EXPECT_CALL(apm_, Initialize()).WillOnce(Return(0)); - EXPECT_CALL(apm_, DetachAecDump()); + EXPECT_CALL(*apm_, ApplyConfig(testing::_)); + EXPECT_CALL(*apm_, SetExtraOptions(testing::_)); + EXPECT_CALL(*apm_, Initialize()).WillOnce(Return(0)); + EXPECT_CALL(*apm_, DetachAecDump()); // Default Options. EXPECT_CALL(apm_ec_, Enable(true)).WillOnce(Return(0)); EXPECT_CALL(apm_ec_, enable_metrics(true)).WillOnce(Return(0)); @@ -186,7 +194,7 @@ class WebRtcVoiceEngineTestFake : public testing::Test { auto encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory(); auto decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory(); engine_.reset(new cricket::WebRtcVoiceEngine(&adm_, encoder_factory, - decoder_factory, nullptr, + decoder_factory, nullptr, apm_, new FakeVoEWrapper(&voe_))); engine_->Init(); send_parameters_.codecs.push_back(kPcmuCodec); @@ -196,8 +204,8 @@ class WebRtcVoiceEngineTestFake : public testing::Test { } bool SetupChannel() { - EXPECT_CALL(apm_, ApplyConfig(testing::_)); - EXPECT_CALL(apm_, SetExtraOptions(testing::_)); + EXPECT_CALL(*apm_, ApplyConfig(testing::_)); + EXPECT_CALL(*apm_, SetExtraOptions(testing::_)); channel_ = engine_->CreateChannel(&call_, cricket::MediaConfig(), cricket::AudioOptions()); return (channel_ != nullptr); @@ -217,7 +225,7 @@ class WebRtcVoiceEngineTestFake : public testing::Test { if (!channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrcX))) { return false; } - EXPECT_CALL(apm_, set_output_will_be_muted(false)); + EXPECT_CALL(*apm_, set_output_will_be_muted(false)); return channel_->SetAudioSend(kSsrcX, true, nullptr, &fake_source_); } @@ -270,26 +278,26 @@ class WebRtcVoiceEngineTestFake : public testing::Test { EXPECT_CALL(adm_, RecordingIsInitialized()).WillOnce(Return(false)); EXPECT_CALL(adm_, Recording()).WillOnce(Return(false)); EXPECT_CALL(adm_, InitRecording()).WillOnce(Return(0)); - EXPECT_CALL(apm_, ApplyConfig(testing::_)); - EXPECT_CALL(apm_, SetExtraOptions(testing::_)); + EXPECT_CALL(*apm_, ApplyConfig(testing::_)); + EXPECT_CALL(*apm_, SetExtraOptions(testing::_)); } channel_->SetSend(enable); } void SetSendParameters(const cricket::AudioSendParameters& params) { - EXPECT_CALL(apm_, ApplyConfig(testing::_)); - EXPECT_CALL(apm_, SetExtraOptions(testing::_)); + EXPECT_CALL(*apm_, ApplyConfig(testing::_)); + EXPECT_CALL(*apm_, SetExtraOptions(testing::_)); ASSERT_TRUE(channel_); EXPECT_TRUE(channel_->SetSendParameters(params)); } void SetAudioSend(uint32_t ssrc, bool enable, cricket::AudioSource* source, const cricket::AudioOptions* options = nullptr) { - EXPECT_CALL(apm_, set_output_will_be_muted(!enable)); + EXPECT_CALL(*apm_, set_output_will_be_muted(!enable)); ASSERT_TRUE(channel_); if (enable && options) { - EXPECT_CALL(apm_, ApplyConfig(testing::_)); - EXPECT_CALL(apm_, SetExtraOptions(testing::_)); + EXPECT_CALL(*apm_, ApplyConfig(testing::_)); + EXPECT_CALL(*apm_, SetExtraOptions(testing::_)); } EXPECT_TRUE(channel_->SetAudioSend(ssrc, enable, options, source)); } @@ -658,7 +666,7 @@ class WebRtcVoiceEngineTestFake : public testing::Test { protected: StrictMock adm_; - StrictMock apm_; + rtc::scoped_refptr> apm_; webrtc::test::MockGainControl& apm_gc_; webrtc::test::MockEchoCancellation& apm_ec_; webrtc::test::MockNoiseSuppression& apm_ns_; @@ -2794,8 +2802,8 @@ TEST_F(WebRtcVoiceEngineTestFake, SetOptionOverridesViaChannels) { RecordingIsInitialized()).Times(2).WillRepeatedly(Return(false)); EXPECT_CALL(adm_, Recording()).Times(2).WillRepeatedly(Return(false)); EXPECT_CALL(adm_, InitRecording()).Times(2).WillRepeatedly(Return(0)); - EXPECT_CALL(apm_, ApplyConfig(testing::_)).Times(10); - EXPECT_CALL(apm_, SetExtraOptions(testing::_)).Times(10); + EXPECT_CALL(*apm_, ApplyConfig(testing::_)).Times(10); + EXPECT_CALL(*apm_, SetExtraOptions(testing::_)).Times(10); std::unique_ptr channel1( static_cast(engine_->CreateChannel( @@ -2905,8 +2913,8 @@ TEST_F(WebRtcVoiceEngineTestFake, TestSetDscpOptions) { cricket::MediaConfig config; std::unique_ptr channel; - EXPECT_CALL(apm_, ApplyConfig(testing::_)).Times(3); - EXPECT_CALL(apm_, SetExtraOptions(testing::_)).Times(3); + EXPECT_CALL(*apm_, ApplyConfig(testing::_)).Times(3); + EXPECT_CALL(*apm_, SetExtraOptions(testing::_)).Times(3); channel.reset( engine_->CreateChannel(&call_, config, cricket::AudioOptions())); @@ -3262,9 +3270,11 @@ TEST_F(WebRtcVoiceEngineTestFake, PreservePlayoutWhenRecreateRecvStream) { TEST(WebRtcVoiceEngineTest, StartupShutdown) { // If the VoiceEngine wants to gather available codecs early, that's fine but // we never want it to create a decoder at this stage. + rtc::scoped_refptr apm = + webrtc::AudioProcessing::Create(); cricket::WebRtcVoiceEngine engine( nullptr, webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), - webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr); + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); engine.Init(); webrtc::RtcEventLogNullImpl event_log; std::unique_ptr call( @@ -3284,9 +3294,11 @@ TEST(WebRtcVoiceEngineTest, StartupShutdownWithExternalADM) { // we could enter a tight loop since the mock would return 0. EXPECT_CALL(adm, TimeUntilNextProcess()).WillRepeatedly(Return(100)); { + rtc::scoped_refptr apm = + webrtc::AudioProcessing::Create(); cricket::WebRtcVoiceEngine engine( &adm, webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), - webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr); + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); engine.Init(); webrtc::RtcEventLogNullImpl event_log; std::unique_ptr call( @@ -3302,9 +3314,11 @@ TEST(WebRtcVoiceEngineTest, StartupShutdownWithExternalADM) { TEST(WebRtcVoiceEngineTest, HasCorrectPayloadTypeMapping) { // TODO(ossu): Why are the payload types of codecs with non-static payload // type assignments checked here? It shouldn't really matter. + rtc::scoped_refptr apm = + webrtc::AudioProcessing::Create(); cricket::WebRtcVoiceEngine engine( nullptr, webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), - webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr); + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); engine.Init(); for (const cricket::AudioCodec& codec : engine.send_codecs()) { auto is_codec = [&codec](const char* name, int clockrate = 0) { @@ -3344,9 +3358,11 @@ TEST(WebRtcVoiceEngineTest, HasCorrectPayloadTypeMapping) { // Tests that VoE supports at least 32 channels TEST(WebRtcVoiceEngineTest, Has32Channels) { + rtc::scoped_refptr apm = + webrtc::AudioProcessing::Create(); cricket::WebRtcVoiceEngine engine( nullptr, webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), - webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr); + webrtc::MockAudioDecoderFactory::CreateUnusedFactory(), nullptr, apm); engine.Init(); webrtc::RtcEventLogNullImpl event_log; std::unique_ptr call( @@ -3379,9 +3395,11 @@ TEST(WebRtcVoiceEngineTest, SetRecvCodecs) { // what we sent in - though it's probably reasonable to expect so, if // SetRecvParameters returns true. // I think it will become clear once audio decoder injection is completed. + rtc::scoped_refptr apm = + webrtc::AudioProcessing::Create(); cricket::WebRtcVoiceEngine engine( nullptr, webrtc::MockAudioEncoderFactory::CreateUnusedFactory(), - webrtc::CreateBuiltinAudioDecoderFactory(), nullptr); + webrtc::CreateBuiltinAudioDecoderFactory(), nullptr, apm); engine.Init(); webrtc::RtcEventLogNullImpl event_log; std::unique_ptr call( @@ -3418,8 +3436,10 @@ TEST(WebRtcVoiceEngineTest, CollectRecvCodecs) { EXPECT_CALL(*mock_decoder_factory.get(), GetSupportedDecoders()) .WillOnce(Return(specs)); + rtc::scoped_refptr apm = + webrtc::AudioProcessing::Create(); cricket::WebRtcVoiceEngine engine(nullptr, unused_encoder_factory, - mock_decoder_factory, nullptr); + mock_decoder_factory, nullptr, apm); engine.Init(); auto codecs = engine.recv_codecs(); EXPECT_EQ(11, codecs.size()); diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index f3c55576cb..e151c43131 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -321,7 +321,8 @@ AudioProcessing* AudioProcessing::Create(const webrtc::Config& config) { AudioProcessing* AudioProcessing::Create(const webrtc::Config& config, NonlinearBeamformer* beamformer) { - AudioProcessingImpl* apm = new AudioProcessingImpl(config, beamformer); + AudioProcessingImpl* apm = + new rtc::RefCountedObject(config, beamformer); if (apm->Initialize() != kNoError) { delete apm; apm = nullptr; diff --git a/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc b/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc index ca6b7f6ff8..2ee1d51936 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc @@ -20,6 +20,7 @@ using ::testing::Invoke; using ::testing::Return; namespace webrtc { +namespace { class MockInitialize : public AudioProcessingImpl { public: @@ -30,8 +31,13 @@ class MockInitialize : public AudioProcessingImpl { int RealInitializeLocked() NO_THREAD_SAFETY_ANALYSIS { return AudioProcessingImpl::InitializeLocked(); } + + MOCK_CONST_METHOD0(AddRef, int()); + MOCK_CONST_METHOD0(Release, int()); }; +} // namespace + TEST(AudioProcessingImplTest, AudioParameterChangeTriggersInit) { webrtc::Config config; MockInitialize mock(config); diff --git a/webrtc/modules/audio_processing/audio_processing_unittest.cc b/webrtc/modules/audio_processing/audio_processing_unittest.cc index 11ce9176f3..59fbb914c5 100644 --- a/webrtc/modules/audio_processing/audio_processing_unittest.cc +++ b/webrtc/modules/audio_processing/audio_processing_unittest.cc @@ -7,7 +7,6 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ - #include #include @@ -2805,7 +2804,7 @@ TEST(ApmConfiguration, DefaultBehavior) { // the config, and that the default initial level is maintained after the // config has been applied. std::unique_ptr apm( - new AudioProcessingImpl(webrtc::Config())); + new rtc::RefCountedObject(webrtc::Config())); AudioProcessing::Config config; EXPECT_FALSE(apm->config_.level_controller.enabled); // TODO(peah): Add test for the existence of the level controller object once @@ -2835,7 +2834,7 @@ TEST(ApmConfiguration, ValidConfigBehavior) { // Verify that the initial level can be specified and is retained after the // config has been applied. std::unique_ptr apm( - new AudioProcessingImpl(webrtc::Config())); + new rtc::RefCountedObject(webrtc::Config())); AudioProcessing::Config config; config.level_controller.initial_peak_level_dbfs = -50.f; apm->ApplyConfig(config); @@ -2857,7 +2856,7 @@ TEST(ApmConfiguration, InValidConfigBehavior) { // Verify that the config is properly reset when the specified initial peak // level is too low. std::unique_ptr apm( - new AudioProcessingImpl(webrtc::Config())); + new rtc::RefCountedObject(webrtc::Config())); AudioProcessing::Config config; config.level_controller.enabled = true; config.level_controller.initial_peak_level_dbfs = -101.f; @@ -2875,7 +2874,7 @@ TEST(ApmConfiguration, InValidConfigBehavior) { // Verify that the config is properly reset when the specified initial peak // level is too high. - apm.reset(new AudioProcessingImpl(webrtc::Config())); + apm.reset(new rtc::RefCountedObject(webrtc::Config())); config = AudioProcessing::Config(); config.level_controller.enabled = true; config.level_controller.initial_peak_level_dbfs = 1.f; diff --git a/webrtc/modules/audio_processing/include/audio_processing.h b/webrtc/modules/audio_processing/include/audio_processing.h index 6a5a81d3b5..05be5feeb1 100644 --- a/webrtc/modules/audio_processing/include/audio_processing.h +++ b/webrtc/modules/audio_processing/include/audio_processing.h @@ -21,6 +21,7 @@ #include "webrtc/base/arraysize.h" #include "webrtc/base/platform_file.h" +#include "webrtc/base/refcount.h" #include "webrtc/modules/audio_processing/beamformer/array_util.h" #include "webrtc/modules/audio_processing/include/config.h" #include "webrtc/typedefs.h" @@ -233,7 +234,7 @@ struct Intelligibility { // // Close the application... // delete apm; // -class AudioProcessing { +class AudioProcessing : public rtc::RefCountInterface { public: // The struct below constitutes the new parameter scheme for the audio // processing. It is being introduced gradually and until it is fully @@ -300,7 +301,7 @@ class AudioProcessing { // Only for testing. static AudioProcessing* Create(const webrtc::Config& config, NonlinearBeamformer* beamformer); - virtual ~AudioProcessing() {} + ~AudioProcessing() override {} // Initializes internal states, while retaining all user settings. This // should be called before beginning to process a new audio stream. However, diff --git a/webrtc/ortc/BUILD.gn b/webrtc/ortc/BUILD.gn index fd8631c438..b6a2cc9459 100644 --- a/webrtc/ortc/BUILD.gn +++ b/webrtc/ortc/BUILD.gn @@ -41,6 +41,7 @@ rtc_static_library("ortc") { "../logging:rtc_event_log_api", "../media:rtc_media", "../media:rtc_media_base", + "../modules/audio_processing:audio_processing", "../p2p:rtc_p2p", "../pc:libjingle_peerconnection", "../pc:rtc_pc", diff --git a/webrtc/ortc/DEPS b/webrtc/ortc/DEPS index de152dd01a..4e182440fb 100644 --- a/webrtc/ortc/DEPS +++ b/webrtc/ortc/DEPS @@ -5,6 +5,7 @@ include_rules = [ "+webrtc/logging/rtc_event_log", "+webrtc/media", "+webrtc/modules/audio_coding", + "+webrtc/modules/audio_processing", "+webrtc/p2p", "+webrtc/pc", diff --git a/webrtc/ortc/ortcfactory.cc b/webrtc/ortc/ortcfactory.cc index bf34f134fb..612e55d904 100644 --- a/webrtc/ortc/ortcfactory.cc +++ b/webrtc/ortc/ortcfactory.cc @@ -27,6 +27,7 @@ #include "webrtc/base/logging.h" #include "webrtc/logging/rtc_event_log/rtc_event_log.h" #include "webrtc/media/base/mediaconstants.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/ortc/ortcrtpreceiveradapter.h" #include "webrtc/ortc/ortcrtpsenderadapter.h" #include "webrtc/ortc/rtpparametersconversion.h" @@ -544,9 +545,9 @@ OrtcFactory::CreateMediaEngine_w() { // Note that |adm_| may be null, in which case the platform-specific default // AudioDeviceModule will be used. return std::unique_ptr( - cricket::WebRtcMediaEngineFactory::Create(adm_, audio_encoder_factory_, - audio_decoder_factory_, nullptr, - nullptr, nullptr)); + cricket::WebRtcMediaEngineFactory::Create( + adm_, audio_encoder_factory_, audio_decoder_factory_, nullptr, + nullptr, nullptr, webrtc::AudioProcessing::Create())); } } // namespace webrtc diff --git a/webrtc/pc/BUILD.gn b/webrtc/pc/BUILD.gn index 60d76ed239..5190a93cf5 100644 --- a/webrtc/pc/BUILD.gn +++ b/webrtc/pc/BUILD.gn @@ -203,6 +203,7 @@ rtc_static_library("create_pc_factory") { "../logging:rtc_event_log_api", "../media:rtc_audio_video", "../modules/audio_device:audio_device", + "../modules/audio_processing:audio_processing", ] configs += [ ":libjingle_peerconnection_warnings_config" ] diff --git a/webrtc/pc/DEPS b/webrtc/pc/DEPS index 4468e6ba10..d77d279769 100644 --- a/webrtc/pc/DEPS +++ b/webrtc/pc/DEPS @@ -8,6 +8,7 @@ include_rules = [ "+webrtc/logging/rtc_event_log", "+webrtc/media", "+webrtc/modules/audio_device", + "+webrtc/modules/audio_processing", "+webrtc/modules/rtp_rtcp", "+webrtc/modules/video_coding", "+webrtc/modules/video_render", diff --git a/webrtc/pc/createpeerconnectionfactory.cc b/webrtc/pc/createpeerconnectionfactory.cc index 9370c0b9e1..c29ef52d31 100644 --- a/webrtc/pc/createpeerconnectionfactory.cc +++ b/webrtc/pc/createpeerconnectionfactory.cc @@ -17,6 +17,7 @@ #include "webrtc/call/callfactoryinterface.h" #include "webrtc/logging/rtc_event_log/rtc_event_log_factory_interface.h" #include "webrtc/media/engine/webrtcmediaengine.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" namespace webrtc { @@ -53,7 +54,8 @@ CreatePeerConnectionFactoryWithAudioMixer( std::unique_ptr media_engine( cricket::WebRtcMediaEngineFactory::Create( default_adm, audio_encoder_factory, audio_decoder_factory, - video_encoder_factory, video_decoder_factory, audio_mixer)); + video_encoder_factory, video_decoder_factory, audio_mixer, + AudioProcessing::Create())); std::unique_ptr call_factory = CreateCallFactory(); diff --git a/webrtc/pc/peerconnectioninterface_unittest.cc b/webrtc/pc/peerconnectioninterface_unittest.cc index 1229f4057c..c36460f73b 100644 --- a/webrtc/pc/peerconnectioninterface_unittest.cc +++ b/webrtc/pc/peerconnectioninterface_unittest.cc @@ -30,6 +30,7 @@ #include "webrtc/media/base/fakevideocapturer.h" #include "webrtc/media/engine/webrtcmediaengine.h" #include "webrtc/media/sctp/sctptransportinternal.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/p2p/base/fakeportallocator.h" #include "webrtc/pc/audiotrack.h" #include "webrtc/pc/mediasession.h" @@ -650,7 +651,7 @@ class PeerConnectionFactoryForTest : public webrtc::PeerConnectionFactory { auto media_engine = std::unique_ptr( cricket::WebRtcMediaEngineFactory::Create( nullptr, audio_encoder_factory, audio_decoder_factory, nullptr, - nullptr, nullptr)); + nullptr, nullptr, webrtc::AudioProcessing::Create())); std::unique_ptr call_factory = webrtc::CreateCallFactory(); diff --git a/webrtc/sdk/android/BUILD.gn b/webrtc/sdk/android/BUILD.gn index a2c0dd4b03..689d6cfb40 100644 --- a/webrtc/sdk/android/BUILD.gn +++ b/webrtc/sdk/android/BUILD.gn @@ -183,6 +183,7 @@ rtc_static_library("media_jni") { "//webrtc/call:call_interfaces", "//webrtc/logging:rtc_event_log_api", "//webrtc/media:rtc_audio_video", + "//webrtc/modules/audio_processing:audio_processing", ] if (is_clang) { @@ -325,8 +326,8 @@ dist_jar("libwebrtc") { deps = [ ":libjingle_peerconnection_java", ":libjingle_peerconnection_metrics_default_java", - "//webrtc/rtc_base:base_java", "//webrtc/modules/audio_device:audio_device_java", + "//webrtc/rtc_base:base_java", ] } @@ -414,8 +415,8 @@ android_library("libjingle_peerconnection_java") { ] deps = [ - "//webrtc/rtc_base:base_java", "//webrtc/modules/audio_device:audio_device_java", + "//webrtc/rtc_base:base_java", ] } diff --git a/webrtc/sdk/android/src/jni/DEPS b/webrtc/sdk/android/src/jni/DEPS index 803232c0b6..ed8a8f4601 100644 --- a/webrtc/sdk/android/src/jni/DEPS +++ b/webrtc/sdk/android/src/jni/DEPS @@ -7,6 +7,7 @@ include_rules = [ "+webrtc/logging/rtc_event_log/rtc_event_log_factory_interface.h", "+webrtc/media/base", "+webrtc/media/engine", + "+webrtc/modules/audio_processing/include/audio_processing.h", "+webrtc/modules/utility/include/jvm_android.h", "+webrtc/modules/video_coding/include/video_codec_interface.h", "+webrtc/modules/video_coding/utility/vp8_header_parser.h", diff --git a/webrtc/sdk/android/src/jni/media_jni.cc b/webrtc/sdk/android/src/jni/media_jni.cc index 0bc2737eb0..fdcdc0fcac 100644 --- a/webrtc/sdk/android/src/jni/media_jni.cc +++ b/webrtc/sdk/android/src/jni/media_jni.cc @@ -12,6 +12,7 @@ #include "webrtc/call/callfactoryinterface.h" #include "webrtc/logging/rtc_event_log/rtc_event_log_factory_interface.h" #include "webrtc/media/engine/webrtcmediaengine.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" namespace webrtc_jni { @@ -34,7 +35,7 @@ cricket::MediaEngineInterface* CreateMediaEngine( rtc::scoped_refptr audio_mixer) { return cricket::WebRtcMediaEngineFactory::Create( adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory, - video_decoder_factory, audio_mixer); + video_decoder_factory, audio_mixer, webrtc::AudioProcessing::Create()); } } // namespace webrtc_jni diff --git a/webrtc/test/call_test.cc b/webrtc/test/call_test.cc index 6e665cf6ed..638e70483a 100644 --- a/webrtc/test/call_test.cc +++ b/webrtc/test/call_test.cc @@ -56,10 +56,13 @@ void CallTest::RunBaseTest(BaseTest* test) { CreateFakeAudioDevices(test->CreateCapturer(), test->CreateRenderer()); test->OnFakeAudioDevicesCreated(fake_send_audio_device_.get(), fake_recv_audio_device_.get()); + apm_send_ = AudioProcessing::Create(); + apm_recv_ = AudioProcessing::Create(); CreateVoiceEngines(); AudioState::Config audio_state_config; audio_state_config.voice_engine = voe_send_.voice_engine; audio_state_config.audio_mixer = AudioMixerImpl::Create(); + audio_state_config.audio_processing = apm_send_; send_config.audio_state = AudioState::Create(audio_state_config); } CreateSenderCall(send_config); @@ -69,6 +72,7 @@ void CallTest::RunBaseTest(BaseTest* test) { AudioState::Config audio_state_config; audio_state_config.voice_engine = voe_recv_.voice_engine; audio_state_config.audio_mixer = AudioMixerImpl::Create(); + audio_state_config.audio_processing = apm_recv_; recv_config.audio_state = AudioState::Create(audio_state_config); } CreateReceiverCall(recv_config); @@ -378,8 +382,8 @@ void CallTest::DestroyStreams() { void CallTest::CreateVoiceEngines() { voe_send_.voice_engine = VoiceEngine::Create(); voe_send_.base = VoEBase::GetInterface(voe_send_.voice_engine); - EXPECT_EQ(0, voe_send_.base->Init(fake_send_audio_device_.get(), nullptr, - decoder_factory_)); + EXPECT_EQ(0, voe_send_.base->Init(fake_send_audio_device_.get(), + apm_send_.get(), decoder_factory_)); VoEBase::ChannelConfig config; config.enable_voice_pacing = true; voe_send_.channel_id = voe_send_.base->CreateChannel(config); @@ -387,8 +391,8 @@ void CallTest::CreateVoiceEngines() { voe_recv_.voice_engine = VoiceEngine::Create(); voe_recv_.base = VoEBase::GetInterface(voe_recv_.voice_engine); - EXPECT_EQ(0, voe_recv_.base->Init(fake_recv_audio_device_.get(), nullptr, - decoder_factory_)); + EXPECT_EQ(0, voe_recv_.base->Init(fake_recv_audio_device_.get(), + apm_recv_.get(), decoder_factory_)); voe_recv_.channel_id = voe_recv_.base->CreateChannel(); EXPECT_GE(voe_recv_.channel_id, 0); } diff --git a/webrtc/test/call_test.h b/webrtc/test/call_test.h index 39df343331..e7b75d6671 100644 --- a/webrtc/test/call_test.h +++ b/webrtc/test/call_test.h @@ -146,6 +146,8 @@ class CallTest : public ::testing::Test { VoiceEngineState voe_send_; VoiceEngineState voe_recv_; + rtc::scoped_refptr apm_send_; + rtc::scoped_refptr apm_recv_; // The audio devices must outlive the voice engines. std::unique_ptr fake_send_audio_device_; diff --git a/webrtc/test/mock_voice_engine.h b/webrtc/test/mock_voice_engine.h index 7655d3b844..e8eb5a2634 100644 --- a/webrtc/test/mock_voice_engine.h +++ b/webrtc/test/mock_voice_engine.h @@ -15,7 +15,6 @@ #include "webrtc/modules/audio_device/include/mock_audio_device.h" #include "webrtc/modules/audio_device/include/mock_audio_transport.h" -#include "webrtc/modules/audio_processing/include/mock_audio_processing.h" #include "webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h" #include "webrtc/test/gmock.h" #include "webrtc/test/mock_voe_channel_proxy.h" @@ -68,8 +67,6 @@ class MockVoiceEngine : public VoiceEngineImpl { .WillByDefault(testing::Return(1000)); ON_CALL(*this, audio_device_module()) .WillByDefault(testing::Return(&mock_audio_device_)); - ON_CALL(*this, audio_processing()) - .WillByDefault(testing::Return(&mock_audio_processing_)); ON_CALL(*this, audio_transport()) .WillByDefault(testing::Return(&mock_audio_transport_)); } @@ -102,9 +99,8 @@ class MockVoiceEngine : public VoiceEngineImpl { MOCK_METHOD3( Init, int(AudioDeviceModule* external_adm, - AudioProcessing* audioproc, + AudioProcessing* external_apm, const rtc::scoped_refptr& decoder_factory)); - MOCK_METHOD0(audio_processing, AudioProcessing*()); MOCK_METHOD0(audio_device_module, AudioDeviceModule*()); MOCK_METHOD0(transmit_mixer, voe::TransmitMixer*()); MOCK_METHOD0(Terminate, int()); @@ -257,7 +253,6 @@ class MockVoiceEngine : public VoiceEngineImpl { std::map> mock_rtp_rtcps_; MockAudioDeviceModule mock_audio_device_; - MockAudioProcessing mock_audio_processing_; MockAudioTransport mock_audio_transport_; }; } // namespace test diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index 37b47867d3..3f0465d361 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -79,12 +79,13 @@ struct VoiceEngineState { int receive_channel_id; }; -void CreateVoiceEngine(VoiceEngineState* voe, - rtc::scoped_refptr - decoder_factory) { +void CreateVoiceEngine( + VoiceEngineState* voe, + webrtc::AudioProcessing* apm, + rtc::scoped_refptr decoder_factory) { voe->voice_engine = webrtc::VoiceEngine::Create(); voe->base = webrtc::VoEBase::GetInterface(voe->voice_engine); - EXPECT_EQ(0, voe->base->Init(nullptr, nullptr, decoder_factory)); + EXPECT_EQ(0, voe->base->Init(nullptr, apm, decoder_factory)); webrtc::VoEBase::ChannelConfig config; config.enable_voice_pacing = true; voe->send_channel_id = voe->base->CreateChannel(config); @@ -1849,11 +1850,15 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { call_config.bitrate_config = params_.call.call_bitrate_config; ::VoiceEngineState voe; + rtc::scoped_refptr audio_processing( + webrtc::AudioProcessing::Create()); + if (params_.audio.enabled) { - CreateVoiceEngine(&voe, decoder_factory_); + CreateVoiceEngine(&voe, audio_processing.get(), decoder_factory_); AudioState::Config audio_state_config; audio_state_config.voice_engine = voe.voice_engine; audio_state_config.audio_mixer = AudioMixerImpl::Create(); + audio_state_config.audio_processing = audio_processing; call_config.audio_state = AudioState::Create(audio_state_config); } diff --git a/webrtc/voice_engine/include/voe_base.h b/webrtc/voice_engine/include/voe_base.h index dec07d6409..05fe4550c4 100644 --- a/webrtc/voice_engine/include/voe_base.h +++ b/webrtc/voice_engine/include/voe_base.h @@ -124,18 +124,19 @@ class WEBRTC_DLLEXPORT VoEBase { // modules: // - The Audio Device Module (ADM) which implements all the audio layer // functionality in a separate (reference counted) module. - // - The AudioProcessing module handles capture-side processing. VoiceEngine - // takes ownership of this object. + // - The AudioProcessing module handles capture-side processing. // - An AudioDecoderFactory - used to create audio decoders. - // If NULL is passed for any of these, VoiceEngine will create its own. - // Returns -1 in case of an error, 0 otherwise. + // If NULL is passed for either of ADM or AudioDecoderFactory, VoiceEngine + // will create its own. Returns -1 in case of an error, 0 otherwise. // TODO(ajm): Remove default NULLs. virtual int Init(AudioDeviceModule* external_adm = NULL, - AudioProcessing* audioproc = NULL, + AudioProcessing* external_apm = nullptr, const rtc::scoped_refptr& decoder_factory = nullptr) = 0; - // Returns NULL before Init() is called. + // Returns null before Init() is called. + // TODO(peah): Remove this when downstream dependencies have properly been + // resolved. virtual AudioProcessing* audio_processing() = 0; // This method is WIP - DO NOT USE! diff --git a/webrtc/voice_engine/shared_data.cc b/webrtc/voice_engine/shared_data.cc index e77f52a956..464374ef31 100644 --- a/webrtc/voice_engine/shared_data.cc +++ b/webrtc/voice_engine/shared_data.cc @@ -61,7 +61,6 @@ void SharedData::set_audio_device( } void SharedData::set_audio_processing(AudioProcessing* audioproc) { - audioproc_.reset(audioproc); _transmitMixerPtr->SetAudioProcessingModule(audioproc); _outputMixerPtr->SetAudioProcessingModule(audioproc); } diff --git a/webrtc/voice_engine/shared_data.h b/webrtc/voice_engine/shared_data.h index 1a910407a2..951a85189d 100644 --- a/webrtc/voice_engine/shared_data.h +++ b/webrtc/voice_engine/shared_data.h @@ -43,7 +43,6 @@ public: AudioDeviceModule* audio_device() { return _audioDevicePtr.get(); } void set_audio_device( const rtc::scoped_refptr& audio_device); - AudioProcessing* audio_processing() { return audioproc_.get(); } void set_audio_processing(AudioProcessing* audio_processing); TransmitMixer* transmit_mixer() { return _transmitMixerPtr; } OutputMixer* output_mixer() { return _outputMixerPtr; } @@ -69,7 +68,6 @@ protected: rtc::scoped_refptr _audioDevicePtr; OutputMixer* _outputMixerPtr; TransmitMixer* _transmitMixerPtr; - std::unique_ptr audioproc_; std::unique_ptr _moduleProcessThreadPtr; // |encoder_queue| is defined last to ensure all pending tasks are cancelled // and deleted before any other members. diff --git a/webrtc/voice_engine/test/auto_test/fakes/conference_transport.cc b/webrtc/voice_engine/test/auto_test/fakes/conference_transport.cc index 6fbf5f1297..f7510c7972 100644 --- a/webrtc/voice_engine/test/auto_test/fakes/conference_transport.cc +++ b/webrtc/voice_engine/test/auto_test/fakes/conference_transport.cc @@ -55,6 +55,9 @@ ConferenceTransport::ConferenceTransport() local_network_ = webrtc::VoENetwork::GetInterface(local_voe_); local_rtp_rtcp_ = webrtc::VoERTP_RTCP::GetInterface(local_voe_); + local_apm_ = webrtc::AudioProcessing::Create(); + local_base_->Init(nullptr, local_apm_.get(), nullptr); + // In principle, we can use one VoiceEngine to achieve the same goal. Well, in // here, we use two engines to make it more like reality. remote_voe_ = webrtc::VoiceEngine::Create(); @@ -64,7 +67,9 @@ ConferenceTransport::ConferenceTransport() remote_rtp_rtcp_ = webrtc::VoERTP_RTCP::GetInterface(remote_voe_); remote_file_ = webrtc::VoEFile::GetInterface(remote_voe_); - EXPECT_EQ(0, local_base_->Init()); + remote_apm_ = webrtc::AudioProcessing::Create(); + remote_base_->Init(nullptr, remote_apm_.get(), nullptr); + local_sender_ = local_base_->CreateChannel(); static_cast(local_voe_) ->GetChannelProxy(local_sender_) @@ -74,10 +79,8 @@ ConferenceTransport::ConferenceTransport() EXPECT_EQ(0, local_rtp_rtcp_-> SetSendAudioLevelIndicationStatus(local_sender_, true, kAudioLevelHeaderId)); - EXPECT_EQ(0, local_base_->StartSend(local_sender_)); - EXPECT_EQ(0, remote_base_->Init()); reflector_ = remote_base_->CreateChannel(); static_cast(remote_voe_) ->GetChannelProxy(reflector_) diff --git a/webrtc/voice_engine/test/auto_test/fakes/conference_transport.h b/webrtc/voice_engine/test/auto_test/fakes/conference_transport.h index 37919e1e99..19fb15f619 100644 --- a/webrtc/voice_engine/test/auto_test/fakes/conference_transport.h +++ b/webrtc/voice_engine/test/auto_test/fakes/conference_transport.h @@ -147,6 +147,7 @@ class ConferenceTransport: public webrtc::Transport { webrtc::VoEBase* local_base_; webrtc::VoERTP_RTCP* local_rtp_rtcp_; webrtc::VoENetwork* local_network_; + rtc::scoped_refptr local_apm_; webrtc::VoiceEngine* remote_voe_; webrtc::VoEBase* remote_base_; @@ -154,7 +155,7 @@ class ConferenceTransport: public webrtc::Transport { webrtc::VoERTP_RTCP* remote_rtp_rtcp_; webrtc::VoENetwork* remote_network_; webrtc::VoEFile* remote_file_; - + rtc::scoped_refptr remote_apm_; LoudestFilter loudest_filter_; const std::unique_ptr rtp_header_parser_; diff --git a/webrtc/voice_engine/test/auto_test/standard/codec_test.cc b/webrtc/voice_engine/test/auto_test/standard/codec_test.cc index 2b5d12d8fd..27c3b0e869 100644 --- a/webrtc/voice_engine/test/auto_test/standard/codec_test.cc +++ b/webrtc/voice_engine/test/auto_test/standard/codec_test.cc @@ -19,6 +19,8 @@ class CodecTest : public AfterStreamingFixture { protected: void SetUp() { memset(&codec_instance_, 0, sizeof(codec_instance_)); + apm_ = webrtc::AudioProcessing::Create(); + voe_base_->Init(nullptr, apm_.get(), nullptr); } void SetArbitrarySendCodec() { @@ -27,6 +29,7 @@ class CodecTest : public AfterStreamingFixture { EXPECT_EQ(0, voe_codec_->SetSendCodec(channel_, codec_instance_)); } + rtc::scoped_refptr apm_; webrtc::CodecInst codec_instance_; }; diff --git a/webrtc/voice_engine/voe_base_impl.cc b/webrtc/voice_engine/voe_base_impl.cc index 1ddf53ca7b..196c6a555f 100644 --- a/webrtc/voice_engine/voe_base_impl.cc +++ b/webrtc/voice_engine/voe_base_impl.cc @@ -225,8 +225,10 @@ int VoEBaseImpl::DeRegisterVoiceEngineObserver() { int VoEBaseImpl::Init( AudioDeviceModule* external_adm, - AudioProcessing* audioproc, + AudioProcessing* external_apm, const rtc::scoped_refptr& decoder_factory) { + // TODO(peah): Add a DCHECK for external_apm when downstream dependencies + // have properly been resolved. rtc::CritScope cs(shared_->crit_sec()); WebRtcSpl_Init(); if (shared_->statistics().Initialized()) { @@ -337,33 +339,43 @@ int VoEBaseImpl::Init( "Init() failed to set mono/stereo recording mode"); } - if (!audioproc) { - audioproc = AudioProcessing::Create(); - if (!audioproc) { + // TODO(peah): Remove this when upstream dependencies have properly been + // resolved. + AudioProcessing* apm = nullptr; + if (!external_apm) { + audio_processing_ = AudioProcessing::Create(); + if (!audio_processing_) { + // This can only happen if there are problems allocating the dynamic + // memory in the Create() call. LOG(LS_ERROR) << "Failed to create AudioProcessing."; shared_->SetLastError(VE_NO_MEMORY); return -1; } + apm = audio_processing_.get(); + } else { + apm = external_apm; } - shared_->set_audio_processing(audioproc); + + shared_->set_audio_processing(apm); // Set the error state for any failures in this block. shared_->SetLastError(VE_APM_ERROR); // Configure AudioProcessing components. - if (audioproc->high_pass_filter()->Enable(true) != 0) { + // TODO(peah): Move this initialization to webrtcvoiceengine.cc. + if (apm->high_pass_filter()->Enable(true) != 0) { LOG_F(LS_ERROR) << "Failed to enable high pass filter."; return -1; } - if (audioproc->echo_cancellation()->enable_drift_compensation(false) != 0) { + if (apm->echo_cancellation()->enable_drift_compensation(false) != 0) { LOG_F(LS_ERROR) << "Failed to disable drift compensation."; return -1; } - if (audioproc->noise_suppression()->set_level(kDefaultNsMode) != 0) { + if (apm->noise_suppression()->set_level(kDefaultNsMode) != 0) { LOG_F(LS_ERROR) << "Failed to set noise suppression level: " << kDefaultNsMode; return -1; } - GainControl* agc = audioproc->gain_control(); + GainControl* agc = apm->gain_control(); if (agc->set_analog_level_limits(kMinVolumeLevel, kMaxVolumeLevel) != 0) { LOG_F(LS_ERROR) << "Failed to set analog level limits with minimum: " << kMinVolumeLevel << " and maximum: " << kMaxVolumeLevel; @@ -687,9 +699,7 @@ int32_t VoEBaseImpl::TerminateInternal() { shared_->set_audio_device(nullptr); } - if (shared_->audio_processing()) { - shared_->set_audio_processing(nullptr); - } + shared_->set_audio_processing(nullptr); return shared_->statistics().SetUnInitialized(); } diff --git a/webrtc/voice_engine/voe_base_impl.h b/webrtc/voice_engine/voe_base_impl.h index b9f3282596..6d7a086926 100644 --- a/webrtc/voice_engine/voe_base_impl.h +++ b/webrtc/voice_engine/voe_base_impl.h @@ -28,12 +28,15 @@ class VoEBaseImpl : public VoEBase, int RegisterVoiceEngineObserver(VoiceEngineObserver& observer) override; int DeRegisterVoiceEngineObserver() override; - int Init(AudioDeviceModule* external_adm = nullptr, - AudioProcessing* audioproc = nullptr, - const rtc::scoped_refptr& decoder_factory = - nullptr) override; + int Init( + AudioDeviceModule* external_adm, + AudioProcessing* external_apm, + const rtc::scoped_refptr& decoder_factory) override; AudioProcessing* audio_processing() override { - return shared_->audio_processing(); + // TODO(peah): Remove this when downstream dependencies have properly been + // resolved. + RTC_DCHECK(audio_processing_); + return audio_processing_.get(); } AudioDeviceModule* audio_device_module() override { return shared_->audio_device(); @@ -121,6 +124,10 @@ class VoEBaseImpl : public VoEBase, rtc::CriticalSection callbackCritSect_; rtc::scoped_refptr decoder_factory_; + // TODO(peah): Remove this when upstream dependencies have properly been + // resolved. + rtc::scoped_refptr audio_processing_; + AudioFrame audioFrame_; voe::SharedData* shared_; }; diff --git a/webrtc/voice_engine/voe_base_unittest.cc b/webrtc/voice_engine/voe_base_unittest.cc index a300f3b75e..55f9634a83 100644 --- a/webrtc/voice_engine/voe_base_unittest.cc +++ b/webrtc/voice_engine/voe_base_unittest.cc @@ -21,17 +21,8 @@ namespace webrtc { class VoEBaseTest : public VoiceEngineFixture {}; -TEST_F(VoEBaseTest, InitWithExternalAudioDeviceAndAudioProcessing) { - AudioProcessing* audioproc = AudioProcessing::Create(); - EXPECT_EQ(0, base_->Init(&adm_, audioproc)); - EXPECT_EQ(audioproc, base_->audio_processing()); - EXPECT_EQ(0, base_->LastError()); -} - TEST_F(VoEBaseTest, InitWithExternalAudioDevice) { - EXPECT_EQ(nullptr, base_->audio_processing()); - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); - EXPECT_NE(nullptr, base_->audio_processing()); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get())); EXPECT_EQ(0, base_->LastError()); } @@ -41,15 +32,14 @@ TEST_F(VoEBaseTest, CreateChannelBeforeInitShouldFail) { } TEST_F(VoEBaseTest, CreateChannelAfterInit) { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); int channelID = base_->CreateChannel(); EXPECT_NE(channelID, -1); EXPECT_EQ(0, base_->DeleteChannel(channelID)); } TEST_F(VoEBaseTest, AssociateSendChannel) { - AudioProcessing* audioproc = AudioProcessing::Create(); - EXPECT_EQ(0, base_->Init(&adm_, audioproc)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get())); const int channel_1 = base_->CreateChannel(); diff --git a/webrtc/voice_engine/voe_codec_unittest.cc b/webrtc/voice_engine/voe_codec_unittest.cc index af04a61308..7d277852a0 100644 --- a/webrtc/voice_engine/voe_codec_unittest.cc +++ b/webrtc/voice_engine/voe_codec_unittest.cc @@ -13,6 +13,7 @@ #include "webrtc/voice_engine/include/voe_codec.h" #include "webrtc/modules/audio_device/include/fake_audio_device.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/test/gtest.h" #include "webrtc/voice_engine/include/voe_base.h" #include "webrtc/voice_engine/voice_engine_defines.h" @@ -79,8 +80,9 @@ TEST(VoECodecInst, RememberOpusDtxAfterSettingChange) { VoEBase* base(VoEBase::GetInterface(voe)); VoECodec* voe_codec(VoECodec::GetInterface(voe)); std::unique_ptr adm(new FakeAudioDeviceModule); + std::unique_ptr apm(AudioProcessing::Create()); - base->Init(adm.get()); + base->Init(adm.get(), apm.get()); CodecInst codec = {111, "opus", 48000, 960, 1, 32000}; diff --git a/webrtc/voice_engine/voe_network_unittest.cc b/webrtc/voice_engine/voe_network_unittest.cc index ef6e4fb5f3..d3d1f8041f 100644 --- a/webrtc/voice_engine/voe_network_unittest.cc +++ b/webrtc/voice_engine/voe_network_unittest.cc @@ -32,7 +32,7 @@ static const int kNonExistingChannel = 1234; class VoENetworkTest : public VoiceEngineFixture { protected: int CreateChannelAndRegisterExternalTransport() { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); int channelID = base_->CreateChannel(); EXPECT_NE(channelID, -1); EXPECT_EQ(0, network_->RegisterExternalTransport(channelID, transport_)); @@ -47,19 +47,19 @@ TEST_F(VoENetworkTest, RegisterAndDeRegisterExternalTransport) { TEST_F(VoENetworkTest, RegisterExternalTransportOnNonExistingChannelShouldFail) { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); EXPECT_NE( 0, network_->RegisterExternalTransport(kNonExistingChannel, transport_)); } TEST_F(VoENetworkTest, DeRegisterExternalTransportOnNonExistingChannelShouldFail) { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); EXPECT_NE(0, network_->DeRegisterExternalTransport(kNonExistingChannel)); } TEST_F(VoENetworkTest, DeRegisterExternalTransportBeforeRegister) { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); int channelID = base_->CreateChannel(); EXPECT_NE(channelID, -1); EXPECT_EQ(0, network_->DeRegisterExternalTransport(channelID)); @@ -72,13 +72,13 @@ TEST_F(VoENetworkTest, ReceivedRTPPacketWithJunkDataShouldFail) { } TEST_F(VoENetworkTest, ReceivedRTPPacketOnNonExistingChannelShouldFail) { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); EXPECT_EQ(-1, network_->ReceivedRTPPacket(kNonExistingChannel, kPacket, sizeof(kPacket))); } TEST_F(VoENetworkTest, ReceivedRTPPacketOnChannelWithoutTransportShouldFail) { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); int channelID = base_->CreateChannel(); EXPECT_NE(channelID, -1); EXPECT_EQ(-1, @@ -105,13 +105,13 @@ TEST_F(VoENetworkTest, ReceivedRTCPPacketWithJunkDataShouldFail) { } TEST_F(VoENetworkTest, ReceivedRTCPPacketOnNonExistingChannelShouldFail) { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); EXPECT_EQ(-1, network_->ReceivedRTCPPacket(kNonExistingChannel, kPacket, sizeof(kPacket))); } TEST_F(VoENetworkTest, ReceivedRTCPPacketOnChannelWithoutTransportShouldFail) { - EXPECT_EQ(0, base_->Init(&adm_, nullptr)); + EXPECT_EQ(0, base_->Init(&adm_, apm_.get(), nullptr)); int channelID = base_->CreateChannel(); EXPECT_NE(channelID, -1); EXPECT_EQ(-1, diff --git a/webrtc/voice_engine/voice_engine_fixture.cc b/webrtc/voice_engine/voice_engine_fixture.cc index faac8dd48c..3cb613d0ba 100644 --- a/webrtc/voice_engine/voice_engine_fixture.cc +++ b/webrtc/voice_engine/voice_engine_fixture.cc @@ -9,6 +9,7 @@ */ #include "webrtc/voice_engine/voice_engine_fixture.h" +#include "webrtc/modules/audio_processing/include/mock_audio_processing.h" namespace webrtc { @@ -19,6 +20,7 @@ VoiceEngineFixture::VoiceEngineFixture() EXPECT_NE(nullptr, base_); EXPECT_NE(nullptr, network_); EXPECT_EQ(0, base_->RegisterVoiceEngineObserver(observer_)); + apm_ = new rtc::RefCountedObject(); } VoiceEngineFixture::~VoiceEngineFixture() { diff --git a/webrtc/voice_engine/voice_engine_fixture.h b/webrtc/voice_engine/voice_engine_fixture.h index 4f4eaa865e..9f2f316567 100644 --- a/webrtc/voice_engine/voice_engine_fixture.h +++ b/webrtc/voice_engine/voice_engine_fixture.h @@ -9,6 +9,7 @@ */ #include "webrtc/modules/audio_device/include/fake_audio_device.h" +#include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/test/gtest.h" #include "webrtc/test/mock_transport.h" #include "webrtc/voice_engine/include/voe_base.h" @@ -28,6 +29,7 @@ class VoiceEngineFixture : public ::testing::Test { MockVoEObserver observer_; FakeAudioDeviceModule adm_; MockTransport transport_; + rtc::scoped_refptr apm_; }; } // namespace webrtc