diff --git a/webrtc/audio/BUILD.gn b/webrtc/audio/BUILD.gn index e89350e89a..8200448401 100644 --- a/webrtc/audio/BUILD.gn +++ b/webrtc/audio/BUILD.gn @@ -32,6 +32,7 @@ rtc_static_library("audio") { "../api:audio_mixer_api", "../api:call_api", "../base:rtc_base_approved", + "../common_audio", "../modules/audio_device", "../modules/audio_processing", "../system_wrappers", @@ -48,8 +49,10 @@ if (rtc_include_tests) { ] deps = [ ":audio", + "../api:mock_audio_mixer", "../base:rtc_base_approved", "../modules/audio_device:mock_audio_device", + "../modules/audio_mixer:audio_mixer_impl", "../test:test_common", "//testing/gmock", "//testing/gtest", diff --git a/webrtc/audio/DEPS b/webrtc/audio/DEPS index 6e6259b9e5..f383c1456b 100644 --- a/webrtc/audio/DEPS +++ b/webrtc/audio/DEPS @@ -1,6 +1,7 @@ include_rules = [ "+webrtc/base", "+webrtc/call", + "+webrtc/common_audio/resampler", "+webrtc/logging/rtc_event_log", "+webrtc/modules/audio_coding/codecs/mock", "+webrtc/modules/audio_device", diff --git a/webrtc/audio/audio_receive_stream.cc b/webrtc/audio/audio_receive_stream.cc index 5a8efc223e..81ffdade5c 100644 --- a/webrtc/audio/audio_receive_stream.cc +++ b/webrtc/audio/audio_receive_stream.cc @@ -138,9 +138,11 @@ AudioReceiveStream::AudioReceiveStream( } AudioReceiveStream::~AudioReceiveStream() { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK_RUN_ON(&thread_checker_); LOG(LS_INFO) << "~AudioReceiveStream: " << config_.ToString(); - Stop(); + if (playing_) { + Stop(); + } channel_proxy_->DisassociateSendChannel(); channel_proxy_->DeRegisterExternalTransport(); channel_proxy_->ResetCongestionControlObjects(); @@ -151,22 +153,39 @@ AudioReceiveStream::~AudioReceiveStream() { } void AudioReceiveStream::Start() { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); - ScopedVoEInterface base(voice_engine()); - int error = base->StartPlayout(config_.voe_channel_id); + RTC_DCHECK_RUN_ON(&thread_checker_); + if (playing_) { + return; + } + + int error = SetVoiceEnginePlayout(true); if (error != 0) { LOG(LS_ERROR) << "AudioReceiveStream::Start failed with error: " << error; + return; } + + if (!audio_state()->mixer()->AddSource(this)) { + LOG(LS_ERROR) << "Failed to add source to mixer."; + SetVoiceEnginePlayout(false); + return; + } + + playing_ = true; } void AudioReceiveStream::Stop() { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); - ScopedVoEInterface base(voice_engine()); - base->StopPlayout(config_.voe_channel_id); + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!playing_) { + return; + } + playing_ = false; + + audio_state()->mixer()->RemoveSource(this); + SetVoiceEnginePlayout(false); } webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK_RUN_ON(&thread_checker_); webrtc::AudioReceiveStream::Stats stats; stats.remote_ssrc = config_.rtp.remote_ssrc; ScopedVoEInterface codec(voice_engine()); @@ -216,17 +235,17 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { } void AudioReceiveStream::SetSink(std::unique_ptr sink) { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK_RUN_ON(&thread_checker_); channel_proxy_->SetSink(std::move(sink)); } void AudioReceiveStream::SetGain(float gain) { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK_RUN_ON(&thread_checker_); channel_proxy_->SetChannelOutputVolumeScaling(gain); } const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK_RUN_ON(&thread_checker_); return config_; } @@ -243,7 +262,7 @@ void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) { } void AudioReceiveStream::SignalNetworkState(NetworkState state) { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK_RUN_ON(&thread_checker_); } bool AudioReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { @@ -296,12 +315,26 @@ int AudioReceiveStream::Ssrc() const { return config_.rtp.local_ssrc; } +internal::AudioState* AudioReceiveStream::audio_state() const { + auto* audio_state = static_cast(audio_state_.get()); + RTC_DCHECK(audio_state); + return audio_state; +} + VoiceEngine* AudioReceiveStream::voice_engine() const { - internal::AudioState* audio_state = - static_cast(audio_state_.get()); - VoiceEngine* voice_engine = audio_state->voice_engine(); + auto* voice_engine = audio_state()->voice_engine(); RTC_DCHECK(voice_engine); return voice_engine; } + +int AudioReceiveStream::SetVoiceEnginePlayout(bool playout) { + ScopedVoEInterface base(voice_engine()); + if (playout) { + return base->StartPlayout(config_.voe_channel_id); + } else { + return base->StopPlayout(config_.voe_channel_id); + } +} + } // namespace internal } // namespace webrtc diff --git a/webrtc/audio/audio_receive_stream.h b/webrtc/audio/audio_receive_stream.h index aeaadc6d68..6e7da09d1d 100644 --- a/webrtc/audio/audio_receive_stream.h +++ b/webrtc/audio/audio_receive_stream.h @@ -16,6 +16,7 @@ #include "webrtc/api/audio/audio_mixer.h" #include "webrtc/api/call/audio_receive_stream.h" #include "webrtc/api/call/audio_state.h" +#include "webrtc/audio/audio_state.h" #include "webrtc/base/constructormagic.h" #include "webrtc/base/thread_checker.h" #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h" @@ -64,6 +65,8 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream, private: VoiceEngine* voice_engine() const; + AudioState* audio_state() const; + int SetVoiceEnginePlayout(bool playout); rtc::ThreadChecker thread_checker_; RemoteBitrateEstimator* remote_bitrate_estimator_ = nullptr; @@ -72,6 +75,8 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream, std::unique_ptr rtp_header_parser_; std::unique_ptr channel_proxy_; + bool playing_ ACCESS_ON(thread_checker_) = false; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioReceiveStream); }; } // namespace internal diff --git a/webrtc/audio/audio_receive_stream_unittest.cc b/webrtc/audio/audio_receive_stream_unittest.cc index f263948fc4..bc071970e1 100644 --- a/webrtc/audio/audio_receive_stream_unittest.cc +++ b/webrtc/audio/audio_receive_stream_unittest.cc @@ -11,11 +11,11 @@ #include #include +#include "webrtc/api/test/mock_audio_mixer.h" #include "webrtc/audio/audio_receive_stream.h" #include "webrtc/audio/conversion.h" #include "webrtc/logging/rtc_event_log/mock/mock_rtc_event_log.h" #include "webrtc/modules/audio_coding/codecs/mock/mock_audio_decoder_factory.h" -#include "webrtc/modules/audio_mixer/audio_mixer_impl.h" #include "webrtc/modules/bitrate_controller/include/mock/mock_bitrate_controller.h" #include "webrtc/modules/congestion_controller/include/mock/mock_congestion_controller.h" #include "webrtc/modules/pacing/packet_router.h" @@ -72,7 +72,8 @@ struct ConfigHelper { congestion_controller_(&simulated_clock_, &bitrate_observer_, &remote_bitrate_observer_, - &event_log_) { + &event_log_), + audio_mixer_(new rtc::RefCountedObject()) { using testing::Invoke; EXPECT_CALL(voice_engine_, @@ -85,7 +86,7 @@ struct ConfigHelper { AudioState::Config config; config.voice_engine = &voice_engine_; - config.audio_mixer = AudioMixerImpl::Create(); + config.audio_mixer = audio_mixer_; audio_state_ = AudioState::Create(config); EXPECT_CALL(voice_engine_, ChannelProxyFactory(kChannelId)) @@ -122,7 +123,6 @@ struct ConfigHelper { EXPECT_CALL(*channel_proxy_, DisassociateSendChannel()).Times(1); return channel_proxy_; })); - EXPECT_CALL(voice_engine_, StopPlayout(kChannelId)).WillOnce(Return(0)); stream_config_.voe_channel_id = kChannelId; stream_config_.rtp.local_ssrc = kLocalSsrc; stream_config_.rtp.remote_ssrc = kRemoteSsrc; @@ -143,6 +143,7 @@ struct ConfigHelper { MockRtcEventLog* event_log() { return &event_log_; } AudioReceiveStream::Config& config() { return stream_config_; } rtc::scoped_refptr audio_state() { return audio_state_; } + rtc::scoped_refptr audio_mixer() { return audio_mixer_; } MockVoiceEngine& voice_engine() { return voice_engine_; } MockVoEChannelProxy* channel_proxy() { return channel_proxy_; } @@ -185,6 +186,7 @@ struct ConfigHelper { MockRtcEventLog event_log_; testing::StrictMock voice_engine_; rtc::scoped_refptr audio_state_; + rtc::scoped_refptr audio_mixer_; AudioReceiveStream::Config stream_config_; testing::StrictMock* channel_proxy_ = nullptr; }; @@ -368,5 +370,31 @@ TEST(AudioReceiveStreamTest, SetGain) { SetChannelOutputVolumeScaling(FloatEq(0.765f))); recv_stream.SetGain(0.765f); } + +TEST(AudioReceiveStreamTest, StreamShouldNotBeAddedToMixerWhenVoEReturnsError) { + ConfigHelper helper; + internal::AudioReceiveStream recv_stream( + helper.congestion_controller(), helper.config(), helper.audio_state(), + helper.event_log()); + + EXPECT_CALL(helper.voice_engine(), StartPlayout(_)).WillOnce(Return(-1)); + EXPECT_CALL(*helper.audio_mixer(), AddSource(_)).Times(0); + + recv_stream.Start(); +} + +TEST(AudioReceiveStreamTest, StreamShouldBeAddedToMixerOnStart) { + ConfigHelper helper; + internal::AudioReceiveStream recv_stream( + helper.congestion_controller(), helper.config(), helper.audio_state(), + helper.event_log()); + + EXPECT_CALL(helper.voice_engine(), StartPlayout(_)).WillOnce(Return(0)); + EXPECT_CALL(helper.voice_engine(), StopPlayout(_)); + EXPECT_CALL(*helper.audio_mixer(), AddSource(&recv_stream)) + .WillOnce(Return(true)); + + recv_stream.Start(); +} } // namespace test } // namespace webrtc diff --git a/webrtc/audio/audio_state_unittest.cc b/webrtc/audio/audio_state_unittest.cc index 38485a88bf..05c5003459 100644 --- a/webrtc/audio/audio_state_unittest.cc +++ b/webrtc/audio/audio_state_unittest.cc @@ -19,26 +19,65 @@ namespace webrtc { namespace test { namespace { -struct ConfigHelper { - ConfigHelper() { - EXPECT_CALL(voice_engine_, - RegisterVoiceEngineObserver(testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(voice_engine_, - DeRegisterVoiceEngineObserver()).WillOnce(testing::Return(0)); - EXPECT_CALL(voice_engine_, audio_device_module()); - EXPECT_CALL(voice_engine_, audio_processing()); - EXPECT_CALL(voice_engine_, audio_transport()); +const int kSampleRate = 8000; +const int kNumberOfChannels = 1; +const int kBytesPerSample = 2; - config_.voice_engine = &voice_engine_; - config_.audio_mixer = AudioMixerImpl::Create(); +struct ConfigHelper { + ConfigHelper() : audio_mixer(AudioMixerImpl::Create()) { + EXPECT_CALL(mock_voice_engine, RegisterVoiceEngineObserver(testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock_voice_engine, DeRegisterVoiceEngineObserver()) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock_voice_engine, audio_device_module()) + .Times(testing::AtLeast(1)); + EXPECT_CALL(mock_voice_engine, audio_processing()) + .Times(testing::AtLeast(1)); + EXPECT_CALL(mock_voice_engine, audio_transport()) + .WillRepeatedly(testing::Return(&audio_transport)); + + auto device = static_cast( + voice_engine().audio_device_module()); + + // Populate the audio transport proxy pointer to the most recent + // transport connected to the Audio Device. + ON_CALL(*device, RegisterAudioCallback(testing::_)) + .WillByDefault(testing::Invoke([this](AudioTransport* transport) { + registered_audio_transport = transport; + return 0; + })); + + audio_state_config.voice_engine = &mock_voice_engine; + audio_state_config.audio_mixer = audio_mixer; } - AudioState::Config& config() { return config_; } - MockVoiceEngine& voice_engine() { return voice_engine_; } + AudioState::Config& config() { return audio_state_config; } + MockVoiceEngine& voice_engine() { return mock_voice_engine; } + rtc::scoped_refptr mixer() { return audio_mixer; } + MockAudioTransport& original_audio_transport() { return audio_transport; } + AudioTransport* audio_transport_proxy() { return registered_audio_transport; } private: - testing::StrictMock voice_engine_; - AudioState::Config config_; + testing::StrictMock mock_voice_engine; + AudioState::Config audio_state_config; + rtc::scoped_refptr audio_mixer; + MockAudioTransport audio_transport; + AudioTransport* registered_audio_transport = nullptr; }; + +class FakeAudioSource : public AudioMixer::Source { + public: + // TODO(aleloi): Valid overrides commented out, because the gmock + // methods don't use any override declarations, and we want to avoid + // warnings from -Winconsistent-missing-override. See + // http://crbug.com/428099. + int Ssrc() const /*override*/ { return 0; } + + int PreferredSampleRate() const /*override*/ { return kSampleRate; } + + MOCK_METHOD2(GetAudioFrameWithInfo, + AudioFrameInfo(int sample_rate_hz, AudioFrame* audio_frame)); +}; + } // namespace TEST(AudioStateTest, Create) { @@ -84,38 +123,52 @@ TEST(AudioStateTest, TypingNoiseDetected) { } // Test that RecordedDataIsAvailable calls get to the original transport. -TEST(AudioStateTest, RecordedAudioArrivesAtOriginalTransport) { - using testing::_; - +TEST(AudioStateAudioPathTest, RecordedAudioArrivesAtOriginalTransport) { ConfigHelper helper; - auto& voice_engine = helper.voice_engine(); - auto device = - static_cast(voice_engine.audio_device_module()); - AudioTransport* audio_transport_proxy = nullptr; - ON_CALL(*device, RegisterAudioCallback(_)) - .WillByDefault( - testing::Invoke([&audio_transport_proxy](AudioTransport* transport) { - audio_transport_proxy = transport; - return 0; + rtc::scoped_refptr audio_state = + AudioState::Create(helper.config()); + + // Setup completed. Ensure call of original transport is forwarded to new. + uint32_t new_mic_level; + EXPECT_CALL( + helper.original_audio_transport(), + RecordedDataIsAvailable(nullptr, kSampleRate / 100, kBytesPerSample, + kNumberOfChannels, kSampleRate, 0, 0, 0, false, + testing::Ref(new_mic_level))); + + helper.audio_transport_proxy()->RecordedDataIsAvailable( + nullptr, kSampleRate / 100, kBytesPerSample, kNumberOfChannels, + kSampleRate, 0, 0, 0, false, new_mic_level); +} + +TEST(AudioStateAudioPathTest, + QueryingProxyForAudioShouldResultInGetAudioCallOnMixerSource) { + ConfigHelper helper; + + rtc::scoped_refptr audio_state = + AudioState::Create(helper.config()); + + FakeAudioSource fake_source; + + helper.mixer()->AddSource(&fake_source); + + EXPECT_CALL(fake_source, GetAudioFrameWithInfo(testing::_, testing::_)) + .WillOnce( + testing::Invoke([](int sample_rate_hz, AudioFrame* audio_frame) { + audio_frame->sample_rate_hz_ = sample_rate_hz; + audio_frame->samples_per_channel_ = sample_rate_hz / 100; + audio_frame->num_channels_ = kNumberOfChannels; + return AudioMixer::Source::AudioFrameInfo::kNormal; })); - MockAudioTransport original_audio_transport; - ON_CALL(voice_engine, audio_transport()) - .WillByDefault(testing::Return(&original_audio_transport)); - - EXPECT_CALL(voice_engine, audio_device_module()); - std::unique_ptr audio_state( - new internal::AudioState(helper.config())); - - // Setup completed. Ensure call of old transport is forwarded to new. - uint32_t new_mic_level; - EXPECT_CALL(original_audio_transport, - RecordedDataIsAvailable(nullptr, 80, 2, 1, 8000, 0, 0, 0, false, - testing::Ref(new_mic_level))); - - audio_transport_proxy->RecordedDataIsAvailable(nullptr, 80, 2, 1, 8000, 0, 0, - 0, false, new_mic_level); + int16_t audio_buffer[kSampleRate / 100 * kNumberOfChannels]; + size_t n_samples_out; + int64_t elapsed_time_ms; + int64_t ntp_time_ms; + helper.audio_transport_proxy()->NeedMorePlayData( + kSampleRate / 100, kBytesPerSample, kNumberOfChannels, kSampleRate, + audio_buffer, n_samples_out, &elapsed_time_ms, &ntp_time_ms); } } // namespace test } // namespace webrtc diff --git a/webrtc/audio/audio_transport_proxy.cc b/webrtc/audio/audio_transport_proxy.cc index ed72200379..163cc11c77 100644 --- a/webrtc/audio/audio_transport_proxy.cc +++ b/webrtc/audio/audio_transport_proxy.cc @@ -12,12 +12,32 @@ namespace webrtc { +namespace { +// Resample audio in |frame| to given sample rate preserving the +// channel count and place the result in |destination|. +int Resample(const AudioFrame& frame, + const int destination_sample_rate, + PushResampler* resampler, + int16_t* destination) { + const int number_of_channels = static_cast(frame.num_channels_); + const int target_number_of_samples_per_channel = + destination_sample_rate / 100; + resampler->InitializeIfNeeded(frame.sample_rate_hz_, destination_sample_rate, + number_of_channels); + + return resampler->Resample( + frame.data_, frame.samples_per_channel_ * number_of_channels, destination, + number_of_channels * target_number_of_samples_per_channel); +} +} // namespace + AudioTransportProxy::AudioTransportProxy(AudioTransport* voe_audio_transport, AudioProcessing* apm, AudioMixer* mixer) - : voe_audio_transport_(voe_audio_transport) { + : voe_audio_transport_(voe_audio_transport), apm_(apm), mixer_(mixer) { RTC_DCHECK(voe_audio_transport); RTC_DCHECK(apm); + RTC_DCHECK(mixer); } AudioTransportProxy::~AudioTransportProxy() {} @@ -53,14 +73,23 @@ int32_t AudioTransportProxy::NeedMorePlayData(const size_t nSamples, RTC_DCHECK_GE( samplesPerSec, static_cast(AudioProcessing::NativeRate::kSampleRate8kHz)); + + // 100 = 1 second / data duration (10 ms). RTC_DCHECK_EQ(nSamples * 100, samplesPerSec); RTC_DCHECK_LE(nBytesPerSample * nSamples * nChannels, sizeof(AudioFrame::data_)); - // Pass call through to original audio transport instance. - return voe_audio_transport_->NeedMorePlayData( - nSamples, nBytesPerSample, nChannels, samplesPerSec, audioSamples, - nSamplesOut, elapsed_time_ms, ntp_time_ms); + mixer_->Mix(nChannels, &mixed_frame_); + *elapsed_time_ms = mixed_frame_.elapsed_time_ms_; + *ntp_time_ms = mixed_frame_.ntp_time_ms_; + + const auto error = apm_->ProcessReverseStream(&mixed_frame_); + RTC_DCHECK_EQ(error, AudioProcessing::kNoError); + + nSamplesOut = Resample(mixed_frame_, samplesPerSec, &resampler_, + static_cast(audioSamples)); + RTC_DCHECK_EQ(nSamplesOut, nChannels * nSamples); + return 0; } void AudioTransportProxy::PushCaptureData(int voe_channel, @@ -81,17 +110,25 @@ void AudioTransportProxy::PullRenderData(int bits_per_sample, void* audio_data, int64_t* elapsed_time_ms, int64_t* ntp_time_ms) { - RTC_DCHECK_EQ(static_cast(bits_per_sample), 8 * sizeof(int16_t)); + RTC_DCHECK_EQ(static_cast(bits_per_sample), 16); RTC_DCHECK_GE(number_of_channels, 1u); RTC_DCHECK_LE(number_of_channels, 2u); RTC_DCHECK_GE(static_cast(sample_rate), AudioProcessing::NativeRate::kSampleRate8kHz); + + // 100 = 1 second / data duration (10 ms). RTC_DCHECK_EQ(static_cast(number_of_frames * 100), sample_rate); + + // 8 = bits per byte. RTC_DCHECK_LE(bits_per_sample / 8 * number_of_frames * number_of_channels, sizeof(AudioFrame::data_)); - voe_audio_transport_->PullRenderData( - bits_per_sample, sample_rate, number_of_channels, number_of_frames, - audio_data, elapsed_time_ms, ntp_time_ms); + mixer_->Mix(number_of_channels, &mixed_frame_); + *elapsed_time_ms = mixed_frame_.elapsed_time_ms_; + *ntp_time_ms = mixed_frame_.ntp_time_ms_; + + const auto output_samples = Resample(mixed_frame_, sample_rate, &resampler_, + static_cast(audio_data)); + RTC_DCHECK_EQ(output_samples, number_of_channels * number_of_frames); } } // namespace webrtc diff --git a/webrtc/audio/audio_transport_proxy.h b/webrtc/audio/audio_transport_proxy.h index 05e52fce85..1d03378402 100644 --- a/webrtc/audio/audio_transport_proxy.h +++ b/webrtc/audio/audio_transport_proxy.h @@ -13,6 +13,8 @@ #include "webrtc/api/audio/audio_mixer.h" #include "webrtc/base/constructormagic.h" +#include "webrtc/base/scoped_ref_ptr.h" +#include "webrtc/common_audio/resampler/include/push_resampler.h" #include "webrtc/modules/audio_device/include/audio_device_defines.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" @@ -63,7 +65,11 @@ class AudioTransportProxy : public AudioTransport { private: AudioTransport* voe_audio_transport_; - AudioFrame frame_for_mixing_; + AudioProcessing* apm_; + rtc::scoped_refptr mixer_; + AudioFrame mixed_frame_; + // Converts mixed audio to the audio device output rate. + PushResampler resampler_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioTransportProxy); };