diff --git a/webrtc/voice_engine/BUILD.gn b/webrtc/voice_engine/BUILD.gn index cb604c6e85..198e62c83d 100644 --- a/webrtc/voice_engine/BUILD.gn +++ b/webrtc/voice_engine/BUILD.gn @@ -137,6 +137,7 @@ rtc_static_library("voice_engine") { "../api/audio_codecs:builtin_audio_decoder_factory", "../audio/utility:audio_frame_operations", "../base:rtc_base_approved", + "../base:rtc_task_queue", # TODO(nisse): Delete when declaration of RtpTransportController # and related interfaces move to api/. diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc index 1ab69714f7..6ff368560d 100644 --- a/webrtc/voice_engine/channel.cc +++ b/webrtc/voice_engine/channel.cc @@ -21,6 +21,8 @@ #include "webrtc/base/location.h" #include "webrtc/base/logging.h" #include "webrtc/base/rate_limiter.h" +#include "webrtc/base/task_queue.h" +#include "webrtc/base/thread_checker.h" #include "webrtc/base/timeutils.h" #include "webrtc/call/rtp_transport_controller_send.h" #include "webrtc/config.h" @@ -409,12 +411,32 @@ class VoERtcpObserver : public RtcpBandwidthObserver { RtcpBandwidthObserver* bandwidth_observer_ GUARDED_BY(crit_); }; +class Channel::ProcessAndEncodeAudioTask : public rtc::QueuedTask { + public: + ProcessAndEncodeAudioTask(std::unique_ptr audio_frame, + Channel* channel) + : audio_frame_(std::move(audio_frame)), channel_(channel) { + RTC_DCHECK(channel_); + } + + private: + bool Run() override { + RTC_DCHECK_RUN_ON(channel_->encoder_queue_); + channel_->ProcessAndEncodeAudioOnTaskQueue(audio_frame_.get()); + return true; + } + + std::unique_ptr audio_frame_; + Channel* const channel_; +}; + int32_t Channel::SendData(FrameType frameType, uint8_t payloadType, uint32_t timeStamp, const uint8_t* payloadData, size_t payloadSize, const RTPFragmentationHeader* fragmentation) { + RTC_DCHECK_RUN_ON(encoder_queue_); WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId), "Channel::SendData(frameType=%u, payloadType=%u, timeStamp=%u," " payloadSize=%" PRIuS ", fragmentation=0x%x)", @@ -442,9 +464,6 @@ int32_t Channel::SendData(FrameType frameType, return -1; } - _lastLocalTimeStamp = timeStamp; - _lastPayloadType = payloadType; - return 0; } @@ -779,11 +798,10 @@ int32_t Channel::NeededFrequency(int32_t id) const { return (highestNeeded); } -int32_t Channel::CreateChannel( - Channel*& channel, - int32_t channelId, - uint32_t instanceId, - const VoEBase::ChannelConfig& config) { +int32_t Channel::CreateChannel(Channel*& channel, + int32_t channelId, + uint32_t instanceId, + const VoEBase::ChannelConfig& config) { WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(instanceId, channelId), "Channel::CreateChannel(channelId=%d, instanceId=%d)", channelId, instanceId); @@ -890,8 +908,6 @@ Channel::Channel(int32_t channelId, previous_frame_muted_(false), _outputGain(1.0f), _mixFileWithMicrophone(false), - _lastLocalTimeStamp(0), - _lastPayloadType(0), _includeAudioLevelIndication(false), transport_overhead_per_packet_(0), rtp_overhead_per_packet_(0), @@ -1125,7 +1141,10 @@ int32_t Channel::SetEngineInformation(Statistics& engineStatistics, ProcessThread& moduleProcessThread, AudioDeviceModule& audioDeviceModule, VoiceEngineObserver* voiceEngineObserver, - rtc::CriticalSection* callbackCritSect) { + rtc::CriticalSection* callbackCritSect, + rtc::TaskQueue* encoder_queue) { + RTC_DCHECK(encoder_queue); + RTC_DCHECK(!encoder_queue_); WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId), "Channel::SetEngineInformation()"); _engineStatisticsPtr = &engineStatistics; @@ -1134,11 +1153,7 @@ int32_t Channel::SetEngineInformation(Statistics& engineStatistics, _audioDeviceModulePtr = &audioDeviceModule; _voiceEngineObserverPtr = voiceEngineObserver; _callbackCritSectPtr = callbackCritSect; - return 0; -} - -int32_t Channel::UpdateLocalTimeStamp() { - _timeStamp += static_cast(_audioFrame.samples_per_channel_); + encoder_queue_ = encoder_queue; return 0; } @@ -1222,14 +1237,25 @@ int32_t Channel::StartSend() { return 0; } -int32_t Channel::StopSend() { +void Channel::StopSend() { WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId), "Channel::StopSend()"); if (!channel_state_.Get().sending) { - return 0; + return; } channel_state_.SetSending(false); + // Post a task to the encoder thread which sets an event when the task is + // executed. We know that no more encoding tasks will be added to the task + // queue for this channel since sending is now deactivated. It means that, + // if we wait for the event to bet set, we know that no more pending tasks + // exists and it is therfore guaranteed that the task queue will never try + // to acccess and invalid channel object. + RTC_DCHECK(encoder_queue_); + rtc::Event flush(false, false); + encoder_queue_->PostTask([&flush]() { flush.Set(); }); + flush.Wait(rtc::Event::kForever); + // Store the sequence number to be able to pick up the same sequence for // the next StartSend(). This is needed for restarting device, otherwise // it might cause libSRTP to complain about packets being replayed. @@ -1246,8 +1272,6 @@ int32_t Channel::StopSend() { "StartSend() RTP/RTCP failed to stop sending"); } _rtpRtcpModule->SetSendingMediaStatus(false); - - return 0; } int32_t Channel::RegisterVoiceEngineObserver(VoiceEngineObserver& observer) { @@ -2648,90 +2672,73 @@ int Channel::ResendPackets(const uint16_t* sequence_numbers, int length) { return _rtpRtcpModule->SendNACK(sequence_numbers, length); } -uint32_t Channel::Demultiplex(const AudioFrame& audioFrame) { - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId), - "Channel::Demultiplex()"); - _audioFrame.CopyFrom(audioFrame); - _audioFrame.id_ = _channelId; - return 0; +void Channel::ProcessAndEncodeAudio(const AudioFrame& audio_input) { + RTC_DCHECK(channel_state_.Get().sending); + std::unique_ptr audio_frame(new AudioFrame()); + // TODO(henrika): try to avoid copying by moving ownership of audio frame + // either into pool of frames or into the task itself. + audio_frame->CopyFrom(audio_input); + audio_frame->id_ = ChannelId(); + encoder_queue_->PostTask(std::unique_ptr( + new ProcessAndEncodeAudioTask(std::move(audio_frame), this))); } -void Channel::Demultiplex(const int16_t* audio_data, - int sample_rate, - size_t number_of_frames, - size_t number_of_channels) { +void Channel::ProcessAndEncodeAudio(const int16_t* audio_data, + int sample_rate, + size_t number_of_frames, + size_t number_of_channels) { + RTC_DCHECK(channel_state_.Get().sending); CodecInst codec; GetSendCodec(codec); - - // Never upsample or upmix the capture signal here. This should be done at the - // end of the send chain. - _audioFrame.sample_rate_hz_ = std::min(codec.plfreq, sample_rate); - _audioFrame.num_channels_ = std::min(number_of_channels, codec.channels); + std::unique_ptr audio_frame(new AudioFrame()); + audio_frame->id_ = ChannelId(); + audio_frame->sample_rate_hz_ = std::min(codec.plfreq, sample_rate); + audio_frame->num_channels_ = std::min(number_of_channels, codec.channels); RemixAndResample(audio_data, number_of_frames, number_of_channels, - sample_rate, &input_resampler_, &_audioFrame); + sample_rate, &input_resampler_, audio_frame.get()); + encoder_queue_->PostTask(std::unique_ptr( + new ProcessAndEncodeAudioTask(std::move(audio_frame), this))); } -uint32_t Channel::PrepareEncodeAndSend(int mixingFrequency) { - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId), - "Channel::PrepareEncodeAndSend()"); - - if (_audioFrame.samples_per_channel_ == 0) { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId), - "Channel::PrepareEncodeAndSend() invalid audio frame"); - return 0xFFFFFFFF; - } +void Channel::ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input) { + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK_GT(audio_input->samples_per_channel_, 0); + RTC_DCHECK_LE(audio_input->num_channels_, 2); + RTC_DCHECK_EQ(audio_input->id_, ChannelId()); if (channel_state_.Get().input_file_playing) { - MixOrReplaceAudioWithFile(mixingFrequency); + MixOrReplaceAudioWithFile(audio_input); } - bool is_muted = InputMute(); // Cache locally as InputMute() takes a lock. - AudioFrameOperations::Mute(&_audioFrame, previous_frame_muted_, is_muted); + bool is_muted = InputMute(); + AudioFrameOperations::Mute(audio_input, previous_frame_muted_, is_muted); if (_includeAudioLevelIndication) { size_t length = - _audioFrame.samples_per_channel_ * _audioFrame.num_channels_; - RTC_CHECK_LE(length, sizeof(_audioFrame.data_)); + audio_input->samples_per_channel_ * audio_input->num_channels_; + RTC_CHECK_LE(length, sizeof(audio_input->data_)); if (is_muted && previous_frame_muted_) { rms_level_.AnalyzeMuted(length); } else { rms_level_.Analyze( - rtc::ArrayView(_audioFrame.data_, length)); + rtc::ArrayView(audio_input->data_, length)); } } previous_frame_muted_ = is_muted; - return 0; -} - -uint32_t Channel::EncodeAndSend() { - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId), - "Channel::EncodeAndSend()"); - - assert(_audioFrame.num_channels_ <= 2); - if (_audioFrame.samples_per_channel_ == 0) { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId), - "Channel::EncodeAndSend() invalid audio frame"); - return 0xFFFFFFFF; - } - - _audioFrame.id_ = _channelId; - - // --- Add 10ms of raw (PCM) audio data to the encoder @ 32kHz. + // Add 10ms of raw (PCM) audio data to the encoder @ 32kHz. // The ACM resamples internally. - _audioFrame.timestamp_ = _timeStamp; + audio_input->timestamp_ = _timeStamp; // This call will trigger AudioPacketizationCallback::SendData if encoding // is done and payload is ready for packetization and transmission. // Otherwise, it will return without invoking the callback. - if (audio_coding_->Add10MsData((AudioFrame&)_audioFrame) < 0) { - WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, _channelId), - "Channel::EncodeAndSend() ACM encoding failed"); - return 0xFFFFFFFF; + if (audio_coding_->Add10MsData(*audio_input) < 0) { + LOG(LS_ERROR) << "ACM::Add10MsData() failed for channel " << _channelId; + return; } - _timeStamp += static_cast(_audioFrame.samples_per_channel_); - return 0; + _timeStamp += static_cast(audio_input->samples_per_channel_); } void Channel::set_associate_send_channel(const ChannelOwner& channel) { @@ -2840,10 +2847,11 @@ int Channel::GetRtpRtcp(RtpRtcp** rtpRtcpModule, // TODO(andrew): refactor Mix functions here and in transmit_mixer.cc to use // a shared helper. -int32_t Channel::MixOrReplaceAudioWithFile(int mixingFrequency) { +int32_t Channel::MixOrReplaceAudioWithFile(AudioFrame* audio_input) { + RTC_DCHECK_RUN_ON(encoder_queue_); std::unique_ptr fileBuffer(new int16_t[640]); size_t fileSamples(0); - + const int mixingFrequency = audio_input->sample_rate_hz_; { rtc::CritScope cs(&_fileCritSect); @@ -2868,18 +2876,18 @@ int32_t Channel::MixOrReplaceAudioWithFile(int mixingFrequency) { } } - assert(_audioFrame.samples_per_channel_ == fileSamples); + RTC_DCHECK_EQ(audio_input->samples_per_channel_, fileSamples); if (_mixFileWithMicrophone) { // Currently file stream is always mono. // TODO(xians): Change the code when FilePlayer supports real stereo. - MixWithSat(_audioFrame.data_, _audioFrame.num_channels_, fileBuffer.get(), + MixWithSat(audio_input->data_, audio_input->num_channels_, fileBuffer.get(), 1, fileSamples); } else { // Replace ACM audio with file. // Currently file stream is always mono. // TODO(xians): Change the code when FilePlayer supports real stereo. - _audioFrame.UpdateFrame( + audio_input->UpdateFrame( _channelId, 0xFFFFFFFF, fileBuffer.get(), fileSamples, mixingFrequency, AudioFrame::kNormalSpeech, AudioFrame::kVadUnknown, 1); } diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h index 24a7d6bfa0..1d2b089a16 100644 --- a/webrtc/voice_engine/channel.h +++ b/webrtc/voice_engine/channel.h @@ -16,6 +16,7 @@ #include "webrtc/api/audio/audio_mixer.h" #include "webrtc/api/call/audio_sink.h" #include "webrtc/base/criticalsection.h" +#include "webrtc/base/event.h" #include "webrtc/base/optional.h" #include "webrtc/base/thread_checker.h" #include "webrtc/common_audio/resampler/include/push_resampler.h" @@ -143,11 +144,10 @@ class Channel enum { KNumSocketThreads = 1 }; enum { KNumberOfSocketBuffers = 8 }; virtual ~Channel(); - static int32_t CreateChannel( - Channel*& channel, - int32_t channelId, - uint32_t instanceId, - const VoEBase::ChannelConfig& config); + static int32_t CreateChannel(Channel*& channel, + int32_t channelId, + uint32_t instanceId, + const VoEBase::ChannelConfig& config); Channel(int32_t channelId, uint32_t instanceId, const VoEBase::ChannelConfig& config); @@ -159,8 +159,8 @@ class Channel ProcessThread& moduleProcessThread, AudioDeviceModule& audioDeviceModule, VoiceEngineObserver* voiceEngineObserver, - rtc::CriticalSection* callbackCritSect); - int32_t UpdateLocalTimeStamp(); + rtc::CriticalSection* callbackCritSect, + rtc::TaskQueue* encoder_queue); void SetSink(std::unique_ptr sink); @@ -178,7 +178,7 @@ class Channel int32_t StartPlayout(); int32_t StopPlayout(); int32_t StartSend(); - int32_t StopSend(); + void StopSend(); int32_t RegisterVoiceEngineObserver(VoiceEngineObserver& observer); int32_t DeRegisterVoiceEngineObserver(); @@ -354,16 +354,27 @@ class Channel } RtpRtcp* RtpRtcpModulePtr() const { return _rtpRtcpModule.get(); } int8_t OutputEnergyLevel() const { return _outputAudioLevel.Level(); } - uint32_t Demultiplex(const AudioFrame& audioFrame); - // Demultiplex the data to the channel's |_audioFrame|. The difference - // between this method and the overloaded method above is that |audio_data| - // does not go through transmit_mixer and APM. - void Demultiplex(const int16_t* audio_data, - int sample_rate, - size_t number_of_frames, - size_t number_of_channels); - uint32_t PrepareEncodeAndSend(int mixingFrequency); - uint32_t EncodeAndSend(); + + // ProcessAndEncodeAudio() creates an audio frame copy and posts a task + // on the shared encoder task queue, wich in turn calls (on the queue) + // ProcessAndEncodeAudioOnTaskQueue() where the actual processing of the + // audio takes place. The processing mainly consists of encoding and preparing + // the result for sending by adding it to a send queue. + // The main reason for using a task queue here is to release the native, + // OS-specific, audio capture thread as soon as possible to ensure that it + // can go back to sleep and be prepared to deliver an new captured audio + // packet. + void ProcessAndEncodeAudio(const AudioFrame& audio_input); + + // This version of ProcessAndEncodeAudio() is used by PushCaptureData() in + // VoEBase and the audio in |audio_data| has not been subject to any APM + // processing. Some extra steps are therfore needed when building up the + // audio frame copy before using the same task as in the default call to + // ProcessAndEncodeAudio(const AudioFrame& audio_input). + void ProcessAndEncodeAudio(const int16_t* audio_data, + int sample_rate, + size_t number_of_frames, + size_t number_of_channels); // Associate to a send channel. // Used for obtaining RTT for a receive-only channel. @@ -389,8 +400,9 @@ class Channel void OnRecoverableUplinkPacketLossRate(float recoverable_packet_loss_rate); private: - void OnUplinkPacketLossRate(float packet_loss_rate); + class ProcessAndEncodeAudioTask; + void OnUplinkPacketLossRate(float packet_loss_rate); bool InputMute() const; bool OnRtpPacketWithHeader(const uint8_t* received_packet, size_t length, @@ -405,7 +417,7 @@ class Channel bool IsPacketInOrder(const RTPHeader& header) const; bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const; int ResendPackets(const uint16_t* sequence_numbers, int length); - int32_t MixOrReplaceAudioWithFile(int mixingFrequency); + int32_t MixOrReplaceAudioWithFile(AudioFrame* audio_frame); int32_t MixAudioWithFile(AudioFrame& audioFrame, int mixingFrequency); void UpdatePlayoutTimestamp(bool rtcp); void RegisterReceiveCodecsToRTPModule(); @@ -420,11 +432,16 @@ class Channel int GetRtpTimestampRateHz() const; int64_t GetRTT(bool allow_associate_channel) const; + // Called on the encoder task queue when a new input audio frame is ready + // for encoding. + void ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input); + + uint32_t _instanceId; + int32_t _channelId; + rtc::CriticalSection _fileCritSect; rtc::CriticalSection _callbackCritSect; rtc::CriticalSection volume_settings_critsect_; - uint32_t _instanceId; - int32_t _channelId; ChannelState channel_state_; @@ -443,7 +460,6 @@ class Channel std::unique_ptr audio_sink_; AudioLevel _outputAudioLevel; bool _externalTransport; - AudioFrame _audioFrame; // Downsamples to the codec rate if necessary. PushResampler input_resampler_; std::unique_ptr input_file_player_; @@ -453,7 +469,7 @@ class Channel int _outputFilePlayerId; int _outputFileRecorderId; bool _outputFileRecording; - uint32_t _timeStamp; + uint32_t _timeStamp ACCESS_ON(encoder_queue_); RemoteNtpTimeEstimator ntp_estimator_ GUARDED_BY(ts_stats_lock_); @@ -483,15 +499,15 @@ class Channel VoiceEngineObserver* _voiceEngineObserverPtr; // owned by base rtc::CriticalSection* _callbackCritSectPtr; // owned by base Transport* _transportPtr; // WebRtc socket or external transport - RmsLevel rms_level_; + RmsLevel rms_level_ ACCESS_ON(encoder_queue_); bool input_mute_ GUARDED_BY(volume_settings_critsect_); - bool previous_frame_muted_; // Only accessed from PrepareEncodeAndSend(). + bool previous_frame_muted_ ACCESS_ON(encoder_queue_); float _outputGain GUARDED_BY(volume_settings_critsect_); // VoEBase bool _mixFileWithMicrophone; // VoeRTP_RTCP - uint32_t _lastLocalTimeStamp; - int8_t _lastPayloadType; + // TODO(henrika): can today be accessed on the main thread and on the + // task queue; hence potential race. bool _includeAudioLevelIndication; size_t transport_overhead_per_packet_ GUARDED_BY(overhead_per_packet_lock_); size_t rtp_overhead_per_packet_ GUARDED_BY(overhead_per_packet_lock_); @@ -519,6 +535,8 @@ class Channel rtc::ThreadChecker construction_thread_; const bool use_twcc_plr_for_ana_; + + rtc::TaskQueue* encoder_queue_ = nullptr; }; } // namespace voe diff --git a/webrtc/voice_engine/shared_data.cc b/webrtc/voice_engine/shared_data.cc index 57a1a59b0d..e77f52a956 100644 --- a/webrtc/voice_engine/shared_data.cc +++ b/webrtc/voice_engine/shared_data.cc @@ -27,19 +27,16 @@ SharedData::SharedData() _channelManager(_gInstanceCounter), _engineStatistics(_gInstanceCounter), _audioDevicePtr(NULL), - _moduleProcessThreadPtr( - ProcessThread::Create("VoiceProcessThread")) { - Trace::CreateTrace(); - if (OutputMixer::Create(_outputMixerPtr, _gInstanceCounter) == 0) - { - _outputMixerPtr->SetEngineInformation(_engineStatistics); - } - if (TransmitMixer::Create(_transmitMixerPtr, _gInstanceCounter) == 0) - { - _transmitMixerPtr->SetEngineInformation(*_moduleProcessThreadPtr, - _engineStatistics, - _channelManager); - } + _moduleProcessThreadPtr(ProcessThread::Create("VoiceProcessThread")), + encoder_queue_("AudioEncoderQueue") { + Trace::CreateTrace(); + if (OutputMixer::Create(_outputMixerPtr, _gInstanceCounter) == 0) { + _outputMixerPtr->SetEngineInformation(_engineStatistics); + } + if (TransmitMixer::Create(_transmitMixerPtr, _gInstanceCounter) == 0) { + _transmitMixerPtr->SetEngineInformation(*_moduleProcessThreadPtr, + _engineStatistics, _channelManager); + } } SharedData::~SharedData() @@ -53,6 +50,11 @@ SharedData::~SharedData() Trace::ReturnTrace(); } +rtc::TaskQueue* SharedData::encoder_queue() { + RTC_DCHECK_RUN_ON(&construction_thread_); + return &encoder_queue_; +} + void SharedData::set_audio_device( const rtc::scoped_refptr& audio_device) { _audioDevicePtr = audio_device; diff --git a/webrtc/voice_engine/shared_data.h b/webrtc/voice_engine/shared_data.h index e301419454..1a910407a2 100644 --- a/webrtc/voice_engine/shared_data.h +++ b/webrtc/voice_engine/shared_data.h @@ -15,6 +15,9 @@ #include "webrtc/base/criticalsection.h" #include "webrtc/base/scoped_ref_ptr.h" +#include "webrtc/base/task_queue.h" +#include "webrtc/base/thread_annotations.h" +#include "webrtc/base/thread_checker.h" #include "webrtc/modules/audio_device/include/audio_device.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" #include "webrtc/modules/utility/include/process_thread.h" @@ -46,6 +49,7 @@ public: OutputMixer* output_mixer() { return _outputMixerPtr; } rtc::CriticalSection* crit_sec() { return &_apiCritPtr; } ProcessThread* process_thread() { return _moduleProcessThreadPtr.get(); } + rtc::TaskQueue* encoder_queue(); int NumOfSendingChannels(); int NumOfPlayingChannels(); @@ -57,18 +61,22 @@ public: const char* msg) const; protected: - const uint32_t _instanceId; - rtc::CriticalSection _apiCritPtr; - ChannelManager _channelManager; - Statistics _engineStatistics; - rtc::scoped_refptr _audioDevicePtr; - OutputMixer* _outputMixerPtr; - TransmitMixer* _transmitMixerPtr; - std::unique_ptr audioproc_; - std::unique_ptr _moduleProcessThreadPtr; + rtc::ThreadChecker construction_thread_; + const uint32_t _instanceId; + rtc::CriticalSection _apiCritPtr; + ChannelManager _channelManager; + Statistics _engineStatistics; + rtc::scoped_refptr _audioDevicePtr; + OutputMixer* _outputMixerPtr; + TransmitMixer* _transmitMixerPtr; + std::unique_ptr audioproc_; + std::unique_ptr _moduleProcessThreadPtr; + // |encoder_queue| is defined last to ensure all pending tasks are cancelled + // and deleted before any other members. + rtc::TaskQueue encoder_queue_ ACCESS_ON(construction_thread_); - SharedData(); - virtual ~SharedData(); + SharedData(); + virtual ~SharedData(); }; } // namespace voe diff --git a/webrtc/voice_engine/transmit_mixer.cc b/webrtc/voice_engine/transmit_mixer.cc index 07b47ec48e..e14b03f6d3 100644 --- a/webrtc/voice_engine/transmit_mixer.cc +++ b/webrtc/voice_engine/transmit_mixer.cc @@ -311,66 +311,14 @@ TransmitMixer::PrepareDemux(const void* audioSamples, return 0; } -int32_t -TransmitMixer::DemuxAndMix() -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::DemuxAndMix()"); - - for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); - it.Increment()) - { - Channel* channelPtr = it.GetChannel(); - if (channelPtr->Sending()) - { - // Demultiplex makes a copy of its input. - channelPtr->Demultiplex(_audioFrame); - channelPtr->PrepareEncodeAndSend(_audioFrame.sample_rate_hz_); - } +void TransmitMixer::ProcessAndEncodeAudio() { + RTC_DCHECK_GT(_audioFrame.samples_per_channel_, 0); + for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); + it.Increment()) { + Channel* const channel = it.GetChannel(); + if (channel->Sending()) { + channel->ProcessAndEncodeAudio(_audioFrame); } - return 0; -} - -void TransmitMixer::DemuxAndMix(const int voe_channels[], - size_t number_of_voe_channels) { - for (size_t i = 0; i < number_of_voe_channels; ++i) { - voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]); - voe::Channel* channel_ptr = ch.channel(); - if (channel_ptr) { - if (channel_ptr->Sending()) { - // Demultiplex makes a copy of its input. - channel_ptr->Demultiplex(_audioFrame); - channel_ptr->PrepareEncodeAndSend(_audioFrame.sample_rate_hz_); - } - } - } -} - -int32_t -TransmitMixer::EncodeAndSend() -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::EncodeAndSend()"); - - for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); - it.Increment()) - { - Channel* channelPtr = it.GetChannel(); - if (channelPtr->Sending()) - { - channelPtr->EncodeAndSend(); - } - } - return 0; -} - -void TransmitMixer::EncodeAndSend(const int voe_channels[], - size_t number_of_voe_channels) { - for (size_t i = 0; i < number_of_voe_channels; ++i) { - voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]); - voe::Channel* channel_ptr = ch.channel(); - if (channel_ptr && channel_ptr->Sending()) - channel_ptr->EncodeAndSend(); } } diff --git a/webrtc/voice_engine/transmit_mixer.h b/webrtc/voice_engine/transmit_mixer.h index 2e728f8e1c..a045cf8322 100644 --- a/webrtc/voice_engine/transmit_mixer.h +++ b/webrtc/voice_engine/transmit_mixer.h @@ -32,7 +32,6 @@ #endif namespace webrtc { - class AudioProcessing; class ProcessThread; @@ -64,16 +63,7 @@ public: uint16_t currentMicLevel, bool keyPressed); - - int32_t DemuxAndMix(); - // Used by the Chrome to pass the recording data to the specific VoE - // channels for demux. - void DemuxAndMix(const int voe_channels[], size_t number_of_voe_channels); - - int32_t EncodeAndSend(); - // Used by the Chrome to pass the recording data to the specific VoE - // channels for encoding and sending to the network. - void EncodeAndSend(const int voe_channels[], size_t number_of_voe_channels); + void ProcessAndEncodeAudio(); // Must be called on the same thread as PrepareDemux(). uint32_t CaptureLevel() const; diff --git a/webrtc/voice_engine/voe_base_impl.cc b/webrtc/voice_engine/voe_base_impl.cc index d1981a4a3a..8072cc86d9 100644 --- a/webrtc/voice_engine/voe_base_impl.cc +++ b/webrtc/voice_engine/voe_base_impl.cc @@ -77,19 +77,66 @@ void VoEBaseImpl::OnWarningIsReported(const WarningCode warning) { } } -int32_t VoEBaseImpl::RecordedDataIsAvailable(const void* audioSamples, - const size_t nSamples, - const size_t nBytesPerSample, - const size_t nChannels, - const uint32_t samplesPerSec, - const uint32_t totalDelayMS, - const int32_t clockDrift, - const uint32_t currentMicLevel, - const bool keyPressed, - uint32_t& newMicLevel) { - newMicLevel = static_cast(ProcessRecordedDataWithAPM( - nullptr, 0, audioSamples, samplesPerSec, nChannels, nSamples, - totalDelayMS, clockDrift, currentMicLevel, keyPressed)); +int32_t VoEBaseImpl::RecordedDataIsAvailable( + const void* audio_data, + const size_t number_of_frames, + const size_t bytes_per_sample, + const size_t number_of_channels, + const uint32_t sample_rate, + const uint32_t audio_delay_milliseconds, + const int32_t clock_drift, + const uint32_t volume, + const bool key_pressed, + uint32_t& new_mic_volume) { + RTC_DCHECK_EQ(2 * number_of_channels, bytes_per_sample); + RTC_DCHECK(shared_->transmit_mixer() != nullptr); + RTC_DCHECK(shared_->audio_device() != nullptr); + + uint32_t max_volume = 0; + uint16_t voe_mic_level = 0; + // Check for zero to skip this calculation; the consumer may use this to + // indicate no volume is available. + if (volume != 0) { + // Scale from ADM to VoE level range + if (shared_->audio_device()->MaxMicrophoneVolume(&max_volume) == 0) { + if (max_volume) { + voe_mic_level = static_cast( + (volume * kMaxVolumeLevel + static_cast(max_volume / 2)) / + max_volume); + } + } + // We learned that on certain systems (e.g Linux) the voe_mic_level + // can be greater than the maxVolumeLevel therefore + // we are going to cap the voe_mic_level to the maxVolumeLevel + // and change the maxVolume to volume if it turns out that + // the voe_mic_level is indeed greater than the maxVolumeLevel. + if (voe_mic_level > kMaxVolumeLevel) { + voe_mic_level = kMaxVolumeLevel; + max_volume = volume; + } + } + + // Perform channel-independent operations + // (APM, mix with file, record to file, mute, etc.) + shared_->transmit_mixer()->PrepareDemux( + audio_data, number_of_frames, number_of_channels, sample_rate, + static_cast(audio_delay_milliseconds), clock_drift, + voe_mic_level, key_pressed); + + // Copy the audio frame to each sending channel and perform + // channel-dependent operations (file mixing, mute, etc.), encode and + // packetize+transmit the RTP packet. + shared_->transmit_mixer()->ProcessAndEncodeAudio(); + + // Scale from VoE to ADM level range. + uint32_t new_voe_mic_level = shared_->transmit_mixer()->CaptureLevel(); + if (new_voe_mic_level != voe_mic_level) { + // Return the new volume if AGC has changed the volume. + return static_cast((new_voe_mic_level * max_volume + + static_cast(kMaxVolumeLevel / 2)) / + kMaxVolumeLevel); + } + return 0; } @@ -112,14 +159,15 @@ void VoEBaseImpl::PushCaptureData(int voe_channel, const void* audio_data, size_t number_of_channels, size_t number_of_frames) { voe::ChannelOwner ch = shared_->channel_manager().GetChannel(voe_channel); - voe::Channel* channel_ptr = ch.channel(); - if (!channel_ptr) return; - - if (channel_ptr->Sending()) { - channel_ptr->Demultiplex(static_cast(audio_data), - sample_rate, number_of_frames, number_of_channels); - channel_ptr->PrepareEncodeAndSend(sample_rate); - channel_ptr->EncodeAndSend(); + voe::Channel* channel = ch.channel(); + if (!channel) + return; + if (channel->Sending()) { + // Send the audio to each channel directly without using the APM in the + // transmit mixer. + channel->ProcessAndEncodeAudio(static_cast(audio_data), + sample_rate, number_of_frames, + number_of_channels); } } @@ -377,7 +425,8 @@ int VoEBaseImpl::InitializeChannel(voe::ChannelOwner* channel_owner) { if (channel_owner->channel()->SetEngineInformation( shared_->statistics(), *shared_->output_mixer(), *shared_->process_thread(), *shared_->audio_device(), - voiceEngineObserverPtr_, &callbackCritSect_) != 0) { + voiceEngineObserverPtr_, &callbackCritSect_, + shared_->encoder_queue()) != 0) { shared_->SetLastError( VE_CHANNEL_NOT_CREATED, kTraceError, "CreateChannel() failed to associate engine and channel." @@ -521,10 +570,7 @@ int VoEBaseImpl::StopSend(int channel) { "StopSend() failed to locate channel"); return -1; } - if (channelPtr->StopSend() != 0) { - LOG_F(LS_WARNING) << "StopSend() failed to stop sending for channel " - << channel; - } + channelPtr->StopSend(); return StopSend(); } @@ -648,73 +694,6 @@ int32_t VoEBaseImpl::TerminateInternal() { return shared_->statistics().SetUnInitialized(); } -int VoEBaseImpl::ProcessRecordedDataWithAPM( - const int voe_channels[], size_t number_of_voe_channels, - const void* audio_data, uint32_t sample_rate, size_t number_of_channels, - size_t number_of_frames, uint32_t audio_delay_milliseconds, - int32_t clock_drift, uint32_t volume, bool key_pressed) { - assert(shared_->transmit_mixer() != nullptr); - assert(shared_->audio_device() != nullptr); - - uint32_t max_volume = 0; - uint16_t voe_mic_level = 0; - // Check for zero to skip this calculation; the consumer may use this to - // indicate no volume is available. - if (volume != 0) { - // Scale from ADM to VoE level range - if (shared_->audio_device()->MaxMicrophoneVolume(&max_volume) == 0) { - if (max_volume) { - voe_mic_level = static_cast( - (volume * kMaxVolumeLevel + static_cast(max_volume / 2)) / - max_volume); - } - } - // We learned that on certain systems (e.g Linux) the voe_mic_level - // can be greater than the maxVolumeLevel therefore - // we are going to cap the voe_mic_level to the maxVolumeLevel - // and change the maxVolume to volume if it turns out that - // the voe_mic_level is indeed greater than the maxVolumeLevel. - if (voe_mic_level > kMaxVolumeLevel) { - voe_mic_level = kMaxVolumeLevel; - max_volume = volume; - } - } - - // Perform channel-independent operations - // (APM, mix with file, record to file, mute, etc.) - shared_->transmit_mixer()->PrepareDemux( - audio_data, number_of_frames, number_of_channels, sample_rate, - static_cast(audio_delay_milliseconds), clock_drift, - voe_mic_level, key_pressed); - - // Copy the audio frame to each sending channel and perform - // channel-dependent operations (file mixing, mute, etc.), encode and - // packetize+transmit the RTP packet. When |number_of_voe_channels| == 0, - // do the operations on all the existing VoE channels; otherwise the - // operations will be done on specific channels. - if (number_of_voe_channels == 0) { - shared_->transmit_mixer()->DemuxAndMix(); - shared_->transmit_mixer()->EncodeAndSend(); - } else { - shared_->transmit_mixer()->DemuxAndMix(voe_channels, - number_of_voe_channels); - shared_->transmit_mixer()->EncodeAndSend(voe_channels, - number_of_voe_channels); - } - - // Scale from VoE to ADM level range. - uint32_t new_voe_mic_level = shared_->transmit_mixer()->CaptureLevel(); - if (new_voe_mic_level != voe_mic_level) { - // Return the new volume if AGC has changed the volume. - return static_cast((new_voe_mic_level * max_volume + - static_cast(kMaxVolumeLevel / 2)) / - kMaxVolumeLevel); - } - - // Return 0 to indicate no change on the volume. - return 0; -} - void VoEBaseImpl::GetPlayoutData(int sample_rate, size_t number_of_channels, size_t number_of_frames, bool feed_data_to_apm, void* audio_data, int64_t* elapsed_time_ms, diff --git a/webrtc/voice_engine/voe_base_impl.h b/webrtc/voice_engine/voe_base_impl.h index b544c09f9a..b9f3282596 100644 --- a/webrtc/voice_engine/voe_base_impl.h +++ b/webrtc/voice_engine/voe_base_impl.h @@ -62,16 +62,16 @@ class VoEBaseImpl : public VoEBase, int AssociateSendChannel(int channel, int accociate_send_channel) override; // AudioTransport - int32_t RecordedDataIsAvailable(const void* audioSamples, - const size_t nSamples, - const size_t nBytesPerSample, - const size_t nChannels, - const uint32_t samplesPerSec, - const uint32_t totalDelayMS, - const int32_t clockDrift, - const uint32_t currentMicLevel, - const bool keyPressed, - uint32_t& newMicLevel) override; + int32_t RecordedDataIsAvailable(const void* audio_data, + const size_t number_of_frames, + const size_t bytes_per_sample, + const size_t number_of_channels, + const uint32_t sample_rate, + const uint32_t audio_delay_milliseconds, + const int32_t clock_drift, + const uint32_t volume, + const bool key_pressed, + uint32_t& new_mic_volume) override; int32_t NeedMorePlayData(const size_t nSamples, const size_t nBytesPerSample, const size_t nChannels, @@ -109,18 +109,6 @@ class VoEBaseImpl : public VoEBase, int32_t StopSend(); int32_t TerminateInternal(); - // Helper function to process the recorded data with AudioProcessing Module, - // demultiplex the data to specific voe channels, encode and send to the - // network. When |number_of_VoE_channels| is 0, it will demultiplex the - // data to all the existing VoE channels. - // It returns new AGC microphone volume or 0 if no volume changes - // should be done. - int ProcessRecordedDataWithAPM( - const int voe_channels[], size_t number_of_voe_channels, - const void* audio_data, uint32_t sample_rate, size_t number_of_channels, - size_t number_of_frames, uint32_t audio_delay_milliseconds, - int32_t clock_drift, uint32_t volume, bool key_pressed); - void GetPlayoutData(int sample_rate, size_t number_of_channels, size_t number_of_frames, bool feed_data_to_apm, void* audio_data, int64_t* elapsed_time_ms,