diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 615c97466e..8f772fe09c 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -50,6 +50,17 @@ class TransformableVideoFrameInterface : public TransformableFrameInterface { virtual std::vector GetAdditionalData() const = 0; }; +// Extends the TransformableFrameInterface to expose audio-specific information. +class TransformableAudioFrameInterface : public TransformableFrameInterface { + public: + virtual ~TransformableAudioFrameInterface() = default; + + // Exposes the frame header, enabling the interface clients to use the + // information in the header as needed, for example to compile the list of + // csrcs. + virtual const RTPHeader& GetHeader() const = 0; +}; + // Objects implement this interface to be notified with the transformed frame. class TransformedFrameCallback : public rtc::RefCountInterface { public: diff --git a/audio/BUILD.gn b/audio/BUILD.gn index cc52664312..7c49cd480c 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -26,6 +26,8 @@ rtc_library("audio") { "audio_transport_impl.h", "channel_receive.cc", "channel_receive.h", + "channel_receive_frame_transformer_delegate.cc", + "channel_receive_frame_transformer_delegate.h", "channel_send.cc", "channel_send.h", "channel_send_frame_transformer_delegate.cc", @@ -88,6 +90,7 @@ rtc_library("audio") { "../rtc_base:safe_minmax", "../rtc_base/experiments:field_trial_parser", "../rtc_base/synchronization:sequence_checker", + "../rtc_base/task_utils:to_queued_task", "../system_wrappers", "../system_wrappers:field_trial", "../system_wrappers:metrics", diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index 1c884215ba..50ad0aa85c 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -23,6 +23,7 @@ #include "api/frame_transformer_interface.h" #include "api/rtc_event_log/rtc_event_log.h" #include "audio/audio_level.h" +#include "audio/channel_receive_frame_transformer_delegate.h" #include "audio/channel_send.h" #include "audio/utility/audio_frame_operations.h" #include "logging/rtc_event_log/events/rtc_event_audio_playout.h" @@ -183,6 +184,9 @@ class ChannelReceive : public ChannelReceiveInterface { void OnReceivedPayloadData(rtc::ArrayView payload, const RTPHeader& rtpHeader); + void InitFrameTransformerDelegate( + rtc::scoped_refptr frame_transformer); + bool Playing() const { rtc::CritScope lock(&playing_lock_); return playing_; @@ -272,7 +276,8 @@ class ChannelReceive : public ChannelReceiveInterface { webrtc::AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_; - rtc::scoped_refptr frame_transformer_; + rtc::scoped_refptr + frame_transformer_delegate_; }; void ChannelReceive::OnReceivedPayloadData( @@ -302,6 +307,25 @@ void ChannelReceive::OnReceivedPayloadData( } } +void ChannelReceive::InitFrameTransformerDelegate( + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK(frame_transformer); + RTC_DCHECK(!frame_transformer_delegate_); + + // Pass a callback to ChannelReceive::ReceivePacket, to be called by the + // delegate to receive transformed audio. + ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback + receive_audio_callback = [this](rtc::ArrayView packet, + const RTPHeader& header) { + ReceivePacket(packet.data(), packet.size(), header); + }; + frame_transformer_delegate_ = + new rtc::RefCountedObject( + std::move(receive_audio_callback), std::move(frame_transformer), + rtc::Thread::Current()); + frame_transformer_delegate_->Init(); +} + AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo( int sample_rate_hz, AudioFrame* audio_frame) { @@ -456,8 +480,7 @@ ChannelReceive::ChannelReceive( associated_send_channel_(nullptr), frame_decryptor_(frame_decryptor), crypto_options_(crypto_options), - absolute_capture_time_receiver_(clock), - frame_transformer_(std::move(frame_transformer)) { + absolute_capture_time_receiver_(clock) { // TODO(nisse): Use _moduleProcessThreadPtr instead? module_process_thread_checker_.Detach(); @@ -481,6 +504,9 @@ ChannelReceive::ChannelReceive( configuration.event_log = event_log_; configuration.local_media_ssrc = local_ssrc; + if (frame_transformer) + InitFrameTransformerDelegate(std::move(frame_transformer)); + _rtpRtcpModule = RtpRtcp::Create(configuration); _rtpRtcpModule->SetSendingMediaStatus(false); _rtpRtcpModule->SetRemoteSSRC(remote_ssrc_); @@ -569,7 +595,13 @@ void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) { rtc::saturated_cast(packet_copy.payload_type_frequency()), header.extension.absolute_capture_time); - ReceivePacket(packet_copy.data(), packet_copy.size(), header); + if (frame_transformer_delegate_) { + // Asynchronously transform the received payload. After the payload is + // transformed, the delegate will call ReceivePacket to handle it. + frame_transformer_delegate_->Transform(packet_copy, header, remote_ssrc_); + } else { + ReceivePacket(packet_copy.data(), packet_copy.size(), header); + } } void ChannelReceive::ReceivePacket(const uint8_t* packet, @@ -758,7 +790,11 @@ void ChannelReceive::SetAssociatedSendChannel( void ChannelReceive::SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr frame_transformer) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); - frame_transformer_ = std::move(frame_transformer); + // Depending on when the channel is created, the transformer might be set + // twice. Don't replace the delegate if it was already initialized. + if (!frame_transformer || frame_transformer_delegate_) + return; + InitFrameTransformerDelegate(std::move(frame_transformer)); } NetworkStatistics ChannelReceive::GetNetworkStatistics() const { diff --git a/audio/channel_receive_frame_transformer_delegate.cc b/audio/channel_receive_frame_transformer_delegate.cc new file mode 100644 index 0000000000..261afbb100 --- /dev/null +++ b/audio/channel_receive_frame_transformer_delegate.cc @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/channel_receive_frame_transformer_delegate.h" + +#include + +#include "rtc_base/buffer.h" +#include "rtc_base/task_utils/to_queued_task.h" + +namespace webrtc { +namespace { + +class TransformableAudioFrame : public TransformableAudioFrameInterface { + public: + TransformableAudioFrame(rtc::ArrayView payload, + const RTPHeader& header, + uint32_t ssrc) + : payload_(payload.data(), payload.size()), + header_(header), + ssrc_(ssrc) {} + ~TransformableAudioFrame() override = default; + rtc::ArrayView GetData() const override { return payload_; } + + void SetData(rtc::ArrayView data) override { + payload_.SetData(data.data(), data.size()); + } + + uint32_t GetTimestamp() const override { return header_.timestamp; } + uint32_t GetSsrc() const override { return ssrc_; } + const RTPHeader& GetHeader() const override { return header_; } + + private: + rtc::Buffer payload_; + RTPHeader header_; + uint32_t ssrc_; +}; +} // namespace + +ChannelReceiveFrameTransformerDelegate::ChannelReceiveFrameTransformerDelegate( + ReceiveFrameCallback receive_frame_callback, + rtc::scoped_refptr frame_transformer, + rtc::Thread* channel_receive_thread) + : receive_frame_callback_(receive_frame_callback), + frame_transformer_(std::move(frame_transformer)), + channel_receive_thread_(channel_receive_thread) {} + +void ChannelReceiveFrameTransformerDelegate::Init() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + frame_transformer_->RegisterTransformedFrameCallback( + rtc::scoped_refptr(this)); +} + +void ChannelReceiveFrameTransformerDelegate::Reset() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + frame_transformer_->UnregisterTransformedFrameCallback(); + frame_transformer_ = nullptr; + receive_frame_callback_ = ReceiveFrameCallback(); +} + +void ChannelReceiveFrameTransformerDelegate::Transform( + rtc::ArrayView packet, + const RTPHeader& header, + uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + frame_transformer_->Transform( + std::make_unique(packet, header, ssrc)); +} + +void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame( + std::unique_ptr frame) { + rtc::scoped_refptr delegate = this; + channel_receive_thread_->PostTask(ToQueuedTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->ReceiveFrame(std::move(frame)); + })); +} + +void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( + std::unique_ptr frame) const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + if (!receive_frame_callback_) + return; + auto* transformed_frame = static_cast(frame.get()); + receive_frame_callback_(transformed_frame->GetData(), + transformed_frame->GetHeader()); +} +} // namespace webrtc diff --git a/audio/channel_receive_frame_transformer_delegate.h b/audio/channel_receive_frame_transformer_delegate.h new file mode 100644 index 0000000000..547946f973 --- /dev/null +++ b/audio/channel_receive_frame_transformer_delegate.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_ +#define AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_ + +#include + +#include "api/frame_transformer_interface.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/thread.h" + +namespace webrtc { + +// Delegates calls to FrameTransformerInterface to transform frames, and to +// ChannelReceive to receive the transformed frames using the +// |receive_frame_callback_| on the |channel_receive_thread_|. +class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { + public: + using ReceiveFrameCallback = + std::function packet, + const RTPHeader& header)>; + ChannelReceiveFrameTransformerDelegate( + ReceiveFrameCallback receive_frame_callback, + rtc::scoped_refptr frame_transformer, + rtc::Thread* channel_receive_thread); + + // Registers |this| as callback for |frame_transformer_|, to get the + // transformed frames. + void Init(); + + // Unregisters and releases the |frame_transformer_| reference, and resets + // |receive_frame_callback_| on |channel_receive_thread_|. Called from + // ChannelReceive destructor to prevent running the callback on a dangling + // channel. + void Reset(); + + // Delegates the call to FrameTransformerInterface::Transform, to transform + // the frame asynchronously. + void Transform(rtc::ArrayView packet, + const RTPHeader& header, + uint32_t ssrc); + + // Implements TransformedFrameCallback. Can be called on any thread. + void OnTransformedFrame( + std::unique_ptr frame) override; + + // Delegates the call to ChannelReceive::ReceivePacket on the + // |channel_receive_thread_|, by calling |receive_frame_callback_|. + void ReceiveFrame(std::unique_ptr frame) const; + + protected: + ~ChannelReceiveFrameTransformerDelegate() override = default; + + private: + SequenceChecker sequence_checker_; + ReceiveFrameCallback receive_frame_callback_ + RTC_GUARDED_BY(sequence_checker_); + rtc::scoped_refptr frame_transformer_ + RTC_GUARDED_BY(sequence_checker_); + rtc::Thread* channel_receive_thread_; +}; + +} // namespace webrtc +#endif // AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_