diff --git a/api/media_transport_interface.cc b/api/media_transport_interface.cc index 13bc98297f..fc97b0b503 100644 --- a/api/media_transport_interface.cc +++ b/api/media_transport_interface.cc @@ -19,6 +19,8 @@ namespace webrtc { +MediaTransportEncodedAudioFrame::~MediaTransportEncodedAudioFrame() {} + MediaTransportEncodedAudioFrame::MediaTransportEncodedAudioFrame( int sampling_rate_hz, int starting_sample_index, @@ -35,7 +37,19 @@ MediaTransportEncodedAudioFrame::MediaTransportEncodedAudioFrame( payload_type_(payload_type), encoded_data_(std::move(encoded_data)) {} -MediaTransportEncodedAudioFrame::~MediaTransportEncodedAudioFrame() = default; +MediaTransportEncodedAudioFrame& MediaTransportEncodedAudioFrame::operator=( + const MediaTransportEncodedAudioFrame&) = default; + +MediaTransportEncodedAudioFrame& MediaTransportEncodedAudioFrame::operator=( + MediaTransportEncodedAudioFrame&&) = default; + +MediaTransportEncodedAudioFrame::MediaTransportEncodedAudioFrame( + const MediaTransportEncodedAudioFrame&) = default; + +MediaTransportEncodedAudioFrame::MediaTransportEncodedAudioFrame( + MediaTransportEncodedAudioFrame&&) = default; + +MediaTransportEncodedVideoFrame::~MediaTransportEncodedVideoFrame() {} MediaTransportEncodedVideoFrame::MediaTransportEncodedVideoFrame( int64_t frame_id, @@ -47,6 +61,16 @@ MediaTransportEncodedVideoFrame::MediaTransportEncodedVideoFrame( frame_id_(frame_id), referenced_frame_ids_(std::move(referenced_frame_ids)) {} -MediaTransportEncodedVideoFrame::~MediaTransportEncodedVideoFrame() = default; +MediaTransportEncodedVideoFrame& MediaTransportEncodedVideoFrame::operator=( + const MediaTransportEncodedVideoFrame&) = default; + +MediaTransportEncodedVideoFrame& MediaTransportEncodedVideoFrame::operator=( + MediaTransportEncodedVideoFrame&&) = default; + +MediaTransportEncodedVideoFrame::MediaTransportEncodedVideoFrame( + const MediaTransportEncodedVideoFrame&) = default; + +MediaTransportEncodedVideoFrame::MediaTransportEncodedVideoFrame( + MediaTransportEncodedVideoFrame&&) = default; } // namespace webrtc diff --git a/api/media_transport_interface.h b/api/media_transport_interface.h index 154fe4a825..33fa2adf6c 100644 --- a/api/media_transport_interface.h +++ b/api/media_transport_interface.h @@ -1,5 +1,4 @@ -/* - * Copyright 2018 The WebRTC project authors. All Rights Reserved. +/* Copyright 2018 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -35,7 +34,7 @@ namespace webrtc { // Represents encoded audio frame in any encoding (type of encoding is opaque). // To avoid copying of encoded data use move semantics when passing by value. -class MediaTransportEncodedAudioFrame { +class MediaTransportEncodedAudioFrame final { public: enum class FrameType { // Normal audio frame (equivalent to webrtc::kAudioFrameSpeech). @@ -79,6 +78,12 @@ class MediaTransportEncodedAudioFrame { std::vector encoded_data); ~MediaTransportEncodedAudioFrame(); + MediaTransportEncodedAudioFrame(const MediaTransportEncodedAudioFrame&); + MediaTransportEncodedAudioFrame& operator=( + const MediaTransportEncodedAudioFrame& other); + MediaTransportEncodedAudioFrame& operator=( + MediaTransportEncodedAudioFrame&& other); + MediaTransportEncodedAudioFrame(MediaTransportEncodedAudioFrame&&); // Getters. int sampling_rate_hz() const { return sampling_rate_hz_; } @@ -121,13 +126,19 @@ class MediaTransportAudioSinkInterface { }; // Represents encoded video frame, along with the codec information. -class MediaTransportEncodedVideoFrame { +class MediaTransportEncodedVideoFrame final { public: MediaTransportEncodedVideoFrame(int64_t frame_id, std::vector referenced_frame_ids, VideoCodecType codec_type, const webrtc::EncodedImage& encoded_image); ~MediaTransportEncodedVideoFrame(); + MediaTransportEncodedVideoFrame(const MediaTransportEncodedVideoFrame&); + MediaTransportEncodedVideoFrame& operator=( + const MediaTransportEncodedVideoFrame& other); + MediaTransportEncodedVideoFrame& operator=( + MediaTransportEncodedVideoFrame&& other); + MediaTransportEncodedVideoFrame(MediaTransportEncodedVideoFrame&&); VideoCodecType codec_type() const { return codec_type_; } const webrtc::EncodedImage& encoded_image() const { return encoded_image_; } diff --git a/api/peerconnectioninterface.h b/api/peerconnectioninterface.h index c6850e17ee..3d8a9c1e41 100644 --- a/api/peerconnectioninterface.h +++ b/api/peerconnectioninterface.h @@ -80,6 +80,7 @@ #include "api/datachannelinterface.h" #include "api/fec_controller.h" #include "api/jsep.h" +#include "api/media_transport_interface.h" #include "api/mediastreaminterface.h" #include "api/rtcerror.h" #include "api/rtceventlogoutput.h" @@ -560,6 +561,12 @@ class PeerConnectionInterface : public rtc::RefCountInterface { // correctly. This flag will be deprecated soon. Do not rely on it. bool active_reset_srtp_params = false; + // If MediaTransportFactory is provided in PeerConnectionFactory, this flag + // informs PeerConnection that it should use the MediaTransportInterface. + // It's invalid to set it to |true| if the MediaTransportFactory wasn't + // provided. + bool use_media_transport = false; + // // Don't forget to update operator== if adding something. // @@ -1153,6 +1160,7 @@ struct PeerConnectionFactoryDependencies final { std::unique_ptr event_log_factory; std::unique_ptr fec_controller_factory; std::unique_ptr network_controller_factory; + std::unique_ptr media_transport_factory; }; // PeerConnectionFactoryInterface is the factory interface used for creating diff --git a/pc/createpeerconnectionfactory.cc b/pc/createpeerconnectionfactory.cc index bbc4ebbe73..b1db69091b 100644 --- a/pc/createpeerconnectionfactory.cc +++ b/pc/createpeerconnectionfactory.cc @@ -129,10 +129,14 @@ rtc::scoped_refptr CreatePeerConnectionFactory( std::unique_ptr event_log_factory = CreateRtcEventLogFactory(); - - return CreateModularPeerConnectionFactory( - network_thread, worker_thread, signaling_thread, std::move(media_engine), - std::move(call_factory), std::move(event_log_factory)); + PeerConnectionFactoryDependencies dependencies; + dependencies.network_thread = network_thread; + dependencies.worker_thread = worker_thread; + dependencies.signaling_thread = signaling_thread; + dependencies.media_engine = std::move(media_engine); + dependencies.call_factory = std::move(call_factory); + dependencies.event_log_factory = std::move(event_log_factory); + return CreateModularPeerConnectionFactory(std::move(dependencies)); } #if defined(USE_BUILTIN_SW_CODECS) diff --git a/pc/peerconnection.cc b/pc/peerconnection.cc index f920ddf72c..7430898912 100644 --- a/pc/peerconnection.cc +++ b/pc/peerconnection.cc @@ -705,6 +705,7 @@ bool PeerConnectionInterface::RTCConfiguration::operator==( SdpSemantics sdp_semantics; absl::optional network_preference; bool active_reset_srtp_params; + bool use_media_transport; }; static_assert(sizeof(stuff_being_tested_for_equality) == sizeof(*this), "Did you add something to RTCConfiguration and forget to " @@ -752,7 +753,8 @@ bool PeerConnectionInterface::RTCConfiguration::operator==( turn_customizer == o.turn_customizer && sdp_semantics == o.sdp_semantics && network_preference == o.network_preference && - active_reset_srtp_params == o.active_reset_srtp_params; + active_reset_srtp_params == o.active_reset_srtp_params && + use_media_transport == o.use_media_transport; } bool PeerConnectionInterface::RTCConfiguration::operator!=( diff --git a/pc/peerconnection.h b/pc/peerconnection.h index 37c9d73bd4..b42d620107 100644 --- a/pc/peerconnection.h +++ b/pc/peerconnection.h @@ -17,6 +17,7 @@ #include #include +#include "api/media_transport_interface.h" #include "api/peerconnectioninterface.h" #include "api/turncustomizer.h" #include "pc/iceserverparsing.h" @@ -1026,6 +1027,10 @@ class PeerConnection : public PeerConnectionInternal, // List of content names for which the remote side triggered an ICE restart. std::set pending_ice_restarts_; + // Optional media transport for sending / receiving encoded frames. + // If available, media transport will be used instead of RTP / SRTP. + std::unique_ptr media_transport_factory_; + std::unique_ptr webrtc_session_desc_factory_; // Member variables for caching global options. diff --git a/pc/peerconnectionfactory.cc b/pc/peerconnectionfactory.cc index b78ed8db1e..edc13034a3 100644 --- a/pc/peerconnectionfactory.cc +++ b/pc/peerconnectionfactory.cc @@ -15,6 +15,7 @@ #include "absl/memory/memory.h" #include "api/fec_controller.h" +#include "api/media_transport_interface.h" #include "api/mediaconstraintsinterface.h" #include "api/mediastreamproxy.h" #include "api/mediastreamtrackproxy.h" @@ -187,7 +188,9 @@ PeerConnectionFactory::PeerConnectionFactory( std::move(dependencies.call_factory), std::move(dependencies.event_log_factory), std::move(dependencies.fec_controller_factory), - std::move(dependencies.network_controller_factory)) {} + std::move(dependencies.network_controller_factory)) { + media_transport_factory_ = std::move(dependencies.media_transport_factory); +} PeerConnectionFactory::~PeerConnectionFactory() { RTC_DCHECK(signaling_thread_->IsCurrent()); diff --git a/pc/peerconnectionfactory.h b/pc/peerconnectionfactory.h index f36a4e0c04..1c1ea94fe2 100644 --- a/pc/peerconnectionfactory.h +++ b/pc/peerconnectionfactory.h @@ -15,6 +15,7 @@ #include #include +#include "api/media_transport_interface.h" #include "api/mediastreaminterface.h" #include "api/peerconnectioninterface.h" #include "media/sctp/sctptransportinternal.h" @@ -96,6 +97,10 @@ class PeerConnectionFactory : public PeerConnectionFactoryInterface { virtual rtc::Thread* network_thread(); const Options& options() const { return options_; } + MediaTransportFactory* media_transport_factory() { + return media_transport_factory_.get(); + } + protected: PeerConnectionFactory( rtc::Thread* network_thread, @@ -148,6 +153,7 @@ class PeerConnectionFactory : public PeerConnectionFactoryInterface { injected_network_controller_factory_; std::unique_ptr bbr_network_controller_factory_; + std::unique_ptr media_transport_factory_; }; } // namespace webrtc diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index c4bee38d6c..b259f51ef4 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -1170,6 +1170,7 @@ if (is_ios || is_mac) { ":videoframebuffer_objc", ":videosource_objc", ":videotoolbox_objc", + "../api:libjingle_peerconnection_api", "../api/audio_codecs:audio_codecs_api", "../api/audio_codecs:builtin_audio_decoder_factory", "../api/audio_codecs:builtin_audio_encoder_factory", diff --git a/sdk/objc/api/peerconnection/RTCConfiguration.h b/sdk/objc/api/peerconnection/RTCConfiguration.h index 9fb8b4df35..c7998a5aa2 100644 --- a/sdk/objc/api/peerconnection/RTCConfiguration.h +++ b/sdk/objc/api/peerconnection/RTCConfiguration.h @@ -168,6 +168,12 @@ RTC_OBJC_EXPORT */ @property(nonatomic, assign) BOOL activeResetSrtpParams; +/** + * If MediaTransportFactory is provided in PeerConnectionFactory, this flag informs PeerConnection + * that it should use the MediaTransportInterface. + */ +@property(nonatomic, assign) BOOL useMediaTransport; + - (instancetype)init; @end diff --git a/sdk/objc/api/peerconnection/RTCConfiguration.mm b/sdk/objc/api/peerconnection/RTCConfiguration.mm index 73a0fa9aa5..bdf4fb6e43 100644 --- a/sdk/objc/api/peerconnection/RTCConfiguration.mm +++ b/sdk/objc/api/peerconnection/RTCConfiguration.mm @@ -48,6 +48,7 @@ @synthesize sdpSemantics = _sdpSemantics; @synthesize turnCustomizer = _turnCustomizer; @synthesize activeResetSrtpParams = _activeResetSrtpParams; +@synthesize useMediaTransport = _useMediaTransport; - (instancetype)init { // Copy defaults. @@ -117,7 +118,7 @@ - (NSString *)description { static NSString *formatString = @"RTCConfiguration: " - @"{\n%@\n%@\n%@\n%@\n%@\n%@\n%@\n%@\n%d\n%d\n%d\n%d\n%d\n%d\n%d\n%@\n%@\n%d\n%d\n%d\n}\n"; + @"{\n%@\n%@\n%@\n%@\n%@\n%@\n%@\n%@\n%d\n%d\n%d\n%d\n%d\n%d\n%d\n%@\n%@\n%d\n%d\n%d\n%@\n}\n"; return [NSString stringWithFormat:formatString, @@ -140,7 +141,8 @@ _iceRegatherIntervalRange, _disableLinkLocalNetworks, _maxIPv6Networks, - _activeResetSrtpParams]; + _activeResetSrtpParams, + _useMediaTransport]; } #pragma mark - Private diff --git a/sdk/objc/api/peerconnection/RTCPeerConnection.mm b/sdk/objc/api/peerconnection/RTCPeerConnection.mm index 5277489747..1ac076b3c2 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnection.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnection.mm @@ -29,6 +29,7 @@ #include #include "api/jsepicecandidate.h" +#include "api/media_transport_interface.h" #include "rtc_base/checks.h" NSString * const kRTCPeerConnectionErrorDomain = diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h index 60aa08f7ca..2752cf4117 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h @@ -17,6 +17,7 @@ namespace webrtc { class AudioDeviceModule; class AudioEncoderFactory; class AudioDecoderFactory; +class MediaTransportFactory; class VideoEncoderFactory; class VideoDecoderFactory; class AudioProcessing; @@ -49,6 +50,25 @@ NS_ASSUME_NONNULL_BEGIN audioProcessingModule: (rtc::scoped_refptr)audioProcessingModule; +- (instancetype) + initWithNativeAudioEncoderFactory: + (rtc::scoped_refptr)audioEncoderFactory + nativeAudioDecoderFactory: + (rtc::scoped_refptr)audioDecoderFactory + nativeVideoEncoderFactory: + (std::unique_ptr)videoEncoderFactory + nativeVideoDecoderFactory: + (std::unique_ptr)videoDecoderFactory + audioDeviceModule:(nullable webrtc::AudioDeviceModule *)audioDeviceModule + audioProcessingModule: + (rtc::scoped_refptr)audioProcessingModule + mediaTransportFactory: + (std::unique_ptr)mediaTransportFactory; + +- (instancetype)initWithEncoderFactory:(nullable id)encoderFactory + decoderFactory:(nullable id)decoderFactory + mediaTransportFactory: + (std::unique_ptr)mediaTransportFactory; @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index 403db040fb..48476f2368 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -50,6 +50,7 @@ // TODO(zhihuang): Remove nogncheck once MediaEngineInterface is moved to C++ // API layer. #include "absl/memory/memory.h" +#include "api/media_transport_interface.h" #include "media/engine/webrtcmediaengine.h" // nogncheck @implementation RTCPeerConnectionFactory { @@ -80,12 +81,15 @@ nativeVideoDecoderFactory:webrtc::ObjCToNativeVideoDecoderFactory( [[RTCVideoDecoderFactoryH264 alloc] init]) audioDeviceModule:[self audioDeviceModule] - audioProcessingModule:nullptr]; + audioProcessingModule:nullptr + mediaTransportFactory:nullptr]; #endif } - (instancetype)initWithEncoderFactory:(nullable id)encoderFactory - decoderFactory:(nullable id)decoderFactory { + decoderFactory:(nullable id)decoderFactory + mediaTransportFactory: + (std::unique_ptr)mediaTransportFactory { #ifdef HAVE_NO_MEDIA return [self initWithNoMedia]; #else @@ -102,9 +106,16 @@ nativeVideoEncoderFactory:std::move(native_encoder_factory) nativeVideoDecoderFactory:std::move(native_decoder_factory) audioDeviceModule:[self audioDeviceModule] - audioProcessingModule:nullptr]; + audioProcessingModule:nullptr + mediaTransportFactory:std::move(mediaTransportFactory)]; #endif } +- (instancetype)initWithEncoderFactory:(nullable id)encoderFactory + decoderFactory:(nullable id)decoderFactory { + return [self initWithEncoderFactory:encoderFactory + decoderFactory:decoderFactory + mediaTransportFactory:nullptr]; +} - (instancetype)initNative { if (self = [super init]) { @@ -152,20 +163,57 @@ (nullable webrtc::AudioDeviceModule *)audioDeviceModule audioProcessingModule: (rtc::scoped_refptr)audioProcessingModule { + return [self initWithNativeAudioEncoderFactory:audioEncoderFactory + nativeAudioDecoderFactory:audioDecoderFactory + nativeVideoEncoderFactory:std::move(videoEncoderFactory) + nativeVideoDecoderFactory:std::move(videoDecoderFactory) + audioDeviceModule:audioDeviceModule + audioProcessingModule:audioProcessingModule + mediaTransportFactory:nullptr]; +} + +- (instancetype) + initWithNativeAudioEncoderFactory: + (rtc::scoped_refptr)audioEncoderFactory + nativeAudioDecoderFactory: + (rtc::scoped_refptr)audioDecoderFactory + nativeVideoEncoderFactory: + (std::unique_ptr)videoEncoderFactory + nativeVideoDecoderFactory: + (std::unique_ptr)videoDecoderFactory + audioDeviceModule:(nullable webrtc::AudioDeviceModule *)audioDeviceModule + audioProcessingModule: + (rtc::scoped_refptr)audioProcessingModule + mediaTransportFactory: + (std::unique_ptr)mediaTransportFactory { #ifdef HAVE_NO_MEDIA return [self initWithNoMedia]; #else if (self = [self initNative]) { - _nativeFactory = webrtc::CreatePeerConnectionFactory(_networkThread.get(), - _workerThread.get(), - _signalingThread.get(), - audioDeviceModule, - audioEncoderFactory, - audioDecoderFactory, - std::move(videoEncoderFactory), - std::move(videoDecoderFactory), - nullptr, // audio mixer - audioProcessingModule); + if (!audioProcessingModule) audioProcessingModule = webrtc::AudioProcessingBuilder().Create(); + + std::unique_ptr media_engine = + cricket::WebRtcMediaEngineFactory::Create(audioDeviceModule, + audioEncoderFactory, + audioDecoderFactory, + std::move(videoEncoderFactory), + std::move(videoDecoderFactory), + nullptr, // audio mixer + audioProcessingModule); + + std::unique_ptr call_factory = webrtc::CreateCallFactory(); + + std::unique_ptr event_log_factory = + webrtc::CreateRtcEventLogFactory(); + webrtc::PeerConnectionFactoryDependencies dependencies; + dependencies.network_thread = _networkThread.get(); + dependencies.worker_thread = _workerThread.get(); + dependencies.signaling_thread = _signalingThread.get(); + dependencies.media_engine = std::move(media_engine); + dependencies.call_factory = std::move(call_factory); + dependencies.event_log_factory = std::move(event_log_factory); + dependencies.media_transport_factory = std::move(mediaTransportFactory); + _nativeFactory = webrtc::CreateModularPeerConnectionFactory(std::move(dependencies)); NSAssert(_nativeFactory, @"Failed to initialize PeerConnectionFactory!"); } return self; diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm index a26a639e28..0adaa30e4f 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm @@ -13,6 +13,7 @@ #include "api/audio_codecs/audio_decoder_factory.h" #include "api/audio_codecs/audio_encoder_factory.h" +#include "api/media_transport_interface.h" #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" #include "modules/audio_device/include/audio_device.h" @@ -25,6 +26,7 @@ rtc::scoped_refptr _audioDecoderFactory; rtc::scoped_refptr _audioDeviceModule; rtc::scoped_refptr _audioProcessingModule; + std::unique_ptr _mediaTransportFactory; } + (RTCPeerConnectionFactoryBuilder *)builder { @@ -38,7 +40,8 @@ nativeVideoEncoderFactory:std::move(_videoEncoderFactory) nativeVideoDecoderFactory:std::move(_videoDecoderFactory) audioDeviceModule:_audioDeviceModule - audioProcessingModule:_audioProcessingModule]; + audioProcessingModule:_audioProcessingModule + mediaTransportFactory:std::move(_mediaTransportFactory)]; } - (void)setVideoEncoderFactory:(std::unique_ptr)videoEncoderFactory { diff --git a/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm b/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm index 9bb5179521..5f889a699d 100644 --- a/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm +++ b/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm @@ -22,6 +22,7 @@ extern "C" { #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/media_transport_interface.h" #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" #include "modules/audio_device/include/audio_device.h" @@ -49,7 +50,8 @@ extern "C" { nativeVideoEncoderFactory:nullptr nativeVideoDecoderFactory:nullptr audioDeviceModule:nullptr - audioProcessingModule:nullptr]); + audioProcessingModule:nullptr + mediaTransportFactory:nullptr]); #endif RTCPeerConnectionFactoryBuilder* builder = [[RTCPeerConnectionFactoryBuilder alloc] init]; RTCPeerConnectionFactory* peerConnectionFactory = [builder createPeerConnectionFactory]; @@ -69,7 +71,8 @@ extern "C" { nativeVideoEncoderFactory:nullptr nativeVideoDecoderFactory:nullptr audioDeviceModule:nullptr - audioProcessingModule:nullptr]); + audioProcessingModule:nullptr + mediaTransportFactory:nullptr]); #endif RTCPeerConnectionFactoryBuilder* builder = [RTCPeerConnectionFactoryBuilder defaultBuilder]; RTCPeerConnectionFactory* peerConnectionFactory = [builder createPeerConnectionFactory];