diff --git a/webrtc/examples/BUILD.gn b/webrtc/examples/BUILD.gn index 004b9a2786..655d6ff0a5 100644 --- a/webrtc/examples/BUILD.gn +++ b/webrtc/examples/BUILD.gn @@ -620,6 +620,8 @@ if (is_win) { "unityplugin/simple_peer_connection.h", "unityplugin/unity_plugin_apis.cc", "unityplugin/unity_plugin_apis.h", + "unityplugin/video_observer.cc", + "unityplugin/video_observer.h", ] if (!build_with_chromium && is_clang) { # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). @@ -637,9 +639,6 @@ if (is_win) { "../media:rtc_media_base", "../modules/video_capture:video_capture_module", "../pc:libjingle_peerconnection", - "../rtc_base:rtc_base", - "../rtc_base:rtc_base_approved", - "../rtc_base:rtc_json", "../system_wrappers:field_trial_default", "../system_wrappers:metrics_default", ] diff --git a/webrtc/examples/unityplugin/README b/webrtc/examples/unityplugin/README index eade9ef5fa..82a466ac0e 100644 --- a/webrtc/examples/unityplugin/README +++ b/webrtc/examples/unityplugin/README @@ -5,200 +5,302 @@ This plugin dll can also be used by Windows C# applications other than Unity. An example of wrapping native plugin into a C# managed class in Unity is given as following: using System; +using System.Collections.Generic; using System.Runtime.InteropServices; namespace SimplePeerConnectionM { - // This is a managed wrap up class for the native c style peer connection APIs. + // A class for ice candidate. + public class IceCandidate { + public IceCandidate(string candidate, int sdpMlineIndex, string sdpMid) { + mCandidate = candidate; + mSdpMlineIndex = sdpMlineIndex; + mSdpMid = sdpMid; + } + string mCandidate; + int mSdpMlineIndex; + string mSdpMid; + + public string Candidate { + get { return mCandidate; } + set { mCandidate = value; } + } + + public int SdpMlineIndex { + get { return mSdpMlineIndex; } + set { mSdpMlineIndex = value; } + } + + public string SdpMid { + get { return mSdpMid; } + set { mSdpMid = value; } + } + } + + // A managed wrapper up class for the native c style peer connection APIs. public class PeerConnectionM { - //private const string dll_path = "SimplePeerConnection"; - private const string dll_path = "webrtc_unity_plugin"; + private const string dllPath = "webrtc_unity_plugin"; - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern int CreatePeerConnection(); + //create a peerconnection with turn servers + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern int CreatePeerConnection(string[] turnUrls, int noOfUrls, + string username, string credential); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool ClosePeerConnection(int peer_connection_id); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool ClosePeerConnection(int peerConnectionId); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool AddStream(int peer_connection_id, bool audio_only); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool AddStream(int peerConnectionId, bool audioOnly); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool AddDataChannel(int peer_connection_id); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool AddDataChannel(int peerConnectionId); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool CreateOffer(int peer_connection_id); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool CreateOffer(int peerConnectionId); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool CreateAnswer(int peer_connection_id); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool CreateAnswer(int peerConnectionId); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool SendDataViaDataChannel(int peer_connection_id, string data); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool SendDataViaDataChannel(int peerConnectionId, string data); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool SetAudioControl(int peer_connection_id, bool is_mute, bool is_record); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool SetAudioControl(int peerConnectionId, bool isMute, bool isRecord); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate void LocalDataChannelReadyInternalDelegate(); public delegate void LocalDataChannelReadyDelegate(int id); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool RegisterOnLocalDataChannelReady(int peer_connection_id, LocalDataChannelReadyInternalDelegate callback); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RegisterOnLocalDataChannelReady( + int peerConnectionId, LocalDataChannelReadyInternalDelegate callback); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate void DataFromDataChannelReadyInternalDelegate(string s); public delegate void DataFromDataChannelReadyDelegate(int id, string s); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool RegisterOnDataFromDataChannelReady(int peer_connection_id, DataFromDataChannelReadyInternalDelegate callback); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RegisterOnDataFromDataChannelReady( + int peerConnectionId, DataFromDataChannelReadyInternalDelegate callback); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate void FailureMessageInternalDelegate(string msg); public delegate void FailureMessageDelegate(int id, string msg); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool RegisterOnFailure(int peer_connection_id, FailureMessageInternalDelegate callback); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RegisterOnFailure(int peerConnectionId, + FailureMessageInternalDelegate callback); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - private delegate void AudioBusReadyInternalDelegate(IntPtr data, int bits_per_sample, - int sample_rate, int number_of_channels, int number_of_frames); - public delegate void AudioBusReadyDelegate(int id, IntPtr data, int bits_per_sample, - int sample_rate, int number_of_channels, int number_of_frames); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool RegisterOnAudioBusReady(int peer_connection_id, AudioBusReadyInternalDelegate callback); + private delegate void AudioBusReadyInternalDelegate(IntPtr data, int bitsPerSample, + int sampleRate, int numberOfChannels, int numberOfFrames); + public delegate void AudioBusReadyDelegate(int id, IntPtr data, int bitsPerSample, + int sampleRate, int numberOfChannels, int numberOfFrames); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RegisterOnAudioBusReady(int peerConnectionId, + AudioBusReadyInternalDelegate callback); + + // Video callbacks. + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate void I420FrameReadyInternalDelegate( + IntPtr dataY, IntPtr dataU, IntPtr dataV, + int strideY, int strideU, int strideV, + uint width, uint height); + public delegate void I420FrameReadyDelegate(int id, + IntPtr dataY, IntPtr dataU, IntPtr dataV, + int strideY, int strideU, int strideV, + uint width, uint height); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RegisterOnLocalI420FrameReady(int peerConnectionId, + I420FrameReadyInternalDelegate callback); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RegisterOnRemoteI420FrameReady(int peerConnectionId, + I420FrameReadyInternalDelegate callback); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - private delegate void LocalSdpReadytoSendInternalDelegate(string s); - public delegate void LocalSdpReadytoSendDelegate(int id, string s); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool RegisterOnLocalSdpReadytoSend(int peer_connection_id, LocalSdpReadytoSendInternalDelegate callback); + private delegate void LocalSdpReadytoSendInternalDelegate(string type, string sdp); + public delegate void LocalSdpReadytoSendDelegate(int id, string type, string sdp); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RegisterOnLocalSdpReadytoSend(int peerConnectionId, + LocalSdpReadytoSendInternalDelegate callback); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - private delegate void IceCandiateReadytoSendInternalDelegate(string s); - public delegate void IceCandiateReadytoSendDelegate(int id, string s); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool RegisterOnIceCandiateReadytoSend(int peer_connection_id, IceCandiateReadytoSendInternalDelegate callback); + private delegate void IceCandiateReadytoSendInternalDelegate( + string candidate, int sdpMlineIndex, string sdpMid); + public delegate void IceCandiateReadytoSendDelegate( + int id, string candidate, int sdpMlineIndex, string sdpMid); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RegisterOnIceCandiateReadytoSend( + int peerConnectionId, IceCandiateReadytoSendInternalDelegate callback); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern int ReceivedSdp(int peer_connection_id, string sdp); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool SetRemoteDescription(int peerConnectionId, string type, string sdp); - [DllImport(dll_path, CallingConvention = CallingConvention.Cdecl)] - private static extern bool ReceivedIceCandidate(int peer_connection_id, string ice_candidate); + [DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)] + private static extern bool AddIceCandidate(int peerConnectionId, string sdp, + int sdpMlineindex, string sdpMid); - public void CreatePeerConnectionM() { - peer_connection_id_ = CreatePeerConnection(); + public PeerConnectionM(List turnUrls, string username, string credential) { + string[] urls = turnUrls != null ? turnUrls.ToArray() : null; + int length = turnUrls != null ? turnUrls.Count : 0; + mPeerConnectionId = CreatePeerConnection(urls, length, username, credential); RegisterCallbacks(); } - private void RegisterCallbacks() { - localDataChannelReadyDelegate_ = new LocalDataChannelReadyInternalDelegate(RaiseLocalDataChannelReady); - RegisterOnLocalDataChannelReady(peer_connection_id_, localDataChannelReadyDelegate_); - - dataFromDataChannelReadyDelegate_ = new DataFromDataChannelReadyInternalDelegate(RaiseDataFromDataChannelReady); - RegisterOnDataFromDataChannelReady(peer_connection_id_, dataFromDataChannelReadyDelegate_); - - failureMessageDelegate_ = new FailureMessageInternalDelegate(RaiseFailureMessage); - RegisterOnFailure(peer_connection_id_, failureMessageDelegate_); - - audioBusReadyDelegate_ = new AudioBusReadyInternalDelegate(RaiseAudioBusReady); - RegisterOnAudioBusReady(peer_connection_id_, audioBusReadyDelegate_); - - localSdpReadytoSendDelegate_ = new LocalSdpReadytoSendInternalDelegate(RaiseLocalSdpReadytoSend); - RegisterOnLocalSdpReadytoSend(peer_connection_id_, localSdpReadytoSendDelegate_); - - iceCandiateReadytoSendDelegate_ = new IceCandiateReadytoSendInternalDelegate(RaiseIceCandiateReadytoSend); - RegisterOnIceCandiateReadytoSend(peer_connection_id_, iceCandiateReadytoSendDelegate_); - } - - public void ClosePeerConnectionM() { - ClosePeerConnection(peer_connection_id_); - peer_connection_id_ = -1; + public void ClosePeerConnection() { + ClosePeerConnection(mPeerConnectionId); + mPeerConnectionId = -1; } // Return -1 if Peerconnection is not available. public int GetUniqueId() { - return peer_connection_id_; + return mPeerConnectionId; } - public void AddStreamM(bool audio_only) { - AddStream(peer_connection_id_, audio_only); + public void AddStream(bool audioOnly) { + AddStream(mPeerConnectionId, audioOnly); } - public void AddDataChannelM() { - AddDataChannel(peer_connection_id_); + public void AddDataChannel() { + AddDataChannel(mPeerConnectionId); } - public void CreateOfferM() { - CreateOffer(peer_connection_id_); + public void CreateOffer() { + CreateOffer(mPeerConnectionId); } - public void CreateAnswerM() { - CreateAnswer(peer_connection_id_); + public void CreateAnswer() { + CreateAnswer(mPeerConnectionId); } - public void SendDataViaDataChannelM(string data) { - SendDataViaDataChannel(peer_connection_id_, data); + public void SendDataViaDataChannel(string data) { + SendDataViaDataChannel(mPeerConnectionId, data); } - public void SetAudioControl(bool is_mute, bool is_record) { - SetAudioControl(peer_connection_id_, is_mute, is_record); + public void SetAudioControl(bool isMute, bool isRecord) { + SetAudioControl(mPeerConnectionId, isMute, isRecord); } - public void ReceivedSdpM(string sdp) { - peer_connection_id_ = ReceivedSdp(peer_connection_id_, sdp); - RegisterCallbacks(); + public void SetRemoteDescription(string type, string sdp) { + SetRemoteDescription(mPeerConnectionId, type, sdp); } - public void ReceivedIceCandidateM(string ice_candidate) { - ReceivedIceCandidate(peer_connection_id_, ice_candidate); + public void AddIceCandidate(string candidate, int sdpMlineindex, string sdpMid) { + AddIceCandidate(mPeerConnectionId, candidate, sdpMlineindex, sdpMid); + } + + private void RegisterCallbacks() { + localDataChannelReadyDelegate = new LocalDataChannelReadyInternalDelegate( + RaiseLocalDataChannelReady); + RegisterOnLocalDataChannelReady(mPeerConnectionId, localDataChannelReadyDelegate); + + dataFromDataChannelReadyDelegate = new DataFromDataChannelReadyInternalDelegate( + RaiseDataFromDataChannelReady); + RegisterOnDataFromDataChannelReady(mPeerConnectionId, dataFromDataChannelReadyDelegate); + + failureMessageDelegate = new FailureMessageInternalDelegate(RaiseFailureMessage); + RegisterOnFailure(mPeerConnectionId, failureMessageDelegate); + + audioBusReadyDelegate = new AudioBusReadyInternalDelegate(RaiseAudioBusReady); + RegisterOnAudioBusReady(mPeerConnectionId, audioBusReadyDelegate); + + localI420FrameReadyDelegate = new I420FrameReadyInternalDelegate( + RaiseLocalVideoFrameReady); + RegisterOnLocalI420FrameReady(mPeerConnectionId, localI420FrameReadyDelegate); + + remoteI420FrameReadyDelegate = new I420FrameReadyInternalDelegate( + RaiseRemoteVideoFrameReady); + RegisterOnRemoteI420FrameReady(mPeerConnectionId, remoteI420FrameReadyDelegate); + + localSdpReadytoSendDelegate = new LocalSdpReadytoSendInternalDelegate( + RaiseLocalSdpReadytoSend); + RegisterOnLocalSdpReadytoSend(mPeerConnectionId, localSdpReadytoSendDelegate); + + iceCandiateReadytoSendDelegate = + new IceCandiateReadytoSendInternalDelegate(RaiseIceCandiateReadytoSend); + RegisterOnIceCandiateReadytoSend( + mPeerConnectionId, iceCandiateReadytoSendDelegate); } private void RaiseLocalDataChannelReady() { if (OnLocalDataChannelReady != null) - OnLocalDataChannelReady(peer_connection_id_); + OnLocalDataChannelReady(mPeerConnectionId); } private void RaiseDataFromDataChannelReady(string data) { if (OnDataFromDataChannelReady != null) - OnDataFromDataChannelReady(peer_connection_id_, data); + OnDataFromDataChannelReady(mPeerConnectionId, data); } private void RaiseFailureMessage(string msg) { if (OnFailureMessage != null) - OnFailureMessage(peer_connection_id_, msg); + OnFailureMessage(mPeerConnectionId, msg); } - private void RaiseAudioBusReady(IntPtr data, int bits_per_sample, - int sample_rate, int number_of_channels, int number_of_frames) { + private void RaiseAudioBusReady(IntPtr data, int bitsPerSample, + int sampleRate, int numberOfChannels, int numberOfFrames) { if (OnAudioBusReady != null) - OnAudioBusReady(peer_connection_id_, data, bits_per_sample, sample_rate, - number_of_channels, number_of_frames); + OnAudioBusReady(mPeerConnectionId, data, bitsPerSample, sampleRate, + numberOfChannels, numberOfFrames); } - private void RaiseLocalSdpReadytoSend(string msg) { + private void RaiseLocalVideoFrameReady( + IntPtr dataY, IntPtr dataU, IntPtr dataV, + int strideY, int strideU, int strideV, + uint width, uint height) { + if (OnLocalVideoFrameReady != null) + OnLocalVideoFrameReady(mPeerConnectionId, dataY, dataU, dataV, strideY, strideU, strideV, + width, height); + } + + private void RaiseRemoteVideoFrameReady( + IntPtr dataY, IntPtr dataU, IntPtr dataV, + int strideY, int strideU, int strideV, + uint width, uint height) { + if (OnRemoteVideoFrameReady != null) + OnRemoteVideoFrameReady(mPeerConnectionId, dataY, dataU, dataV, strideY, strideU, strideV, + width, height); + } + + + private void RaiseLocalSdpReadytoSend(string type, string sdp) { if (OnLocalSdpReadytoSend != null) - OnLocalSdpReadytoSend(peer_connection_id_, msg); + OnLocalSdpReadytoSend(mPeerConnectionId, type, sdp); } - private void RaiseIceCandiateReadytoSend(string msg) { + private void RaiseIceCandiateReadytoSend(string candidate, int sdpMlineIndex, string sdpMid) { if (OnIceCandiateReadytoSend != null) - OnIceCandiateReadytoSend(peer_connection_id_, msg); + OnIceCandiateReadytoSend(mPeerConnectionId, candidate, sdpMlineIndex, sdpMid); } - private LocalDataChannelReadyInternalDelegate localDataChannelReadyDelegate_ = null; + public void AddQueuedIceCandidate(List iceCandidateQueue) { + if (iceCandidateQueue != null) { + foreach (IceCandidate ic in iceCandidateQueue) { + AddIceCandidate(mPeerConnectionId, ic.Candidate, ic.SdpMlineIndex, ic.SdpMid); + } + } + } + + private LocalDataChannelReadyInternalDelegate localDataChannelReadyDelegate = null; public event LocalDataChannelReadyDelegate OnLocalDataChannelReady; - private DataFromDataChannelReadyInternalDelegate dataFromDataChannelReadyDelegate_ = null; + private DataFromDataChannelReadyInternalDelegate dataFromDataChannelReadyDelegate = null; public event DataFromDataChannelReadyDelegate OnDataFromDataChannelReady; - private FailureMessageInternalDelegate failureMessageDelegate_ = null; + private FailureMessageInternalDelegate failureMessageDelegate = null; public event FailureMessageDelegate OnFailureMessage; - private AudioBusReadyInternalDelegate audioBusReadyDelegate_ = null; + private AudioBusReadyInternalDelegate audioBusReadyDelegate = null; public event AudioBusReadyDelegate OnAudioBusReady; - private LocalSdpReadytoSendInternalDelegate localSdpReadytoSendDelegate_ = null; + private I420FrameReadyInternalDelegate localI420FrameReadyDelegate = null; + public event I420FrameReadyDelegate OnLocalVideoFrameReady; + + private I420FrameReadyInternalDelegate remoteI420FrameReadyDelegate = null; + public event I420FrameReadyDelegate OnRemoteVideoFrameReady; + + private LocalSdpReadytoSendInternalDelegate localSdpReadytoSendDelegate = null; public event LocalSdpReadytoSendDelegate OnLocalSdpReadytoSend; - private IceCandiateReadytoSendInternalDelegate iceCandiateReadytoSendDelegate_ = null; + private IceCandiateReadytoSendInternalDelegate iceCandiateReadytoSendDelegate = null; public event IceCandiateReadytoSendDelegate OnIceCandiateReadytoSend; - private int peer_connection_id_ = -1; + private int mPeerConnectionId = -1; } } diff --git a/webrtc/examples/unityplugin/simple_peer_connection.cc b/webrtc/examples/unityplugin/simple_peer_connection.cc index c302e9c433..dd92f7be5a 100644 --- a/webrtc/examples/unityplugin/simple_peer_connection.cc +++ b/webrtc/examples/unityplugin/simple_peer_connection.cc @@ -15,16 +15,6 @@ #include "webrtc/api/test/fakeconstraints.h" #include "webrtc/media/engine/webrtcvideocapturerfactory.h" #include "webrtc/modules/video_capture/video_capture_factory.h" -#include "webrtc/rtc_base/json.h" - -// Names used for a IceCandidate JSON object. -const char kCandidateSdpMidName[] = "sdpMid"; -const char kCandidateSdpMlineIndexName[] = "sdpMLineIndex"; -const char kCandidateSdpName[] = "candidate"; - -// Names used for a SessionDescription JSON object. -const char kSessionDescriptionTypeName[] = "type"; -const char kSessionDescriptionSdpName[] = "sdp"; // Names used for media stream labels. const char kAudioLabel[] = "audio_label"; @@ -73,7 +63,11 @@ class DummySetSessionDescriptionObserver } // namespace -bool SimplePeerConnection::InitializePeerConnection(bool is_receiver) { +bool SimplePeerConnection::InitializePeerConnection(const char** turn_urls, + const int no_of_urls, + const char* username, + const char* credential, + bool is_receiver) { RTC_DCHECK(peer_connection_.get() == nullptr); if (g_peer_connection_factory == nullptr) { @@ -92,21 +86,51 @@ bool SimplePeerConnection::InitializePeerConnection(bool is_receiver) { } g_peer_count++; - if (!CreatePeerConnection(is_receiver)) { + if (!CreatePeerConnection(turn_urls, no_of_urls, username, credential, + is_receiver)) { DeletePeerConnection(); return false; } return peer_connection_.get() != nullptr; } -bool SimplePeerConnection::CreatePeerConnection(bool is_receiver) { +bool SimplePeerConnection::CreatePeerConnection(const char** turn_urls, + const int no_of_urls, + const char* username, + const char* credential, + bool is_receiver) { RTC_DCHECK(g_peer_connection_factory.get() != nullptr); RTC_DCHECK(peer_connection_.get() == nullptr); - webrtc::PeerConnectionInterface::RTCConfiguration config; - webrtc::PeerConnectionInterface::IceServer server; - server.uri = GetPeerConnectionString(); - config.servers.push_back(server); + local_video_observer_.reset(new VideoObserver()); + remote_video_observer_.reset(new VideoObserver()); + + // Add the turn server. + if (turn_urls != nullptr) { + if (no_of_urls > 0) { + webrtc::PeerConnectionInterface::IceServer turn_server; + for (int i = 0; i < no_of_urls; i++) { + std::string url(turn_urls[i]); + if (url.length() > 0) + turn_server.urls.push_back(turn_urls[i]); + } + + std::string user_name(username); + if (user_name.length() > 0) + turn_server.username = username; + + std::string password(credential); + if (password.length() > 0) + turn_server.password = credential; + + config_.servers.push_back(turn_server); + } + } + + // Add the stun server. + webrtc::PeerConnectionInterface::IceServer stun_server; + stun_server.uri = GetPeerConnectionString(); + config_.servers.push_back(stun_server); webrtc::FakeConstraints constraints; constraints.SetAllowDtlsSctpDataChannels(); @@ -117,7 +141,7 @@ bool SimplePeerConnection::CreatePeerConnection(bool is_receiver) { } peer_connection_ = g_peer_connection_factory->CreatePeerConnection( - config, &constraints, nullptr, nullptr, this); + config_, &constraints, nullptr, nullptr, this); return peer_connection_.get() != nullptr; } @@ -160,13 +184,8 @@ void SimplePeerConnection::OnSuccess( std::string sdp; desc->ToString(&sdp); - Json::StyledWriter writer; - Json::Value jmessage; - jmessage[kSessionDescriptionTypeName] = desc->type(); - jmessage[kSessionDescriptionSdpName] = sdp; - if (OnLocalSdpReady) - OnLocalSdpReady(writer.write(jmessage).c_str()); + OnLocalSdpReady(desc->type().c_str(), sdp.c_str()); } void SimplePeerConnection::OnFailure(const std::string& error) { @@ -180,25 +199,27 @@ void SimplePeerConnection::OnIceCandidate( const webrtc::IceCandidateInterface* candidate) { LOG(INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index(); - Json::StyledWriter writer; - Json::Value jmessage; - - jmessage[kCandidateSdpMidName] = candidate->sdp_mid(); - jmessage[kCandidateSdpMlineIndexName] = candidate->sdp_mline_index(); std::string sdp; if (!candidate->ToString(&sdp)) { LOG(LS_ERROR) << "Failed to serialize candidate"; return; } - jmessage[kCandidateSdpName] = sdp; if (OnIceCandiateReady) - OnIceCandiateReady(writer.write(jmessage).c_str()); + OnIceCandiateReady(sdp.c_str(), candidate->sdp_mline_index(), + candidate->sdp_mid().c_str()); } -void SimplePeerConnection::RegisterOnVideoFramReady( - VIDEOFRAMEREADY_CALLBACK callback) { - OnVideoFrameReady = callback; +void SimplePeerConnection::RegisterOnLocalI420FrameReady( + I420FRAMEREADY_CALLBACK callback) { + if (local_video_observer_) + local_video_observer_->SetVideoCallback(callback); +} + +void SimplePeerConnection::RegisterOnRemoteI420FrameReady( + I420FRAMEREADY_CALLBACK callback) { + if (remote_video_observer_) + remote_video_observer_->SetVideoCallback(callback); } void SimplePeerConnection::RegisterOnLocalDataChannelReady( @@ -230,88 +251,47 @@ void SimplePeerConnection::RegisterOnIceCandiateReadytoSend( OnIceCandiateReady = callback; } -bool SimplePeerConnection::ReceivedSdp(const char* msg) { +bool SimplePeerConnection::SetRemoteDescription(const char* type, + const char* sdp) { if (!peer_connection_) return false; - std::string message(msg); - - Json::Reader reader; - Json::Value jmessage; - if (!reader.parse(message, jmessage)) { - LOG(WARNING) << "Received unknown message. " << message; - return false; - } - std::string type; - std::string json_object; - - rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionTypeName, &type); - if (type.empty()) - return false; - - std::string sdp; - if (!rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionSdpName, - &sdp)) { - LOG(WARNING) << "Can't parse received session description message."; - return false; - } + std::string remote_desc(sdp); + std::string sdp_type(type); webrtc::SdpParseError error; webrtc::SessionDescriptionInterface* session_description( - webrtc::CreateSessionDescription(type, sdp, &error)); + webrtc::CreateSessionDescription(sdp_type, remote_desc, &error)); if (!session_description) { LOG(WARNING) << "Can't parse received session description message. " << "SdpParseError was: " << error.description; return false; } - LOG(INFO) << " Received session description :" << message; + LOG(INFO) << " Received session description :" << remote_desc; peer_connection_->SetRemoteDescription( DummySetSessionDescriptionObserver::Create(), session_description); return true; } -bool SimplePeerConnection::ReceivedIceCandidate(const char* ice_candidate) { +bool SimplePeerConnection::AddIceCandidate(const char* candidate, + const int sdp_mlineindex, + const char* sdp_mid) { if (!peer_connection_) return false; - std::string message(ice_candidate); - - Json::Reader reader; - Json::Value jmessage; - if (!reader.parse(message, jmessage)) { - LOG(WARNING) << "Received unknown message. " << message; - return false; - } - std::string type; - std::string json_object; - - rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionTypeName, &type); - if (!type.empty()) - return false; - - std::string sdp_mid; - int sdp_mlineindex = 0; - std::string sdp; - if (!rtc::GetStringFromJsonObject(jmessage, kCandidateSdpMidName, &sdp_mid) || - !rtc::GetIntFromJsonObject(jmessage, kCandidateSdpMlineIndexName, - &sdp_mlineindex) || - !rtc::GetStringFromJsonObject(jmessage, kCandidateSdpName, &sdp)) { - LOG(WARNING) << "Can't parse received message."; - return false; - } webrtc::SdpParseError error; - std::unique_ptr candidate( - webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, sdp, &error)); - if (!candidate.get()) { + std::unique_ptr ice_candidate( + webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, candidate, &error)); + if (!ice_candidate.get()) { LOG(WARNING) << "Can't parse received candidate message. " << "SdpParseError was: " << error.description; return false; } - if (!peer_connection_->AddIceCandidate(candidate.get())) { + if (!peer_connection_->AddIceCandidate(ice_candidate.get())) { LOG(WARNING) << "Failed to apply the received candidate"; return false; } - LOG(INFO) << " Received candidate :" << message; + LOG(INFO) << " Received candidate :" << candidate; return true; } @@ -348,7 +328,10 @@ void SimplePeerConnection::OnAddStream( rtc::scoped_refptr stream) { LOG(INFO) << __FUNCTION__ << " " << stream->label(); remote_stream_ = stream; - + if (remote_video_observer_ && !remote_stream_->GetVideoTracks().empty()) { + remote_stream_->GetVideoTracks()[0]->AddOrUpdateSink( + remote_video_observer_.get(), rtc::VideoSinkWants()); + } SetAudioControl(); } @@ -402,9 +385,13 @@ void SimplePeerConnection::AddStreams(bool audio_only) { rtc::scoped_refptr video_track( g_peer_connection_factory->CreateVideoTrack( kVideoLabel, g_peer_connection_factory->CreateVideoSource( - OpenVideoCaptureDevice(), nullptr))); + std::move(capture), nullptr))); stream->AddTrack(video_track); + if (local_video_observer_ && !stream->GetVideoTracks().empty()) { + stream->GetVideoTracks()[0]->AddOrUpdateSink( + local_video_observer_.get(), rtc::VideoSinkWants()); + } } } diff --git a/webrtc/examples/unityplugin/simple_peer_connection.h b/webrtc/examples/unityplugin/simple_peer_connection.h index 2950e12749..15961d8d9d 100644 --- a/webrtc/examples/unityplugin/simple_peer_connection.h +++ b/webrtc/examples/unityplugin/simple_peer_connection.h @@ -20,6 +20,7 @@ #include "webrtc/api/mediastreaminterface.h" #include "webrtc/api/peerconnectioninterface.h" #include "webrtc/examples/unityplugin/unity_plugin_apis.h" +#include "webrtc/examples/unityplugin/video_observer.h" class SimplePeerConnection : public webrtc::PeerConnectionObserver, public webrtc::CreateSessionDescriptionObserver, @@ -29,7 +30,11 @@ class SimplePeerConnection : public webrtc::PeerConnectionObserver, SimplePeerConnection() {} ~SimplePeerConnection() {} - bool InitializePeerConnection(bool is_receiver); + bool InitializePeerConnection(const char** turn_urls, + const int no_of_urls, + const char* username, + const char* credential, + bool is_receiver); void DeletePeerConnection(); void AddStreams(bool audio_only); bool CreateDataChannel(); @@ -39,7 +44,8 @@ class SimplePeerConnection : public webrtc::PeerConnectionObserver, void SetAudioControl(bool is_mute, bool is_record); // Register callback functions. - void RegisterOnVideoFramReady(VIDEOFRAMEREADY_CALLBACK callback); + void RegisterOnLocalI420FrameReady(I420FRAMEREADY_CALLBACK callback); + void RegisterOnRemoteI420FrameReady(I420FRAMEREADY_CALLBACK callback); void RegisterOnLocalDataChannelReady(LOCALDATACHANNELREADY_CALLBACK callback); void RegisterOnDataFromDataChannelReady( DATAFROMEDATECHANNELREADY_CALLBACK callback); @@ -48,16 +54,18 @@ class SimplePeerConnection : public webrtc::PeerConnectionObserver, void RegisterOnLocalSdpReadytoSend(LOCALSDPREADYTOSEND_CALLBACK callback); void RegisterOnIceCandiateReadytoSend( ICECANDIDATEREADYTOSEND_CALLBACK callback); - bool ReceivedSdp(const char* sdp); - bool ReceivedIceCandidate(const char* ice_candidate); - - bool SetHeadPosition(float x, float y, float z); - bool SetHeadRotation(float rx, float ry, float rz, float rw); - bool SetRemoteAudioPosition(float x, float y, float z); - bool SetRemoteAudioRotation(float rx, float ry, float rz, float rw); + bool SetRemoteDescription(const char* type, const char* sdp); + bool AddIceCandidate(const char* sdp, + const int sdp_mlineindex, + const char* sdp_mid); protected: - bool CreatePeerConnection(bool receiver); + // create a peerconneciton and add the turn servers info to the configuration. + bool CreatePeerConnection(const char** turn_urls, + const int no_of_urls, + const char* username, + const char* credential, + bool is_receiver); void CloseDataChannel(); std::unique_ptr OpenVideoCaptureDevice(); void SetAudioControl(); @@ -103,9 +111,12 @@ class SimplePeerConnection : public webrtc::PeerConnectionObserver, std::map > active_streams_; - webrtc::MediaStreamInterface* remote_stream_ = nullptr; + std::unique_ptr local_video_observer_; + std::unique_ptr remote_video_observer_; + + webrtc::MediaStreamInterface* remote_stream_ = nullptr; + webrtc::PeerConnectionInterface::RTCConfiguration config_; - VIDEOFRAMEREADY_CALLBACK OnVideoFrameReady = nullptr; LOCALDATACHANNELREADY_CALLBACK OnLocalDataChannelReady = nullptr; DATAFROMEDATECHANNELREADY_CALLBACK OnDataFromDataChannelReady = nullptr; FAILURE_CALLBACK OnFailureMessage = nullptr; diff --git a/webrtc/examples/unityplugin/unity_plugin_apis.cc b/webrtc/examples/unityplugin/unity_plugin_apis.cc index 7b510bdf09..08b334786b 100644 --- a/webrtc/examples/unityplugin/unity_plugin_apis.cc +++ b/webrtc/examples/unityplugin/unity_plugin_apis.cc @@ -21,12 +21,15 @@ static std::map> g_peer_connection_map; } // namespace -int CreatePeerConnection() { +int CreatePeerConnection(const char** turn_urls, + const int no_of_urls, + const char* username, + const char* credential) { g_peer_connection_map[g_peer_connection_id] = new rtc::RefCountedObject(); if (!g_peer_connection_map[g_peer_connection_id]->InitializePeerConnection( - false)) + turn_urls, no_of_urls, username, credential, false)) return -1; return g_peer_connection_id++; @@ -89,13 +92,45 @@ bool SetAudioControl(int peer_connection_id, bool is_mute, bool is_record) { return true; } -// Register callback functions. -bool RegisterOnVideoFramReady(int peer_connection_id, - VIDEOFRAMEREADY_CALLBACK callback) { +bool SetRemoteDescription(int peer_connection_id, + const char* type, + const char* sdp) { if (!g_peer_connection_map.count(peer_connection_id)) return false; - g_peer_connection_map[peer_connection_id]->RegisterOnVideoFramReady(callback); + return g_peer_connection_map[peer_connection_id]->SetRemoteDescription(type, + sdp); +} + +bool AddIceCandidate(const int peer_connection_id, + const char* candidate, + const int sdp_mlineindex, + const char* sdp_mid) { + if (!g_peer_connection_map.count(peer_connection_id)) + return false; + + return g_peer_connection_map[peer_connection_id]->AddIceCandidate( + candidate, sdp_mlineindex, sdp_mid); +} + +// Register callback functions. +bool RegisterOnLocalI420FrameReady(int peer_connection_id, + I420FRAMEREADY_CALLBACK callback) { + if (!g_peer_connection_map.count(peer_connection_id)) + return false; + + g_peer_connection_map[peer_connection_id]->RegisterOnLocalI420FrameReady( + callback); + return true; +} + +bool RegisterOnRemoteI420FrameReady(int peer_connection_id, + I420FRAMEREADY_CALLBACK callback) { + if (!g_peer_connection_map.count(peer_connection_id)) + return false; + + g_peer_connection_map[peer_connection_id]->RegisterOnRemoteI420FrameReady( + callback); return true; } @@ -158,28 +193,3 @@ bool RegisterOnIceCandiateReadytoSend( callback); return true; } - -int ReceivedSdp(int peer_connection_id, const char* sdp) { - // Create a peer_connection if no one exists. - int id = -1; - if (g_peer_connection_map.count(peer_connection_id)) { - id = peer_connection_id; - } else { - id = g_peer_connection_id++; - g_peer_connection_map[id] = - new rtc::RefCountedObject(); - if (!g_peer_connection_map[id]->InitializePeerConnection(true)) - return -1; - } - - g_peer_connection_map[id]->ReceivedSdp(sdp); - return id; -} - -bool ReceivedIceCandidate(int peer_connection_id, const char* ice_candidate) { - if (!g_peer_connection_map.count(peer_connection_id)) - return false; - - return g_peer_connection_map[peer_connection_id]->ReceivedIceCandidate( - ice_candidate); -} diff --git a/webrtc/examples/unityplugin/unity_plugin_apis.h b/webrtc/examples/unityplugin/unity_plugin_apis.h index bcd1af362d..ee7d709583 100644 --- a/webrtc/examples/unityplugin/unity_plugin_apis.h +++ b/webrtc/examples/unityplugin/unity_plugin_apis.h @@ -15,16 +15,22 @@ #include -// Defintions of callback functions. -typedef void (*VIDEOFRAMEREADY_CALLBACK)(uint8_t* buffer, - uint32_t width, - uint32_t height, - uint32_t stride); +// Definitions of callback functions. +typedef void (*I420FRAMEREADY_CALLBACK)(const uint8_t* data_y, + const uint8_t* data_u, + const uint8_t* data_v, + int stride_y, + int stride_u, + int stride_v, + uint32_t width, + uint32_t height); typedef void (*LOCALDATACHANNELREADY_CALLBACK)(); typedef void (*DATAFROMEDATECHANNELREADY_CALLBACK)(const char* msg); typedef void (*FAILURE_CALLBACK)(const char* msg); -typedef void (*LOCALSDPREADYTOSEND_CALLBACK)(const char* msg); -typedef void (*ICECANDIDATEREADYTOSEND_CALLBACK)(const char* msg); +typedef void (*LOCALSDPREADYTOSEND_CALLBACK)(const char* type, const char* sdp); +typedef void (*ICECANDIDATEREADYTOSEND_CALLBACK)(const char* candidate, + const int sdp_mline_index, + const char* sdp_mid); typedef void (*AUDIOBUSREADY_CALLBACK)(const void* audio_data, int bits_per_sample, int sample_rate, @@ -34,7 +40,10 @@ typedef void (*AUDIOBUSREADY_CALLBACK)(const void* audio_data, #define WEBRTC_PLUGIN_API __declspec(dllexport) extern "C" { // Create a peerconnection and return a unique peer connection id. -WEBRTC_PLUGIN_API int CreatePeerConnection(); +WEBRTC_PLUGIN_API int CreatePeerConnection(const char** turn_urls, + const int no_of_urls, + const char* username, + const char* credential); // Close a peerconnection. WEBRTC_PLUGIN_API bool ClosePeerConnection(int peer_connection_id); // Add a audio stream. If audio_only is true, the stream only has an audio @@ -54,11 +63,23 @@ WEBRTC_PLUGIN_API bool SendDataViaDataChannel(int peer_connection_id, WEBRTC_PLUGIN_API bool SetAudioControl(int peer_connection_id, bool is_mute, bool is_record); +// Set remote sdp. +WEBRTC_PLUGIN_API bool SetRemoteDescription(int peer_connection_id, + const char* type, + const char* sdp); +// Add ice candidate. +WEBRTC_PLUGIN_API bool AddIceCandidate(const int peer_connection_id, + const char* candidate, + const int sdp_mlineindex, + const char* sdp_mid); // Register callback functions. -WEBRTC_PLUGIN_API bool RegisterOnVideoFramReady( +WEBRTC_PLUGIN_API bool RegisterOnLocalI420FrameReady( int peer_connection_id, - VIDEOFRAMEREADY_CALLBACK callback); + I420FRAMEREADY_CALLBACK callback); +WEBRTC_PLUGIN_API bool RegisterOnRemoteI420FrameReady( + int peer_connection_id, + I420FRAMEREADY_CALLBACK callback); WEBRTC_PLUGIN_API bool RegisterOnLocalDataChannelReady( int peer_connection_id, LOCALDATACHANNELREADY_CALLBACK callback); @@ -75,9 +96,6 @@ WEBRTC_PLUGIN_API bool RegisterOnLocalSdpReadytoSend( WEBRTC_PLUGIN_API bool RegisterOnIceCandiateReadytoSend( int peer_connection_id, ICECANDIDATEREADYTOSEND_CALLBACK callback); -WEBRTC_PLUGIN_API int ReceivedSdp(int peer_connection_id, const char* sdp); -WEBRTC_PLUGIN_API bool ReceivedIceCandidate(int peer_connection_id, - const char* ice_candidate); } #endif // WEBRTC_EXAMPLES_UNITYPLUGIN_UNITY_PLUGIN_APIS_H_ diff --git a/webrtc/examples/unityplugin/video_observer.cc b/webrtc/examples/unityplugin/video_observer.cc new file mode 100644 index 0000000000..4f3bd567c2 --- /dev/null +++ b/webrtc/examples/unityplugin/video_observer.cc @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/examples/unityplugin/video_observer.h" + +void VideoObserver::SetVideoCallback(I420FRAMEREADY_CALLBACK callback) { + std::lock_guard lock(mutex); + OnI420FrameReady = callback; +} + +void VideoObserver::OnFrame(const webrtc::VideoFrame& frame) { + std::unique_lock lock(mutex); + rtc::scoped_refptr buffer( + frame.video_frame_buffer()->ToI420()); + if (OnI420FrameReady) { + OnI420FrameReady(buffer->DataY(), buffer->DataU(), buffer->DataV(), + buffer->StrideY(), buffer->StrideU(), buffer->StrideV(), + frame.width(), frame.height()); + } +} diff --git a/webrtc/examples/unityplugin/video_observer.h b/webrtc/examples/unityplugin/video_observer.h new file mode 100644 index 0000000000..9c3c7c65cc --- /dev/null +++ b/webrtc/examples/unityplugin/video_observer.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_EXAMPLES_UNITYPLUGIN_VIDEO_OBSERVER_H_ +#define WEBRTC_EXAMPLES_UNITYPLUGIN_VIDEO_OBSERVER_H_ + +#include + +#include "webrtc/api/mediastreaminterface.h" +#include "webrtc/examples/unityplugin/unity_plugin_apis.h" +#include "webrtc/media/base/videosinkinterface.h" + +class VideoObserver : public rtc::VideoSinkInterface { + public: + VideoObserver() {} + ~VideoObserver() {} + void SetVideoCallback(I420FRAMEREADY_CALLBACK callback); + + protected: + // VideoSinkInterface implementation + void OnFrame(const webrtc::VideoFrame& frame) override; + + private: + I420FRAMEREADY_CALLBACK OnI420FrameReady = nullptr; + std::mutex mutex; +}; + +#endif // WEBRTC_EXAMPLES_UNITYPLUGIN_VIDEO_OBSERVER_H_