diff --git a/webrtc/sdk/BUILD.gn b/webrtc/sdk/BUILD.gn index d08a2f7f56..0257fcf459 100644 --- a/webrtc/sdk/BUILD.gn +++ b/webrtc/sdk/BUILD.gn @@ -310,6 +310,11 @@ if (is_ios || is_mac) { sources = [ "objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory+Private.h", "objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory.mm", + "objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm", + "objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.h", + "objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.mm", + "objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.h", + "objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.mm", ] public_configs = [ ":objc_common_config" ] @@ -326,11 +331,16 @@ if (is_ios || is_mac) { ":objc_peerconnectionfactory_base", ":objc_video", ":objc_videotoolbox", + ":objc_videotracksource", "../api:video_frame_api", + "../api/video_codecs:video_codecs_api", "../base:rtc_base", + "../media:rtc_audio_video", "../media:rtc_media_base", + "../modules:module_api", "../pc:create_pc_factory", "../pc:peerconnection", + "../system_wrappers:field_trial_api", ] } @@ -377,6 +387,7 @@ if (is_ios || is_mac) { "objc/Framework/Classes/PeerConnection/RTCDataChannel.mm", "objc/Framework/Classes/PeerConnection/RTCDataChannelConfiguration+Private.h", "objc/Framework/Classes/PeerConnection/RTCDataChannelConfiguration.mm", + "objc/Framework/Classes/PeerConnection/RTCEncodedImage.mm", "objc/Framework/Classes/PeerConnection/RTCIceCandidate+Private.h", "objc/Framework/Classes/PeerConnection/RTCIceCandidate.mm", "objc/Framework/Classes/PeerConnection/RTCIceServer+Private.h", @@ -404,6 +415,7 @@ if (is_ios || is_mac) { "objc/Framework/Classes/PeerConnection/RTCRtpCodecParameters.mm", "objc/Framework/Classes/PeerConnection/RTCRtpEncodingParameters+Private.h", "objc/Framework/Classes/PeerConnection/RTCRtpEncodingParameters.mm", + "objc/Framework/Classes/PeerConnection/RTCRtpFragmentationHeader.mm", "objc/Framework/Classes/PeerConnection/RTCRtpParameters+Private.h", "objc/Framework/Classes/PeerConnection/RTCRtpParameters.mm", "objc/Framework/Classes/PeerConnection/RTCRtpReceiver+Private.h", @@ -415,6 +427,8 @@ if (is_ios || is_mac) { "objc/Framework/Classes/PeerConnection/RTCSessionDescription.mm", "objc/Framework/Classes/PeerConnection/RTCTracing.mm", "objc/Framework/Classes/PeerConnection/RTCVideoCapturer.m", + "objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h", + "objc/Framework/Classes/PeerConnection/RTCVideoCodec.mm", "objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm", "objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter+Private.h", "objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.h", @@ -476,6 +490,7 @@ if (is_ios || is_mac) { "../base:rtc_base", "../common_video", "../media:rtc_media_base", + "../modules:module_api", "../pc:peerconnection", ] } @@ -501,6 +516,8 @@ if (is_ios || is_mac) { "objc/Framework/UnitTests/RTCSessionDescriptionTest.mm", "objc/Framework/UnitTests/RTCTracingTest.mm", "objc/Framework/UnitTests/avformatmappertests.mm", + "objc/Framework/UnitTests/objc_video_decoder_factory_tests.mm", + "objc/Framework/UnitTests/objc_video_encoder_factory_tests.mm", ] if (is_ios && !(use_ios_simulator && @@ -553,6 +570,8 @@ if (is_ios || is_mac) { common_objc_headers = [ "objc/Framework/Headers/WebRTC/RTCAudioSession.h", + "objc/Framework/Headers/WebRTC/RTCVideoCodec.h", + "objc/Framework/Headers/WebRTC/RTCVideoCodecFactory.h", "objc/Framework/Headers/WebRTC/RTCAudioSessionConfiguration.h", "objc/Framework/Headers/WebRTC/RTCAVFoundationVideoSource.h", "objc/Framework/Headers/WebRTC/RTCAudioSource.h", @@ -595,6 +614,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCVideoTrack.h", "objc/Framework/Headers/WebRTC/RTCVideoViewShading.h", "objc/Framework/Headers/WebRTC/UIDevice+RTCDevice.h", + "objc/Framework/Headers/WebRTC/RTCVideoCodecH264.h", "objc/Framework/Headers/WebRTC/WebRTC.h", ] if (rtc_use_metal_rendering) { diff --git a/webrtc/sdk/objc/Framework/Classes/Common/helpers.h b/webrtc/sdk/objc/Framework/Classes/Common/helpers.h index d9e1d7fa64..fdcd9ccb56 100644 --- a/webrtc/sdk/objc/Framework/Classes/Common/helpers.h +++ b/webrtc/sdk/objc/Framework/Classes/Common/helpers.h @@ -11,8 +11,6 @@ #ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_COMMON_HELPERS_H_ #define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_COMMON_HELPERS_H_ -#if defined(WEBRTC_IOS) - #include namespace webrtc { @@ -33,6 +31,7 @@ std::string GetThreadInfo(); // Example: {number = 1, name = main} std::string GetCurrentThreadDescription(); +#if defined(WEBRTC_IOS) // Returns the current name of the operating system. std::string GetSystemName(); @@ -46,6 +45,7 @@ double GetSystemVersion(); // Returns the device type. // Examples: ”iPhone” and ”iPod touch”. std::string GetDeviceType(); +#endif // defined(WEBRTC_IOS) // Returns a more detailed device name. // Examples: "iPhone 5s (GSM)" and "iPhone 6 Plus". @@ -65,12 +65,12 @@ std::string GetOSVersionString(); // Returns the number of processing cores available on the device. int GetProcessorCount(); +#if defined(WEBRTC_IOS) // Indicates whether Low Power Mode is enabled on the iOS device. bool GetLowPowerModeEnabled(); +#endif } // namespace ios } // namespace webrtc -#endif // defined(WEBRTC_IOS) - #endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_COMMON_HELPERS_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCEncodedImage.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCEncodedImage.mm new file mode 100644 index 0000000000..50550214d0 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCEncodedImage.mm @@ -0,0 +1,74 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoCodec.h" + +#import "RTCVideoCodec+Private.h" + +@implementation RTCEncodedImage + +@synthesize buffer = _buffer; +@synthesize encodedWidth = _encodedWidth; +@synthesize encodedHeight = _encodedHeight; +@synthesize timeStamp = _timeStamp; +@synthesize captureTimeMs = _captureTimeMs; +@synthesize ntpTimeMs = _ntpTimeMs; +@synthesize isTimingFrame = _isTimingFrame; +@synthesize encodeStartMs = _encodeStartMs; +@synthesize encodeFinishMs = _encodeFinishMs; +@synthesize frameType = _frameType; +@synthesize rotation = _rotation; +@synthesize completeFrame = _completeFrame; +@synthesize qp = _qp; + +- (instancetype)initWithEncodedImage:(webrtc::EncodedImage)encodedImage { + if (self = [super init]) { + // Wrap the buffer in NSData without copying, do not take ownership. + _buffer = [NSData dataWithBytesNoCopy:encodedImage._buffer + length:encodedImage._length + freeWhenDone:NO]; + _encodedWidth = encodedImage._encodedWidth; + _encodedHeight = encodedImage._encodedHeight; + _timeStamp = encodedImage._timeStamp; + _captureTimeMs = encodedImage.capture_time_ms_; + _ntpTimeMs = encodedImage.ntp_time_ms_; + _isTimingFrame = encodedImage.timing_.is_timing_frame; + _encodeStartMs = encodedImage.timing_.encode_start_ms; + _encodeFinishMs = encodedImage.timing_.encode_finish_ms; + _frameType = (RTCFrameType)encodedImage._frameType; + _rotation = encodedImage.rotation_; + _completeFrame = encodedImage._completeFrame; + _qp = encodedImage.qp_ == -1 ? nil : @(encodedImage.qp_); + } + + return self; +} + +- (webrtc::EncodedImage)toCpp { + // Return the pointer without copying. + webrtc::EncodedImage encodedImage( + (uint8_t *)_buffer.bytes, (size_t)_buffer.length, (size_t)_buffer.length); + encodedImage._encodedWidth = _encodedWidth; + encodedImage._encodedHeight = _encodedHeight; + encodedImage._timeStamp = _timeStamp; + encodedImage.capture_time_ms_ = _captureTimeMs; + encodedImage.ntp_time_ms_ = _ntpTimeMs; + encodedImage.timing_.is_timing_frame = _isTimingFrame; + encodedImage.timing_.encode_start_ms = _encodeStartMs; + encodedImage.timing_.encode_finish_ms = _encodeFinishMs; + encodedImage._frameType = webrtc::FrameType(_frameType); + encodedImage.rotation_ = webrtc::VideoRotation(_rotation); + encodedImage._completeFrame = _completeFrame; + encodedImage.qp_ = _qp ? _qp.intValue : -1; + + return encodedImage; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory.mm index 7accdbbe38..7c5ba1d16f 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory.mm @@ -11,6 +11,7 @@ #import "RTCPeerConnectionFactory+Private.h" #import "NSString+StdString.h" +#import "RTCAVFoundationVideoSource+Private.h" #import "RTCAudioSource+Private.h" #import "RTCAudioTrack+Private.h" #import "RTCMediaConstraints+Private.h" @@ -18,11 +19,15 @@ #import "RTCPeerConnection+Private.h" #import "RTCVideoSource+Private.h" #import "RTCVideoTrack+Private.h" -#import "RTCAVFoundationVideoSource+Private.h" #import "WebRTC/RTCLogging.h" +#import "WebRTC/RTCVideoCodecFactory.h" +#ifndef HAVE_NO_MEDIA +#import "WebRTC/RTCVideoCodecH264.h" +#endif +#include "PeerConnection/objc_video_decoder_factory.h" +#include "PeerConnection/objc_video_encoder_factory.h" #include "Video/objcvideotracksource.h" -#include "VideoToolbox/videocodecfactory.h" #include "webrtc/api/videosourceproxy.h" // Adding the nogncheck to disable the including header check. // The no-media version PeerConnectionFactory doesn't depend on media related @@ -41,7 +46,17 @@ @synthesize nativeFactory = _nativeFactory; - (instancetype)init { - if ((self = [super init])) { +#ifdef HAVE_NO_MEDIA + return [self initWithEncoderFactory:nil decoderFactory:nil]; +#else + return [self initWithEncoderFactory:[[RTCVideoEncoderFactoryH264 alloc] init] + decoderFactory:[[RTCVideoDecoderFactoryH264 alloc] init]]; +#endif +} + +- (instancetype)initWithEncoderFactory:(nullable id)encoderFactory + decoderFactory:(nullable id)decoderFactory { + if (self = [super init]) { _networkThread = rtc::Thread::CreateWithSocketServer(); BOOL result = _networkThread->Start(); NSAssert(result, @"Failed to start network thread."); @@ -68,8 +83,14 @@ std::unique_ptr(), std::unique_ptr()); #else - const auto encoder_factory = new webrtc::VideoToolboxVideoEncoderFactory(); - const auto decoder_factory = new webrtc::VideoToolboxVideoDecoderFactory(); + cricket::WebRtcVideoEncoderFactory *encoder_factory = nullptr; + cricket::WebRtcVideoDecoderFactory *decoder_factory = nullptr; + if (encoderFactory) { + encoder_factory = new webrtc::ObjCVideoEncoderFactory(encoderFactory); + } + if (decoderFactory) { + decoder_factory = new webrtc::ObjCVideoDecoderFactory(decoderFactory); + } // Ownership of encoder/decoder factories is passed on to the // peerconnectionfactory, that handles deleting them. diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCRtpFragmentationHeader.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCRtpFragmentationHeader.mm new file mode 100644 index 0000000000..eea07ec92e --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCRtpFragmentationHeader.mm @@ -0,0 +1,61 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoCodec.h" + +#include "webrtc/modules/include/module_common_types.h" + +@implementation RTCRtpFragmentationHeader + +@synthesize fragmentationOffset = _fragmentationOffset; +@synthesize fragmentationLength = _fragmentationLength; +@synthesize fragmentationTimeDiff = _fragmentationTimeDiff; +@synthesize fragmentationPlType = _fragmentationPlType; + +- (instancetype)initWithFragmentationHeader: + (const webrtc::RTPFragmentationHeader *__nullable)fragmentationHeader { + if (self = [super init]) { + if (fragmentationHeader) { + int count = fragmentationHeader->fragmentationVectorSize; + NSMutableArray *offsets = [NSMutableArray array]; + NSMutableArray *lengths = [NSMutableArray array]; + NSMutableArray *timeDiffs = [NSMutableArray array]; + NSMutableArray *plTypes = [NSMutableArray array]; + for (int i = 0; i < count; ++i) { + [offsets addObject:@(fragmentationHeader->fragmentationOffset[i])]; + [lengths addObject:@(fragmentationHeader->fragmentationLength[i])]; + [timeDiffs addObject:@(fragmentationHeader->fragmentationTimeDiff[i])]; + [plTypes addObject:@(fragmentationHeader->fragmentationPlType[i])]; + } + _fragmentationOffset = [offsets copy]; + _fragmentationLength = [lengths copy]; + _fragmentationTimeDiff = [timeDiffs copy]; + _fragmentationPlType = [plTypes copy]; + } + } + + return self; +} + +- (webrtc::RTPFragmentationHeader *)toCpp { + webrtc::RTPFragmentationHeader *fragmentationHeader = new webrtc::RTPFragmentationHeader; + fragmentationHeader->VerifyAndAllocateFragmentationHeader(_fragmentationOffset.count); + for (NSUInteger i = 0; i < _fragmentationOffset.count; ++i) { + fragmentationHeader->fragmentationOffset[i] = (size_t)_fragmentationOffset[i].unsignedIntValue; + fragmentationHeader->fragmentationLength[i] = (size_t)_fragmentationLength[i].unsignedIntValue; + fragmentationHeader->fragmentationTimeDiff[i] = + (uint16_t)_fragmentationOffset[i].unsignedIntValue; + fragmentationHeader->fragmentationPlType[i] = (uint8_t)_fragmentationOffset[i].unsignedIntValue; + } + + return fragmentationHeader; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h new file mode 100644 index 0000000000..27815189e8 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h @@ -0,0 +1,57 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoCodec.h" + +#import "WebRTC/RTCVideoCodecH264.h" + +#include "webrtc/common_video/include/video_frame.h" +#include "webrtc/media/base/codec.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +/* Interfaces for converting to/from internal C++ formats. */ +@interface RTCEncodedImage () + +- (instancetype)initWithEncodedImage:(webrtc::EncodedImage)encodedImage; +- (webrtc::EncodedImage)toCpp; + +@end + +@interface RTCVideoEncoderSettings () + +- (instancetype)initWithVideoCodec:(const webrtc::VideoCodec *__nullable)videoCodec; +- (webrtc::VideoCodec *)toCpp; + +@end + +@interface RTCCodecSpecificInfoH264 () + +- (webrtc::CodecSpecificInfo)toCpp; + +@end + +@interface RTCRtpFragmentationHeader () + +- (instancetype)initWithFragmentationHeader: + (const webrtc::RTPFragmentationHeader *__nullable)fragmentationHeader; +- (webrtc::RTPFragmentationHeader *)toCpp; + +@end + +@interface RTCVideoCodecInfo () + +- (instancetype)initWithVideoCodec:(cricket::VideoCodec)videoCodec; +- (cricket::VideoCodec)toCpp; + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec.mm new file mode 100644 index 0000000000..a1b9109fd3 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec.mm @@ -0,0 +1,114 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoCodec.h" + +#import "RTCVideoCodec+Private.h" +#import "WebRTC/RTCVideoCodecFactory.h" + +#include "webrtc/sdk/objc/Framework/Classes/Common/helpers.h" + +@implementation RTCVideoCodecInfo + +@synthesize payload = _payload; +@synthesize name = _name; +@synthesize parameters = _parameters; + +- (instancetype)initWithPayload:(int)payload + name:(NSString *)name + parameters:(NSDictionary *)parameters { + if (self = [super init]) { + _payload = payload; + _name = name; + _parameters = parameters; + } + + return self; +} + +- (instancetype)initWithVideoCodec:(cricket::VideoCodec)videoCodec { + NSMutableDictionary *params = [NSMutableDictionary dictionary]; + for (auto it = videoCodec.params.begin(); it != videoCodec.params.end(); ++it) { + [params setObject:webrtc::ios::NSStringFromStdString(it->second) + forKey:webrtc::ios::NSStringFromStdString(it->first)]; + } + return [self initWithPayload:videoCodec.id + name:webrtc::ios::NSStringFromStdString(videoCodec.name) + parameters:params]; +} + +- (cricket::VideoCodec)toCpp { + cricket::VideoCodec codec(webrtc::ios::StdStringFromNSString(_name)); + for (NSString *paramKey in [_parameters allKeys]) { + codec.SetParam(webrtc::ios::StdStringFromNSString(paramKey), + webrtc::ios::StdStringFromNSString(_parameters[paramKey])); + } + + return codec; +} + +@end + +@implementation RTCVideoEncoderSettings + +@synthesize name = _name; +@synthesize width = _width; +@synthesize height = _height; +@synthesize startBitrate = _startBitrate; +@synthesize maxBitrate = _maxBitrate; +@synthesize minBitrate = _minBitrate; +@synthesize targetBitrate = _targetBitrate; +@synthesize maxFramerate = _maxFramerate; +@synthesize qpMax = _qpMax; + +- (instancetype)initWithVideoCodec:(const webrtc::VideoCodec *__nullable)videoCodec { + if (self = [super init]) { + if (videoCodec) { + rtc::Optional codecName = CodecTypeToPayloadName(videoCodec->codecType); + if (codecName) { + _name = [NSString stringWithUTF8String:codecName.value()]; + } + + _width = videoCodec->width; + _height = videoCodec->height; + _startBitrate = videoCodec->startBitrate; + _maxBitrate = videoCodec->maxBitrate; + _minBitrate = videoCodec->minBitrate; + _targetBitrate = videoCodec->targetBitrate; + _maxFramerate = videoCodec->maxFramerate; + _qpMax = videoCodec->qpMax; + } + } + + return self; +} + +- (webrtc::VideoCodec *)toCpp { + webrtc::VideoCodec *codecSettings = new webrtc::VideoCodec; + + rtc::Optional codecType = + webrtc::PayloadNameToCodecType(webrtc::ios::StdStringFromNSString(_name)); + if (codecType) { + codecSettings->codecType = codecType.value(); + } + + codecSettings->width = _width; + codecSettings->height = _height; + codecSettings->startBitrate = _startBitrate; + codecSettings->maxBitrate = _maxBitrate; + codecSettings->minBitrate = _minBitrate; + codecSettings->targetBitrate = _targetBitrate; + codecSettings->maxFramerate = _maxFramerate; + codecSettings->qpMax = _qpMax; + + return codecSettings; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm new file mode 100644 index 0000000000..28b386e428 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm @@ -0,0 +1,262 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoCodecH264.h" + +#include + +#import "RTCVideoCodec+Private.h" +#import "WebRTC/RTCVideoCodec.h" +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" + +#include "webrtc/rtc_base/timeutils.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.h" +#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h" +#include "webrtc/system_wrappers/include/field_trial.h" + +const size_t kDefaultPayloadSize = 1440; + +const char kHighProfileExperiment[] = "WebRTC-H264HighProfile"; + +bool IsHighProfileEnabled() { + return webrtc::field_trial::IsEnabled(kHighProfileExperiment); +} + +// H264 specific settings. +@implementation RTCCodecSpecificInfoH264 + +@synthesize packetizationMode = _packetizationMode; + +- (webrtc::CodecSpecificInfo)toCpp { + webrtc::CodecSpecificInfo codecSpecificInfo; + codecSpecificInfo.codecType = webrtc::kVideoCodecH264; + codecSpecificInfo.codec_name = "H264"; + codecSpecificInfo.codecSpecific.H264.packetization_mode = + (webrtc::H264PacketizationMode)_packetizationMode; + + return codecSpecificInfo; +} + +@end + +namespace { + +class H264VideoToolboxEncodeCompleteCallback : public webrtc::EncodedImageCallback { + public: + Result OnEncodedImage(const webrtc::EncodedImage &encoded_image, + const webrtc::CodecSpecificInfo *codec_specific_info, + const webrtc::RTPFragmentationHeader *fragmentation) { + RTCEncodedImage *image = [[RTCEncodedImage alloc] initWithEncodedImage:encoded_image]; + + RTCCodecSpecificInfoH264 *info = [[RTCCodecSpecificInfoH264 alloc] init]; + info.packetizationMode = + (RTCH264PacketizationMode)codec_specific_info->codecSpecific.H264.packetization_mode; + + RTCRtpFragmentationHeader *header = + [[RTCRtpFragmentationHeader alloc] initWithFragmentationHeader:fragmentation]; + + callback(image, info, header); + return Result(Result::OK, 0); + } + + RTCVideoEncoderCallback callback; +}; + +class H264VideoToolboxDecodeCompleteCallback : public webrtc::DecodedImageCallback { + public: + int32_t Decoded(webrtc::VideoFrame &decodedImage) { + rtc::scoped_refptr video_frame_buffer = + decodedImage.video_frame_buffer(); + id rtcFrameBuffer; + rtc::scoped_refptr objc_frame_buffer( + static_cast(video_frame_buffer.get())); + rtcFrameBuffer = (id)objc_frame_buffer->wrapped_frame_buffer(); + + RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] + initWithBuffer:rtcFrameBuffer + rotation:static_cast(decodedImage.rotation()) + timeStampNs:decodedImage.timestamp_us() * rtc::kNumNanosecsPerMicrosec]; + videoFrame.timeStamp = decodedImage.timestamp(); + + callback(videoFrame); + + return 0; + } + + RTCVideoDecoderCallback callback; +}; + +} // namespace + +// Encoder. +@implementation RTCVideoEncoderH264 { + webrtc::H264VideoToolboxEncoder *_videoToolboxEncoder; + H264VideoToolboxEncodeCompleteCallback *_toolboxCallback; +} + +- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo { + if (self = [super init]) { + cricket::VideoCodec codec = [codecInfo toCpp]; + _videoToolboxEncoder = new webrtc::H264VideoToolboxEncoder(codec); + } + return self; +} + +- (void)setCallback:(RTCVideoEncoderCallback)callback { + _toolboxCallback = new H264VideoToolboxEncodeCompleteCallback(); + _toolboxCallback->callback = callback; + _videoToolboxEncoder->RegisterEncodeCompleteCallback(_toolboxCallback); +} + +- (int)initEncodeWithSettings:(RTCVideoEncoderSettings *)settings numberOfCores:(int)numberOfCores { + webrtc::VideoCodec *codecSettings = [settings toCpp]; + return _videoToolboxEncoder->InitEncode(codecSettings, numberOfCores, kDefaultPayloadSize); +} + +- (int)releaseEncode { + return _videoToolboxEncoder->Release(); +} + +- (int)encode:(RTCVideoFrame *)frame + codecSpecificInfo:(id)info + frameTypes:(NSArray *)frameTypes { + rtc::scoped_refptr frameBuffer = + new rtc::RefCountedObject(frame.buffer); + webrtc::VideoFrame videoFrame(frameBuffer, + (webrtc::VideoRotation)frame.rotation, + frame.timeStampNs / rtc::kNumNanosecsPerMicrosec); + videoFrame.set_timestamp(frame.timeStamp); + + // Handle types than can be converted into one of webrtc::CodecSpecificInfo's hard coded cases. + webrtc::CodecSpecificInfo codecSpecificInfo; + if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) { + codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info toCpp]; + } + + std::vector nativeFrameTypes; + for (NSNumber *frameType in frameTypes) { + RTCFrameType rtcFrameType = (RTCFrameType)frameType.unsignedIntegerValue; + nativeFrameTypes.push_back((webrtc::FrameType)rtcFrameType); + } + + return _videoToolboxEncoder->Encode(videoFrame, &codecSpecificInfo, &nativeFrameTypes); +} + +- (BOOL)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { + return _videoToolboxEncoder->SetRates(bitrateKbit, framerate) == WEBRTC_VIDEO_CODEC_OK; +} + +@end + +// Decoder. +@implementation RTCVideoDecoderH264 { + webrtc::H264VideoToolboxDecoder *_videoToolboxDecoder; + H264VideoToolboxDecodeCompleteCallback *_toolboxCallback; +} + +- (instancetype)init { + if (self = [super init]) { + cricket::VideoCodec codec(cricket::kH264CodecName); + _videoToolboxDecoder = new webrtc::H264VideoToolboxDecoder(); + } + return self; +} + +- (int)initDecodeWithSettings:(RTCVideoEncoderSettings *)settings numberOfCores:(int)numberOfCores { + webrtc::VideoCodec *codecSettings = [settings toCpp]; + return _videoToolboxDecoder->InitDecode(codecSettings, numberOfCores); +} + +- (void)setCallback:(RTCVideoDecoderCallback)callback { + _toolboxCallback = new H264VideoToolboxDecodeCompleteCallback(); + _toolboxCallback->callback = callback; + _videoToolboxDecoder->RegisterDecodeCompleteCallback(_toolboxCallback); +} + +- (int32_t)releaseDecode { + return _videoToolboxDecoder->Release(); +} + +- (int)decode:(RTCEncodedImage *)encodedImage + missingFrames:(BOOL)missingFrames + fragmentationHeader:(RTCRtpFragmentationHeader *)fragmentationHeader + codecSpecificInfo:(__nullable id)info + renderTimeMs:(int64_t)renderTimeMs { + webrtc::EncodedImage image = [encodedImage toCpp]; + + // Handle types than can be converted into one of webrtc::CodecSpecificInfo's hard coded cases. + webrtc::CodecSpecificInfo codecSpecificInfo; + if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) { + codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info toCpp]; + } + + webrtc::RTPFragmentationHeader *header = [fragmentationHeader toCpp]; + + return _videoToolboxDecoder->Decode( + image, missingFrames, header, &codecSpecificInfo, renderTimeMs); +} + +@end + +// Encoder factory. +@implementation RTCVideoEncoderFactoryH264 + +- (NSArray *)supportedCodecs { + NSMutableArray *codecs = [NSMutableArray array]; + NSString *codecName = [NSString stringWithUTF8String:cricket::kH264CodecName]; + + if (IsHighProfileEnabled()) { + NSDictionary *constrainedHighParams = @{ + @"profile-level-id" : @"640c1f", // Level 3.1 Constrained High. + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTCVideoCodecInfo *constrainedHighInfo = + [[RTCVideoCodecInfo alloc] initWithPayload:0 + name:codecName + parameters:constrainedHighParams]; + [codecs addObject:constrainedHighInfo]; + } + + NSDictionary *constrainedBaselineParams = @{ + @"profile-level-id" : @"42e01f", // Level 3.1 Constrained Baseline. + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTCVideoCodecInfo *constrainedBaselineInfo = + [[RTCVideoCodecInfo alloc] initWithPayload:0 + name:codecName + parameters:constrainedBaselineParams]; + [codecs addObject:constrainedBaselineInfo]; + + return [codecs copy]; +} + +- (id)createEncoder:(RTCVideoCodecInfo *)info { + return [[RTCVideoEncoderH264 alloc] initWithCodecInfo:info]; +} + +@end + +// Decoder factory. +@implementation RTCVideoDecoderFactoryH264 + +- (id)createDecoder:(RTCVideoCodecInfo *)info { + return [[RTCVideoDecoderH264 alloc] init]; +} + +- (NSArray *)supportedCodecs { + NSString *codecName = [NSString stringWithUTF8String:cricket::kH264CodecName]; + return @[ [[RTCVideoCodecInfo alloc] initWithPayload:0 name:codecName parameters:@{}] ]; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm index ef93fef8cd..daac9fde08 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm @@ -17,6 +17,7 @@ } @synthesize buffer = _buffer; +@synthesize timeStamp; - (int)width { return _buffer.width; diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.h b/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.h new file mode 100644 index 0000000000..dca7fe2899 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.h @@ -0,0 +1,39 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_DECODER_FACTORY_H_ +#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_DECODER_FACTORY_H_ + +#include "webrtc/media/base/codec.h" +#include "webrtc/media/engine/webrtcvideodecoderfactory.h" + +@protocol RTCVideoDecoderFactory; + +namespace webrtc { + +class ObjCVideoDecoderFactory : public cricket::WebRtcVideoDecoderFactory { + public: + explicit ObjCVideoDecoderFactory(id); + ~ObjCVideoDecoderFactory(); + + id wrapped_decoder_factory() const; + + webrtc::VideoDecoder* CreateVideoDecoder( + webrtc::VideoCodecType type) override; + void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override; + + private: + id decoder_factory_; + std::vector supported_codecs_; +}; + +} // namespace webrtc + +#endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_DECODER_FACTORY_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.mm new file mode 100644 index 0000000000..0f2937bc4e --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.mm @@ -0,0 +1,124 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.h" + +#import "RTCVideoCodec+Private.h" +#import "WebRTC/RTCVideoCodec.h" +#import "WebRTC/RTCVideoCodecFactory.h" +#import "WebRTC/RTCVideoCodecH264.h" +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" + +#include "webrtc/api/video_codecs/video_decoder.h" +#include "webrtc/modules/include/module_common_types.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/modules/video_coding/include/video_error_codes.h" +#include "webrtc/rtc_base/logging.h" +#include "webrtc/rtc_base/timeutils.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" + +namespace webrtc { + +namespace { +class ObjCVideoDecoder : public VideoDecoder { + public: + ObjCVideoDecoder(id decoder) : decoder_(decoder) {} + ~ObjCVideoDecoder() {} + + int32_t InitDecode(const VideoCodec *codec_settings, int32_t number_of_cores) { + RTCVideoEncoderSettings *settings = + [[RTCVideoEncoderSettings alloc] initWithVideoCodec:codec_settings]; + return [decoder_ initDecodeWithSettings:settings numberOfCores:number_of_cores]; + } + + int32_t Decode(const EncodedImage &input_image, + bool missing_frames, + const RTPFragmentationHeader *fragmentation, + const CodecSpecificInfo *codec_specific_info = NULL, + int64_t render_time_ms = -1) { + RTCEncodedImage *encodedImage = [[RTCEncodedImage alloc] initWithEncodedImage:input_image]; + RTCRtpFragmentationHeader *header = + [[RTCRtpFragmentationHeader alloc] initWithFragmentationHeader:fragmentation]; + + // webrtc::CodecSpecificInfo only handles a hard coded list of codecs + id rtcCodecSpecificInfo = nil; + if (codec_specific_info) { + if (codec_specific_info->codecType == kVideoCodecH264) { + RTCCodecSpecificInfoH264 *h264Info = [[RTCCodecSpecificInfoH264 alloc] init]; + h264Info.packetizationMode = + (RTCH264PacketizationMode)codec_specific_info->codecSpecific.H264.packetization_mode; + rtcCodecSpecificInfo = h264Info; + } + } + + return [decoder_ decode:encodedImage + missingFrames:missing_frames + fragmentationHeader:header + codecSpecificInfo:rtcCodecSpecificInfo + renderTimeMs:render_time_ms]; + } + + int32_t RegisterDecodeCompleteCallback(DecodedImageCallback *callback) { + [decoder_ setCallback:^(RTCVideoFrame *frame) { + const rtc::scoped_refptr buffer = + new rtc::RefCountedObject(frame.buffer); + VideoFrame videoFrame(buffer, + (uint32_t)(frame.timeStampNs / rtc::kNumNanosecsPerMicrosec), + 0, + (VideoRotation)frame.rotation); + videoFrame.set_timestamp(frame.timeStamp); + + callback->Decoded(videoFrame); + }]; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() { return [decoder_ releaseDecode]; } + + private: + id decoder_; +}; +} // namespace + +ObjCVideoDecoderFactory::ObjCVideoDecoderFactory(id decoder_factory) + : decoder_factory_(decoder_factory) {} + +ObjCVideoDecoderFactory::~ObjCVideoDecoderFactory() {} + +id ObjCVideoDecoderFactory::wrapped_decoder_factory() const { + return decoder_factory_; +} + +VideoDecoder *ObjCVideoDecoderFactory::CreateVideoDecoder(VideoCodecType type) { + const rtc::Optional codec_name = CodecTypeToPayloadName(type); + if (!codec_name) { + LOG(LS_ERROR) << "Invalid codec type: " << type; + return nullptr; + } + + NSString *codecName = [NSString stringWithUTF8String:codec_name.value()]; + for (RTCVideoCodecInfo *codecInfo in decoder_factory_.supportedCodecs) { + if ([codecName isEqualToString:codecInfo.name]) { + id decoder = [decoder_factory_ createDecoder:codecInfo]; + return new ObjCVideoDecoder(decoder); + } + } + + return nullptr; +} + +void ObjCVideoDecoderFactory::DestroyVideoDecoder(VideoDecoder *decoder) { + delete decoder; + decoder = nullptr; +} + +} // namespace webrtc diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.h b/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.h new file mode 100644 index 0000000000..b0c45887f0 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.h @@ -0,0 +1,41 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_ENCODER_FACTORY_H_ +#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_ENCODER_FACTORY_H_ + +#import + +#include "webrtc/media/engine/webrtcvideoencoderfactory.h" + +@protocol RTCVideoEncoderFactory; + +namespace webrtc { + +class ObjCVideoEncoderFactory : public cricket::WebRtcVideoEncoderFactory { + public: + explicit ObjCVideoEncoderFactory(id); + ~ObjCVideoEncoderFactory(); + + id wrapped_encoder_factory() const; + + webrtc::VideoEncoder* CreateVideoEncoder( + const cricket::VideoCodec& codec) override; + const std::vector& supported_codecs() const override; + void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override; + + private: + id encoder_factory_; + mutable std::vector supported_codecs_; +}; + +} // namespace webrtc + +#endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_ENCODER_FACTORY_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.mm new file mode 100644 index 0000000000..d4fa416d05 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.mm @@ -0,0 +1,149 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.h" + +#import "RTCVideoCodec+Private.h" +#import "WebRTC/RTCVideoCodec.h" +#import "WebRTC/RTCVideoCodecFactory.h" +#import "WebRTC/RTCVideoCodecH264.h" +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" + +#include "webrtc/api/video/video_frame.h" +#include "webrtc/api/video_codecs/video_encoder.h" +#include "webrtc/modules/include/module_common_types.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/modules/video_coding/include/video_error_codes.h" +#include "webrtc/rtc_base/logging.h" +#include "webrtc/rtc_base/timeutils.h" +#include "webrtc/sdk/objc/Framework/Classes/Common/helpers.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" + +namespace webrtc { + +namespace { +class ObjCVideoEncoder : public VideoEncoder { + public: + ObjCVideoEncoder(id encoder) : encoder_(encoder) {} + ~ObjCVideoEncoder() {} + + int32_t InitEncode(const VideoCodec *codec_settings, + int32_t number_of_cores, + size_t max_payload_size) { + RTCVideoEncoderSettings *settings = + [[RTCVideoEncoderSettings alloc] initWithVideoCodec:codec_settings]; + return [encoder_ initEncodeWithSettings:settings numberOfCores:number_of_cores]; + } + + int32_t RegisterEncodeCompleteCallback(EncodedImageCallback *callback) { + [encoder_ setCallback:^(RTCEncodedImage *frame, + id info, + RTCRtpFragmentationHeader *header) { + EncodedImage encodedImage = [frame toCpp]; + + // Handle types than can be converted into one of webrtc::CodecSpecificInfo's hard coded + // cases. + CodecSpecificInfo codecSpecificInfo; + if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) { + codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info toCpp]; + } + + RTPFragmentationHeader *fragmentationHeader = [header toCpp]; + callback->OnEncodedImage(encodedImage, &codecSpecificInfo, fragmentationHeader); + }]; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() { return [encoder_ releaseEncode]; } + + int32_t Encode(const VideoFrame &frame, + const CodecSpecificInfo *codec_specific_info, + const std::vector *frame_types) { + RTC_CHECK(frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative); + + id frame_buffer = + static_cast(frame.video_frame_buffer().get())->wrapped_frame_buffer(); + RTCVideoFrame *rtcFrame = + [[RTCVideoFrame alloc] initWithBuffer:frame_buffer + rotation:RTCVideoRotation(frame.rotation()) + timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec]; + rtcFrame.timeStamp = frame.timestamp(); + + // webrtc::CodecSpecificInfo only handles a hard coded list of codecs + id rtcCodecSpecificInfo = nil; + if (codec_specific_info) { + if (strcmp(codec_specific_info->codec_name, "H264") == 0) { + RTCCodecSpecificInfoH264 *h264Info = [[RTCCodecSpecificInfoH264 alloc] init]; + h264Info.packetizationMode = + (RTCH264PacketizationMode)codec_specific_info->codecSpecific.H264.packetization_mode; + rtcCodecSpecificInfo = h264Info; + } + } + + NSMutableArray *rtcFrameTypes = [NSMutableArray array]; + for (size_t i = 0; i < frame_types->size(); ++i) { + [rtcFrameTypes addObject:@(RTCFrameType(frame_types->at(i)))]; + } + + return + [encoder_ encode:rtcFrame codecSpecificInfo:rtcCodecSpecificInfo frameTypes:rtcFrameTypes]; + } + + int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) { return WEBRTC_VIDEO_CODEC_OK; } + + int32_t SetRates(uint32_t bitrate, uint32_t framerate) { + if ([encoder_ setBitrate:bitrate framerate:framerate]) { + return WEBRTC_VIDEO_CODEC_OK; + } else { + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + bool SupportsNativeHandle() const { return true; } + + private: + id encoder_; +}; +} // namespace + +ObjCVideoEncoderFactory::ObjCVideoEncoderFactory(id encoder_factory) + : encoder_factory_(encoder_factory) {} + +ObjCVideoEncoderFactory::~ObjCVideoEncoderFactory() {} + +id ObjCVideoEncoderFactory::wrapped_encoder_factory() const { + return encoder_factory_; +} + +webrtc::VideoEncoder *ObjCVideoEncoderFactory::CreateVideoEncoder( + const cricket::VideoCodec &codec) { + RTCVideoCodecInfo *info = [[RTCVideoCodecInfo alloc] initWithVideoCodec:codec]; + id encoder = [encoder_factory_ createEncoder:info]; + return new ObjCVideoEncoder(encoder); +} + +const std::vector &ObjCVideoEncoderFactory::supported_codecs() const { + supported_codecs_.clear(); + for (RTCVideoCodecInfo *supportedCodec in encoder_factory_.supportedCodecs) { + cricket::VideoCodec codec = [supportedCodec toCpp]; + supported_codecs_.push_back(codec); + } + + return supported_codecs_; +} + +void ObjCVideoEncoderFactory::DestroyVideoEncoder(webrtc::VideoEncoder *encoder) { + delete encoder; + encoder = nullptr; +} + +} // namespace webrtc diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h index 3924cdd053..b6a13c1bb0 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h @@ -24,11 +24,19 @@ NS_ASSUME_NONNULL_BEGIN @class RTCVideoSource; @class RTCVideoTrack; @protocol RTCPeerConnectionDelegate; +@protocol RTCVideoEncoderFactory; +@protocol RTCVideoDecoderFactory; RTC_EXPORT @interface RTCPeerConnectionFactory : NSObject -- (instancetype)init NS_DESIGNATED_INITIALIZER; +/* Initialize object with default H264 video encoder/decoder factories */ +- (instancetype)init; + +/* Initialize object with injectable video encoder/decoder factories */ +- (instancetype)initWithEncoderFactory:(nullable id)encoderFactory + decoderFactory:(nullable id)decoderFactory + NS_DESIGNATED_INITIALIZER; /** Initialize an RTCAudioSource with constraints. */ - (RTCAudioSource *)audioSourceWithConstraints:(nullable RTCMediaConstraints *)constraints; diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h new file mode 100644 index 0000000000..f093049faf --- /dev/null +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h @@ -0,0 +1,137 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import + +@class RTCVideoFrame; + +NS_ASSUME_NONNULL_BEGIN + +/** Represents an encoded frame's type. */ +typedef NS_ENUM(NSUInteger, RTCFrameType) { + EmptyFrame, + VideoFrameKey, + VideoFrameDelta, +}; + +/** Represents an encoded frame. Corresponds to webrtc::EncodedImage. */ +RTC_EXPORT +@interface RTCEncodedImage : NSObject + +@property(nonatomic, retain) NSData *buffer; +@property(nonatomic, assign) int encodedWidth; +@property(nonatomic, assign) int encodedHeight; +@property(nonatomic, assign) uint32_t timeStamp; +@property(nonatomic, assign) long captureTimeMs; +@property(nonatomic, assign) long ntpTimeMs; +@property(nonatomic, assign) BOOL isTimingFrame; +@property(nonatomic, assign) long encodeStartMs; +@property(nonatomic, assign) long encodeFinishMs; +@property(nonatomic, assign) RTCFrameType frameType; +@property(nonatomic, assign) int rotation; +@property(nonatomic, assign) BOOL completeFrame; +@property(nonatomic, retain) NSNumber *qp; + +@end + +/** Information for header. Corresponds to webrtc::RTPFragmentationHeader. */ +RTC_EXPORT +@interface RTCRtpFragmentationHeader : NSObject + +@property(nonatomic, retain) NSArray *fragmentationOffset; +@property(nonatomic, retain) NSArray *fragmentationLength; +@property(nonatomic, retain) NSArray *fragmentationTimeDiff; +@property(nonatomic, retain) NSArray *fragmentationPlType; + +@end + +/** Implement this protocol to pass codec specific info from the encoder. + * Corresponds to webrtc::CodecSpecificInfo. + */ +RTC_EXPORT +@protocol RTCCodecSpecificInfo + +@end + +/** Callback block for encoder. */ +typedef void (^RTCVideoEncoderCallback)(RTCEncodedImage *frame, + id info, + RTCRtpFragmentationHeader *header); + +/** Callback block for decoder. */ +typedef void (^RTCVideoDecoderCallback)(RTCVideoFrame *frame); + +/** Settings for encoder. Corresponds to webrtc::VideoCodec. */ +RTC_EXPORT +@interface RTCVideoEncoderSettings : NSObject + +@property(nonatomic, retain) NSString *name; + +@property(nonatomic, assign) unsigned short width; +@property(nonatomic, assign) unsigned short height; + +@property(nonatomic, assign) unsigned int startBitrate; // kilobits/sec. +@property(nonatomic, assign) unsigned int maxBitrate; +@property(nonatomic, assign) unsigned int minBitrate; +@property(nonatomic, assign) unsigned int targetBitrate; + +@property(nonatomic, assign) uint32_t maxFramerate; + +@property(nonatomic, assign) unsigned int qpMax; + +@end + +/** Holds information to identify a codec. Corresponds to cricket::VideoCodec. */ +RTC_EXPORT +@interface RTCVideoCodecInfo : NSObject + +- (instancetype)initWithPayload:(int)payload + name:(NSString *)name + parameters:(NSDictionary *)parameters; + +@property(nonatomic, readonly) int payload; +@property(nonatomic, readonly) NSString *name; +@property(nonatomic, readonly) NSDictionary *parameters; + +@end + +/** Protocol for encoder implementations. */ +RTC_EXPORT +@protocol RTCVideoEncoder + +- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo; +- (void)setCallback:(RTCVideoEncoderCallback)callback; +- (int)initEncodeWithSettings:(RTCVideoEncoderSettings *)settings numberOfCores:(int)numberOfCores; +- (int)releaseEncode; +- (int)encode:(RTCVideoFrame *)frame + codecSpecificInfo:(id)info + frameTypes:(NSArray *)frameTypes; +- (BOOL)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate; + +@end + +/** Protocol for decoder implementations. */ +RTC_EXPORT +@protocol RTCVideoDecoder + +- (void)setCallback:(RTCVideoDecoderCallback)callback; +- (int)initDecodeWithSettings:(RTCVideoEncoderSettings *)settings numberOfCores:(int)numberOfCores; +- (int)releaseDecode; +- (int)decode:(RTCEncodedImage *)encodedImage + missingFrames:(BOOL)missingFrames + fragmentationHeader:(RTCRtpFragmentationHeader *)fragmentationHeader + codecSpecificInfo:(__nullable id)info + renderTimeMs:(int64_t)renderTimeMs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecFactory.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecFactory.h new file mode 100644 index 0000000000..42842eb020 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecFactory.h @@ -0,0 +1,36 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +/** RTCVideoEncoderFactory is an Objective-C version of cricket::WebRtcVideoEncoderFactory. */ +RTC_EXPORT +@protocol RTCVideoEncoderFactory + +- (id)createEncoder:(RTCVideoCodecInfo *)info; +- (NSArray *)supportedCodecs; + +@end + +/** RTCVideoDecoderFactory is an Objective-C version of cricket::WebRtcVideoDecoderFactory. */ +RTC_EXPORT +@protocol RTCVideoDecoderFactory + +- (id)createDecoder:(RTCVideoCodecInfo *)info; +- (NSArray *)supportedCodecs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH264.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH264.h new file mode 100644 index 0000000000..fbb2bd29b6 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH264.h @@ -0,0 +1,47 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import +#import + +/** Class for H264 specific config. */ +typedef NS_ENUM(NSUInteger, RTCH264PacketizationMode) { + NonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed + SingleNalUnit // Mode 0 - only single NALU allowed +}; + +RTC_EXPORT +@interface RTCCodecSpecificInfoH264 : NSObject + +@property(nonatomic, assign) RTCH264PacketizationMode packetizationMode; + +@end + +/** Encoder. */ +RTC_EXPORT +@interface RTCVideoEncoderH264 : NSObject +@end + +/** Decoder. */ +RTC_EXPORT +@interface RTCVideoDecoderH264 : NSObject +@end + +/** Encoder factory. */ +RTC_EXPORT +@interface RTCVideoEncoderFactoryH264 : NSObject +@end + +/** Decoder factory. */ +RTC_EXPORT +@interface RTCVideoDecoderFactoryH264 : NSObject +@end diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h index 78eee5f224..4467b06c9d 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h @@ -51,6 +51,9 @@ RTC_EXPORT /** Timestamp in nanoseconds. */ @property(nonatomic, readonly) int64_t timeStampNs; +/** Timestamp 90 kHz. */ +@property(nonatomic, assign) int32_t timeStamp; + /** The native handle should be a pixel buffer on iOS. */ @property(nonatomic, readonly) CVPixelBufferRef nativeHandle DEPRECATED_MSG_ATTRIBUTE("use buffer instead"); diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/WebRTC.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/WebRTC.h index 904cc84e03..17019a2a64 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/WebRTC.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/WebRTC.h @@ -47,6 +47,9 @@ #import #import #import +#import +#import +#import #import #import #import diff --git a/webrtc/sdk/objc/Framework/UnitTests/objc_video_decoder_factory_tests.mm b/webrtc/sdk/objc/Framework/UnitTests/objc_video_decoder_factory_tests.mm new file mode 100644 index 0000000000..7143f3dfd9 --- /dev/null +++ b/webrtc/sdk/objc/Framework/UnitTests/objc_video_decoder_factory_tests.mm @@ -0,0 +1,103 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#include "webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_decoder_factory.h" + +#import "WebRTC/RTCVideoCodec.h" +#import "WebRTC/RTCVideoCodecFactory.h" +#include "webrtc/modules/include/module_common_types.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/modules/video_coding/include/video_error_codes.h" +#include "webrtc/rtc_base/gunit.h" + +id CreateDecoderFactoryReturning(int return_code) { + id decoderMock = OCMProtocolMock(@protocol(RTCVideoDecoder)); + OCMStub([decoderMock initDecodeWithSettings:[OCMArg any] numberOfCores:1]).andReturn(return_code); + OCMStub([decoderMock decode:[OCMArg any] + missingFrames:NO + fragmentationHeader:[OCMArg any] + codecSpecificInfo:[OCMArg any] + renderTimeMs:0]) + .andReturn(return_code); + OCMStub([decoderMock releaseDecode]).andReturn(return_code); + + id decoderFactoryMock = OCMProtocolMock(@protocol(RTCVideoDecoderFactory)); + RTCVideoCodecInfo *supported = + [[RTCVideoCodecInfo alloc] initWithPayload:0 name:@"H264" parameters:@{}]; + OCMStub([decoderFactoryMock supportedCodecs]).andReturn(@[ supported ]); + OCMStub([decoderFactoryMock createDecoder:[OCMArg any]]).andReturn(decoderMock); + return decoderFactoryMock; +} + +id CreateOKDecoderFactory() { + return CreateDecoderFactoryReturning(WEBRTC_VIDEO_CODEC_OK); +} + +id CreateErrorDecoderFactory() { + return CreateDecoderFactoryReturning(WEBRTC_VIDEO_CODEC_ERROR); +} + +webrtc::VideoDecoder *GetObjCDecoder(id factory) { + webrtc::ObjCVideoDecoderFactory decoder_factory(factory); + return decoder_factory.CreateVideoDecoder(webrtc::kVideoCodecH264); +} + +#pragma mark - + +TEST(ObjCVideoDecoderFactoryTest, InitDecodeReturnsOKOnSuccess) { + webrtc::VideoDecoder *decoder = GetObjCDecoder(CreateOKDecoderFactory()); + + auto settings = new webrtc::VideoCodec(); + EXPECT_EQ(decoder->InitDecode(settings, 1), WEBRTC_VIDEO_CODEC_OK); +} + +TEST(ObjCVideoDecoderFactoryTest, InitDecodeReturnsErrorOnFail) { + webrtc::VideoDecoder *decoder = GetObjCDecoder(CreateErrorDecoderFactory()); + + auto settings = new webrtc::VideoCodec(); + EXPECT_EQ(decoder->InitDecode(settings, 1), WEBRTC_VIDEO_CODEC_ERROR); +} + +TEST(ObjCVideoDecoderFactoryTest, DecodeReturnsOKOnSuccess) { + webrtc::VideoDecoder *decoder = GetObjCDecoder(CreateOKDecoderFactory()); + + webrtc::EncodedImage encoded_image; + webrtc::RTPFragmentationHeader header; + webrtc::CodecSpecificInfo info; + info.codecType = webrtc::kVideoCodecH264; + + EXPECT_EQ(decoder->Decode(encoded_image, false, &header, &info, 0), WEBRTC_VIDEO_CODEC_OK); +} + +TEST(ObjCVideoDecoderFactoryTest, DecodeReturnsErrorOnFail) { + webrtc::VideoDecoder *decoder = GetObjCDecoder(CreateErrorDecoderFactory()); + + webrtc::EncodedImage encoded_image; + webrtc::RTPFragmentationHeader header; + webrtc::CodecSpecificInfo info; + info.codecType = webrtc::kVideoCodecH264; + + EXPECT_EQ(decoder->Decode(encoded_image, false, &header, &info, 0), WEBRTC_VIDEO_CODEC_ERROR); +} + +TEST(ObjCVideoDecoderFactoryTest, ReleaseDecodeReturnsOKOnSuccess) { + webrtc::VideoDecoder *decoder = GetObjCDecoder(CreateOKDecoderFactory()); + + EXPECT_EQ(decoder->Release(), WEBRTC_VIDEO_CODEC_OK); +} + +TEST(ObjCVideoDecoderFactoryTest, ReleaseDecodeReturnsErrorOnFail) { + webrtc::VideoDecoder *decoder = GetObjCDecoder(CreateErrorDecoderFactory()); + + EXPECT_EQ(decoder->Release(), WEBRTC_VIDEO_CODEC_ERROR); +} diff --git a/webrtc/sdk/objc/Framework/UnitTests/objc_video_encoder_factory_tests.mm b/webrtc/sdk/objc/Framework/UnitTests/objc_video_encoder_factory_tests.mm new file mode 100644 index 0000000000..1f858006f9 --- /dev/null +++ b/webrtc/sdk/objc/Framework/UnitTests/objc_video_encoder_factory_tests.mm @@ -0,0 +1,133 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#include "webrtc/sdk/objc/Framework/Classes/PeerConnection/objc_video_encoder_factory.h" + +#import "WebRTC/RTCVideoCodec.h" +#import "WebRTC/RTCVideoCodecFactory.h" +#import "WebRTC/RTCVideoFrameBuffer.h" +#include "webrtc/modules/include/module_common_types.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/modules/video_coding/include/video_error_codes.h" +#include "webrtc/rtc_base/gunit.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" + +id CreateEncoderFactoryReturning(int return_code) { + id encoderMock = OCMProtocolMock(@protocol(RTCVideoEncoder)); + OCMStub([encoderMock initEncodeWithSettings:[OCMArg any] numberOfCores:1]).andReturn(return_code); + OCMStub([encoderMock encode:[OCMArg any] codecSpecificInfo:[OCMArg any] frameTypes:[OCMArg any]]) + .andReturn(return_code); + OCMStub([encoderMock releaseEncode]).andReturn(return_code); + OCMStub([encoderMock setBitrate:0 framerate:0]).andReturn(return_code == WEBRTC_VIDEO_CODEC_OK); + + id encoderFactoryMock = OCMProtocolMock(@protocol(RTCVideoEncoderFactory)); + RTCVideoCodecInfo *supported = + [[RTCVideoCodecInfo alloc] initWithPayload:0 name:@"H264" parameters:@{}]; + OCMStub([encoderFactoryMock supportedCodecs]).andReturn(@[ supported ]); + OCMStub([encoderFactoryMock createEncoder:[OCMArg any]]).andReturn(encoderMock); + return encoderFactoryMock; +} + +id CreateOKEncoderFactory() { + return CreateEncoderFactoryReturning(WEBRTC_VIDEO_CODEC_OK); +} + +id CreateErrorEncoderFactory() { + return CreateEncoderFactoryReturning(WEBRTC_VIDEO_CODEC_ERROR); +} + +webrtc::VideoEncoder *GetObjCEncoder(id factory) { + webrtc::ObjCVideoEncoderFactory encoder_factory(factory); + cricket::VideoCodec codec("H264"); + return encoder_factory.CreateVideoEncoder(codec); +} + +#pragma mark - + +TEST(ObjCVideoEncoderFactoryTest, InitEncodeReturnsOKOnSuccess) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateOKEncoderFactory()); + + auto settings = new webrtc::VideoCodec(); + EXPECT_EQ(encoder->InitEncode(settings, 1, 0), WEBRTC_VIDEO_CODEC_OK); +} + +TEST(ObjCVideoEncoderFactoryTest, InitEncodeReturnsErrorOnFail) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateErrorEncoderFactory()); + + auto settings = new webrtc::VideoCodec(); + EXPECT_EQ(encoder->InitEncode(settings, 1, 0), WEBRTC_VIDEO_CODEC_ERROR); +} + +TEST(ObjCVideoEncoderFactoryTest, EncodeReturnsOKOnSuccess) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateOKEncoderFactory()); + + CVPixelBufferRef pixel_buffer; + CVPixelBufferCreate(kCFAllocatorDefault, 640, 480, kCVPixelFormatType_32ARGB, nil, &pixel_buffer); + rtc::scoped_refptr buffer = + new rtc::RefCountedObject( + [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixel_buffer]); + webrtc::VideoFrame frame(buffer, webrtc::kVideoRotation_0, 0); + webrtc::CodecSpecificInfo info; + info.codecType = webrtc::kVideoCodecH264; + info.codec_name = "H264"; + std::vector frame_types; + + EXPECT_EQ(encoder->Encode(frame, &info, &frame_types), WEBRTC_VIDEO_CODEC_OK); +} + +TEST(ObjCVideoEncoderFactoryTest, EncodeReturnsErrorOnFail) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateErrorEncoderFactory()); + + CVPixelBufferRef pixel_buffer; + CVPixelBufferCreate(kCFAllocatorDefault, 640, 480, kCVPixelFormatType_32ARGB, nil, &pixel_buffer); + rtc::scoped_refptr buffer = + new rtc::RefCountedObject( + [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixel_buffer]); + webrtc::VideoFrame frame(buffer, webrtc::kVideoRotation_0, 0); + webrtc::CodecSpecificInfo info; + info.codecType = webrtc::kVideoCodecH264; + info.codec_name = "H264"; + std::vector frame_types; + + EXPECT_EQ(encoder->Encode(frame, &info, &frame_types), WEBRTC_VIDEO_CODEC_ERROR); +} + +TEST(ObjCVideoEncoderFactoryTest, ReleaseEncodeReturnsOKOnSuccess) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateOKEncoderFactory()); + + EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_OK); +} + +TEST(ObjCVideoEncoderFactoryTest, ReleaseEncodeReturnsErrorOnFail) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateErrorEncoderFactory()); + + EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_ERROR); +} + +TEST(ObjCVideoEncoderFactoryTest, SetChannelParametersAlwaysReturnsOK) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateErrorEncoderFactory()); + + EXPECT_EQ(encoder->SetChannelParameters(1, 1), WEBRTC_VIDEO_CODEC_OK); +} + +TEST(ObjCVideoEncoderFactoryTest, SetRatesReturnsOKOnSuccess) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateOKEncoderFactory()); + + EXPECT_EQ(encoder->SetRates(0, 0), WEBRTC_VIDEO_CODEC_OK); +} + +TEST(ObjCVideoEncoderFactoryTest, SetRatesReturnsErrorOnFail) { + webrtc::VideoEncoder *encoder = GetObjCEncoder(CreateErrorEncoderFactory()); + + EXPECT_EQ(encoder->SetRates(0, 0), WEBRTC_VIDEO_CODEC_ERROR); +}