diff --git a/webrtc/sdk/BUILD.gn b/webrtc/sdk/BUILD.gn index 9526d60ce4..9fc22aa60f 100644 --- a/webrtc/sdk/BUILD.gn +++ b/webrtc/sdk/BUILD.gn @@ -106,11 +106,8 @@ if (is_ios || is_mac) { sources = [ "objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.h", "objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm", - "objc/Framework/Classes/Video/RTCCVPixelBuffer.mm", "objc/Framework/Classes/Video/RTCDefaultShader.h", "objc/Framework/Classes/Video/RTCDefaultShader.mm", - "objc/Framework/Classes/Video/RTCI420Buffer+Private.h", - "objc/Framework/Classes/Video/RTCI420Buffer.mm", "objc/Framework/Classes/Video/RTCI420TextureCache.h", "objc/Framework/Classes/Video/RTCI420TextureCache.mm", "objc/Framework/Classes/Video/RTCOpenGLDefines.h", @@ -120,11 +117,8 @@ if (is_ios || is_mac) { "objc/Framework/Classes/Video/avfoundationformatmapper.mm", "objc/Framework/Classes/Video/avfoundationvideocapturer.h", "objc/Framework/Classes/Video/avfoundationvideocapturer.mm", - "objc/Framework/Classes/Video/objc_frame_buffer.h", - "objc/Framework/Classes/Video/objc_frame_buffer.mm", "objc/Framework/Classes/Video/objcvideotracksource.h", "objc/Framework/Classes/Video/objcvideotracksource.mm", - "objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h", ] libs = [] if (is_ios) { @@ -149,6 +143,8 @@ if (is_ios || is_mac) { deps = [ ":objc_common", + ":objc_corevideoframebuffer", + ":objc_videotoolbox", "../api:libjingle_peerconnection_api", "../base:rtc_base", "../common_video", @@ -275,6 +271,7 @@ if (is_ios || is_mac) { "objc/Framework/Classes/PeerConnection/RTCSessionDescription.mm", "objc/Framework/Classes/PeerConnection/RTCTracing.mm", "objc/Framework/Classes/PeerConnection/RTCVideoCapturer.m", + "objc/Framework/Classes/PeerConnection/RTCVideoFrame+Private.h", "objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm", "objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter+Private.h", "objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.h", @@ -311,7 +308,6 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCTracing.h", "objc/Framework/Headers/WebRTC/RTCVideoCapturer.h", "objc/Framework/Headers/WebRTC/RTCVideoFrame.h", - "objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h", "objc/Framework/Headers/WebRTC/RTCVideoRenderer.h", "objc/Framework/Headers/WebRTC/RTCVideoSource.h", "objc/Framework/Headers/WebRTC/RTCVideoTrack.h", @@ -342,10 +338,8 @@ if (is_ios || is_mac) { ":objc_common", ":objc_corevideoframebuffer", ":objc_video", - ":objc_videotoolbox", "../api:video_frame_api", "../base:rtc_base", - "../common_video", "../media:rtc_media_base", "../pc:libjingle_peerconnection", ] @@ -467,7 +461,6 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCTracing.h", "objc/Framework/Headers/WebRTC/RTCVideoCapturer.h", "objc/Framework/Headers/WebRTC/RTCVideoFrame.h", - "objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h", "objc/Framework/Headers/WebRTC/RTCVideoRenderer.h", "objc/Framework/Headers/WebRTC/RTCVideoSource.h", "objc/Framework/Headers/WebRTC/RTCVideoTrack.h", @@ -559,14 +552,13 @@ if (is_ios || is_mac) { "objc/Framework/Classes/VideoToolbox/nalu_rewriter.h", "objc/Framework/Classes/VideoToolbox/videocodecfactory.h", "objc/Framework/Classes/VideoToolbox/videocodecfactory.mm", - "objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h", ] configs += [ "..:common_objc" ] deps = [ ":objc_common", - ":objc_video", + ":objc_corevideoframebuffer", "../base:rtc_base_approved", "../common_video", "../media:rtc_media", diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm index 027db05b39..7ca1d4e766 100644 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm +++ b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm @@ -9,7 +9,6 @@ */ #import "RTCMTLI420Renderer.h" -#import "WebRTC/RTCVideoFrameBuffer.h" #import #import @@ -97,8 +96,6 @@ static NSString *const shaderSource = MTL_STRINGIFY( return NO; } - id buffer = [frame.buffer toI420]; - // Luma (y) texture. if (!_descriptor || (_width != frame.width && _height != frame.height)) { _width = frame.width; @@ -114,8 +111,8 @@ static NSString *const shaderSource = MTL_STRINGIFY( // Chroma (u,v) textures [_yTexture replaceRegion:MTLRegionMake2D(0, 0, _width, _height) mipmapLevel:0 - withBytes:buffer.dataY - bytesPerRow:buffer.strideY]; + withBytes:frame.dataY + bytesPerRow:frame.strideY]; if (!_chromaDescriptor || (_chromaWidth != frame.width / 2 && _chromaHeight != frame.height / 2)) { @@ -133,12 +130,12 @@ static NSString *const shaderSource = MTL_STRINGIFY( [_uTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight) mipmapLevel:0 - withBytes:buffer.dataU - bytesPerRow:buffer.strideU]; + withBytes:frame.dataU + bytesPerRow:frame.strideU]; [_vTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight) mipmapLevel:0 - withBytes:buffer.dataV - bytesPerRow:buffer.strideV]; + withBytes:frame.dataV + bytesPerRow:frame.strideV]; return (_uTexture != nil) && (_yTexture != nil) && (_vTexture != nil); } diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm index 70ecf549ab..b5b9a0f59b 100644 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm +++ b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm @@ -15,7 +15,6 @@ #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" -#import "WebRTC/RTCVideoFrameBuffer.h" #import "RTCMTLRenderer+Private.h" @@ -86,7 +85,7 @@ static NSString *const shaderSource = MTL_STRINGIFY( - (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame { [super setupTexturesForFrame:frame]; - CVPixelBufferRef pixelBuffer = ((RTCCVPixelBuffer *)frame.buffer).pixelBuffer; + CVPixelBufferRef pixelBuffer = frame.nativeHandle; id lumaTexture = nil; id chromaTexture = nil; diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m index e36cb3ab4c..96778c4095 100644 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m +++ b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m @@ -15,7 +15,6 @@ #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" -#import "WebRTC/RTCVideoFrameBuffer.h" #import "RTCMTLI420Renderer.h" #import "RTCMTLNV12Renderer.h" @@ -109,7 +108,7 @@ } id renderer = nil; - if ([self.videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + if (self.videoFrame.nativeHandle) { if (!self.rendererNV12) { self.rendererNV12 = [RTCMTLVideoView createNV12Renderer]; if (![self.rendererNV12 addRenderingDestination:self.metalView]) { diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m index a2290c2458..cbeddfe627 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m @@ -12,7 +12,6 @@ #import "WebRTC/RTCCameraVideoCapturer.h" #import "WebRTC/RTCLogging.h" -#import "WebRTC/RTCVideoFrameBuffer.h" #if TARGET_OS_IPHONE #import "WebRTC/UIDevice+RTCDevice.h" @@ -192,12 +191,11 @@ static inline BOOL IsMediaSubTypeSupported(FourCharCode mediaSubType) { return; } - RTCCVPixelBuffer *rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer]; int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * kNanosecondsPerSecond; - RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer - rotation:_rotation - timeStampNs:timeStampNs]; + RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithPixelBuffer:pixelBuffer + rotation:_rotation + timeStampNs:timeStampNs]; [self.delegate capturer:self didCaptureVideoFrame:videoFrame]; } diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCFileVideoCapturer.m b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCFileVideoCapturer.m index 178a958041..933207df44 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCFileVideoCapturer.m +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCFileVideoCapturer.m @@ -11,7 +11,6 @@ #import "RTCFileVideoCapturer.h" #import "WebRTC/RTCLogging.h" -#import "WebRTC/RTCVideoFrameBuffer.h" @implementation RTCFileVideoCapturer { AVAssetReader *_reader; @@ -134,11 +133,10 @@ return; } - RTCCVPixelBuffer *rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer]; NSTimeInterval timeStampSeconds = CACurrentMediaTime(); int64_t timeStampNs = lroundf(timeStampSeconds * NSEC_PER_SEC); RTCVideoFrame *videoFrame = - [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer rotation:0 timeStampNs:timeStampNs]; + [[RTCVideoFrame alloc] initWithPixelBuffer:pixelBuffer rotation:0 timeStampNs:timeStampNs]; CFRelease(sampleBuffer); dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame+Private.h b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame+Private.h new file mode 100644 index 0000000000..3b36f5b008 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoFrame.h" + +#include "webrtc/api/video/video_frame_buffer.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCVideoFrame () + +@property(nonatomic, readonly) rtc::scoped_refptr videoBuffer; + +- (instancetype)initWithVideoBuffer: + (rtc::scoped_refptr)videoBuffer + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm index ef93fef8cd..20e8f927e2 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm @@ -8,22 +8,22 @@ * be found in the AUTHORS file in the root of the source tree. */ -#import "webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h" -#import "webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h" +#import "RTCVideoFrame+Private.h" + +#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" @implementation RTCVideoFrame { + rtc::scoped_refptr _videoBuffer; RTCVideoRotation _rotation; int64_t _timeStampNs; } -@synthesize buffer = _buffer; - - (int)width { - return _buffer.width; + return _videoBuffer->width(); } - (int)height { - return _buffer.height; + return _videoBuffer->height(); } - (RTCVideoRotation)rotation { @@ -31,51 +31,27 @@ } - (const uint8_t *)dataY { - if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { - return ((id)_buffer).dataY; - } else { - return nullptr; - } + return _videoBuffer->GetI420()->DataY(); } - (const uint8_t *)dataU { - if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { - return ((id)_buffer).dataU; - } else { - return nullptr; - } + return _videoBuffer->GetI420()->DataU(); } - (const uint8_t *)dataV { - if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { - return ((id)_buffer).dataV; - } else { - return nullptr; - } + return _videoBuffer->GetI420()->DataV(); } - (int)strideY { - if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { - return ((id)_buffer).strideY; - } else { - return 0; - } + return _videoBuffer->GetI420()->StrideY(); } - (int)strideU { - if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { - return ((id)_buffer).strideU; - } else { - return 0; - } + return _videoBuffer->GetI420()->StrideU(); } - (int)strideV { - if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { - return ((id)_buffer).strideV; - } else { - return 0; - } + return _videoBuffer->GetI420()->StrideV(); } - (int64_t)timeStampNs { @@ -83,25 +59,26 @@ } - (CVPixelBufferRef)nativeHandle { - if ([_buffer isKindOfClass:[RTCCVPixelBuffer class]]) { - return ((RTCCVPixelBuffer *)_buffer).pixelBuffer; - } else { - return nullptr; - } + return (_videoBuffer->type() == webrtc::VideoFrameBuffer::Type::kNative) ? + static_cast(_videoBuffer.get())->pixel_buffer() : + nil; } - (RTCVideoFrame *)newI420VideoFrame { - return [[RTCVideoFrame alloc] initWithBuffer:[_buffer toI420] - rotation:_rotation - timeStampNs:_timeStampNs]; + return [[RTCVideoFrame alloc] + initWithVideoBuffer:_videoBuffer->ToI420() + rotation:_rotation + timeStampNs:_timeStampNs]; } - (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer rotation:(RTCVideoRotation)rotation timeStampNs:(int64_t)timeStampNs { - return [self initWithBuffer:[[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer] - rotation:rotation - timeStampNs:timeStampNs]; + rtc::scoped_refptr videoBuffer( + new rtc::RefCountedObject(pixelBuffer)); + return [self initWithVideoBuffer:videoBuffer + rotation:rotation + timeStampNs:timeStampNs]; } - (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer @@ -113,26 +90,33 @@ cropY:(int)cropY rotation:(RTCVideoRotation)rotation timeStampNs:(int64_t)timeStampNs { - RTCCVPixelBuffer *rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer - adaptedWidth:scaledWidth - adaptedHeight:scaledHeight - cropWidth:cropWidth - cropHeight:cropHeight - cropX:cropX - cropY:cropY]; - return [self initWithBuffer:rtcPixelBuffer rotation:rotation timeStampNs:timeStampNs]; + rtc::scoped_refptr videoBuffer( + new rtc::RefCountedObject( + pixelBuffer, + scaledWidth, scaledHeight, + cropWidth, cropHeight, + cropX, cropY)); + return [self initWithVideoBuffer:videoBuffer + rotation:rotation + timeStampNs:timeStampNs]; } -- (instancetype)initWithBuffer:(id)buffer - rotation:(RTCVideoRotation)rotation - timeStampNs:(int64_t)timeStampNs { +#pragma mark - Private + +- (instancetype)initWithVideoBuffer: + (rtc::scoped_refptr)videoBuffer + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs { if (self = [super init]) { - _buffer = buffer; + _videoBuffer = videoBuffer; _rotation = rotation; _timeStampNs = timeStampNs; } - return self; } +- (rtc::scoped_refptr)videoBuffer { + return _videoBuffer; +} + @end diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.mm index 736803b913..1d5107b1b0 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.mm @@ -8,11 +8,9 @@ * be found in the AUTHORS file in the root of the source tree. */ -#import "RTCI420Buffer+Private.h" #import "RTCVideoRendererAdapter+Private.h" -#import "WebRTC/RTCVideoFrame.h" -#import "WebRTC/RTCVideoFrameBuffer.h" -#import "objc_frame_buffer.h" + +#import "RTCVideoFrame+Private.h" #include @@ -27,20 +25,12 @@ class VideoRendererAdapter } void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override { - rtc::scoped_refptr video_frame_buffer = nativeVideoFrame.video_frame_buffer(); - id rtc_frame_buffer; - if (video_frame_buffer->type() == VideoFrameBuffer::Type::kNative) { - rtc::scoped_refptr objc_frame_buffer( - static_cast(video_frame_buffer.get())); - rtc_frame_buffer = (id)objc_frame_buffer->wrapped_frame_buffer(); - } else { - rtc_frame_buffer = [[RTCI420Buffer alloc] initWithFrameBuffer:video_frame_buffer->ToI420()]; - } RTCVideoFrame* videoFrame = [[RTCVideoFrame alloc] - initWithBuffer:rtc_frame_buffer - rotation:static_cast(nativeVideoFrame.rotation()) - timeStampNs:nativeVideoFrame.timestamp_us() * rtc::kNumNanosecsPerMicrosec]; - + initWithVideoBuffer:nativeVideoFrame.video_frame_buffer() + rotation:static_cast( + nativeVideoFrame.rotation()) + timeStampNs:nativeVideoFrame.timestamp_us() * + rtc::kNumNanosecsPerMicrosec]; CGSize current_size = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width); diff --git a/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m b/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m index 89b7dfcc2e..851c4b4a9e 100644 --- a/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m +++ b/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m @@ -17,7 +17,6 @@ #import "RTCNV12TextureCache.h" #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" -#import "WebRTC/RTCVideoFrameBuffer.h" // RTCDisplayLinkTimer wraps a CADisplayLink and is set to fire every two screen // refreshes, which should be 30fps. We wrap the display link in order to avoid @@ -221,7 +220,7 @@ } [self ensureGLContext]; glClear(GL_COLOR_BUFFER_BIT); - if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + if (frame.nativeHandle) { if (!_nv12TextureCache) { _nv12TextureCache = [[RTCNV12TextureCache alloc] initWithContext:_glContext]; } diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm deleted file mode 100644 index a45c212713..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "WebRTC/RTCVideoFrameBuffer.h" - -#include "webrtc/base/checks.h" -#include "webrtc/base/logging.h" -#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" - -@implementation RTCCVPixelBuffer { - int _width; - int _height; - int _bufferWidth; - int _bufferHeight; - int _cropWidth; - int _cropHeight; - int _cropX; - int _cropY; -} - -@synthesize pixelBuffer = _pixelBuffer; - -- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer { - return [self initWithPixelBuffer:pixelBuffer - adaptedWidth:CVPixelBufferGetWidth(pixelBuffer) - adaptedHeight:CVPixelBufferGetHeight(pixelBuffer) - cropWidth:CVPixelBufferGetWidth(pixelBuffer) - cropHeight:CVPixelBufferGetHeight(pixelBuffer) - cropX:0 - cropY:0]; -} - -- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer - adaptedWidth:(int)adaptedWidth - adaptedHeight:(int)adaptedHeight - cropWidth:(int)cropWidth - cropHeight:(int)cropHeight - cropX:(int)cropX - cropY:(int)cropY { - if (self = [super init]) { - _width = adaptedWidth; - _height = adaptedHeight; - _pixelBuffer = pixelBuffer; - _bufferWidth = CVPixelBufferGetWidth(_pixelBuffer); - _bufferHeight = CVPixelBufferGetHeight(_pixelBuffer); - _cropWidth = cropWidth; - _cropHeight = cropHeight; - // Can only crop at even pixels. - _cropX = cropX & ~1; - _cropY = cropY & ~1; - CVBufferRetain(_pixelBuffer); - } - - return self; -} - -- (void)dealloc { - CVBufferRelease(_pixelBuffer); -} - -- (int)width { - return _width; -} - -- (int)height { - return _height; -} - -- (BOOL)requiresCropping { - return _cropWidth != _bufferWidth || _cropHeight != _bufferHeight; -} - -- (BOOL)requiresScalingToWidth:(int)width height:(int)height { - return _cropWidth != width || _cropHeight != height; -} - -- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height { - int srcChromaWidth = (_cropWidth + 1) / 2; - int srcChromaHeight = (_cropHeight + 1) / 2; - int dstChromaWidth = (width + 1) / 2; - int dstChromaHeight = (height + 1) / 2; - - return srcChromaWidth * srcChromaHeight * 2 + dstChromaWidth * dstChromaHeight * 2; -} - -- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer { - // Prepare output pointers. - RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(outputPixelBuffer), - kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); - CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); - if (cvRet != kCVReturnSuccess) { - LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; - return NO; - } - const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); - const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); - uint8_t* dstY = - reinterpret_cast(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0)); - const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0); - uint8_t* dstUV = - reinterpret_cast(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1)); - const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1); - - // Prepare source pointers. - const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); - RTC_DCHECK(srcPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || - srcPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); - CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); - const uint8_t* srcY = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); - const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); - const uint8_t* srcUV = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); - const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); - - // Crop just by modifying pointers. - srcY += srcYStride * _cropY + _cropX; - srcUV += srcUVStride * (_cropY / 2) + _cropX; - - webrtc::NV12Scale(tmpBuffer, - srcY, - srcYStride, - srcUV, - srcUVStride, - _cropWidth, - _cropHeight, - dstY, - dstYStride, - dstUV, - dstUVStride, - dstWidth, - dstHeight); - - CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); - CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); - - return YES; -} - -- (id)toI420 { - const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); - RTC_DCHECK(pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || - pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); - - CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); - const uint8_t* srcY = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); - const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); - const uint8_t* srcUV = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); - const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); - - // Crop just by modifying pointers. - srcY += srcYStride * _cropY + _cropX; - srcUV += srcUVStride * (_cropY / 2) + _cropX; - - // TODO(magjed): Use a frame buffer pool. - webrtc::NV12ToI420Scaler nv12ToI420Scaler; - RTCMutableI420Buffer* i420Buffer = - [[RTCMutableI420Buffer alloc] initWithWidth:[self width] height:[self height]]; - nv12ToI420Scaler.NV12ToI420Scale(srcY, - srcYStride, - srcUV, - srcUVStride, - _cropWidth, - _cropHeight, - i420Buffer.mutableDataY, - i420Buffer.strideY, - i420Buffer.mutableDataU, - i420Buffer.strideU, - i420Buffer.mutableDataV, - i420Buffer.strideV, - i420Buffer.width, - i420Buffer.height); - - CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); - - return i420Buffer; -} - -@end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer+Private.h b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer+Private.h deleted file mode 100644 index 7883c414b5..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer+Private.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "WebRTC/RTCVideoFrameBuffer.h" - -#include "webrtc/api/video/i420_buffer.h" - -NS_ASSUME_NONNULL_BEGIN - -@interface RTCI420Buffer () - -/** Initialize an RTCI420Buffer with its backing I420BufferInterface. */ -- (instancetype)initWithFrameBuffer:(rtc::scoped_refptr)i420Buffer; - -@end - -NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm deleted file mode 100644 index 042fcc06ba..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "WebRTC/RTCVideoFrameBuffer.h" - -#include "webrtc/api/video/i420_buffer.h" - -@implementation RTCI420Buffer { - @protected - rtc::scoped_refptr _i420Buffer; -} - -- (instancetype)initWithWidth:(int)width height:(int)height { - if (self = [super init]) { - _i420Buffer = webrtc::I420Buffer::Create(width, height); - } - - return self; -} - -- (instancetype)initWithWidth:(int)width - height:(int)height - strideY:(int)strideY - strideU:(int)strideU - strideV:(int)strideV { - if (self = [super init]) { - _i420Buffer = webrtc::I420Buffer::Create(width, height, strideY, strideU, strideV); - } - - return self; -} - -- (instancetype)initWithFrameBuffer:(rtc::scoped_refptr)i420Buffer { - if (self = [super init]) { - _i420Buffer = i420Buffer; - } - - return self; -} - -- (int)width { - return _i420Buffer->width(); -} - -- (int)height { - return _i420Buffer->height(); -} - -- (int)strideY { - return _i420Buffer->StrideY(); -} - -- (int)strideU { - return _i420Buffer->StrideU(); -} - -- (int)strideV { - return _i420Buffer->StrideV(); -} - -- (int)chromaWidth { - return _i420Buffer->ChromaWidth(); -} - -- (int)chromaHeight { - return _i420Buffer->ChromaHeight(); -} - -- (const uint8_t *)dataY { - return _i420Buffer->DataY(); -} - -- (const uint8_t *)dataU { - return _i420Buffer->DataU(); -} - -- (const uint8_t *)dataV { - return _i420Buffer->DataV(); -} - -- (id)toI420 { - return self; -} - -@end - -@implementation RTCMutableI420Buffer - -- (uint8_t *)mutableDataY { - return static_cast(_i420Buffer.get())->MutableDataY(); -} - -- (uint8_t *)mutableDataU { - return static_cast(_i420Buffer.get())->MutableDataU(); -} - -- (uint8_t *)mutableDataV { - return static_cast(_i420Buffer.get())->MutableDataV(); -} - -@end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm index b603130ea7..eeae867bc0 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm @@ -9,7 +9,6 @@ */ #import "RTCI420TextureCache.h" -#import "WebRTC/RTCVideoFrameBuffer.h" #if TARGET_OS_IPHONE #import @@ -124,32 +123,31 @@ static const GLsizei kNumTextures = kNumTexturesPerSet * kNumTextureSets; - (void)uploadFrameToTextures:(RTCVideoFrame *)frame { _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; - id buffer = [frame.buffer toI420]; - - const int chromaWidth = buffer.chromaWidth; - const int chromaHeight = buffer.chromaHeight; - if (buffer.strideY != frame.width || buffer.strideU != chromaWidth || - buffer.strideV != chromaWidth) { - _planeBuffer.resize(buffer.width * buffer.height); + const int chromaWidth = (frame.width + 1) / 2; + const int chromaHeight = (frame.height + 1) / 2; + if (frame.strideY != frame.width || + frame.strideU != chromaWidth || + frame.strideV != chromaWidth) { + _planeBuffer.resize(frame.width * frame.height); } - [self uploadPlane:buffer.dataY + [self uploadPlane:frame.dataY texture:self.yTexture - width:buffer.width - height:buffer.height - stride:buffer.strideY]; + width:frame.width + height:frame.height + stride:frame.strideY]; - [self uploadPlane:buffer.dataU + [self uploadPlane:frame.dataU texture:self.uTexture width:chromaWidth height:chromaHeight - stride:buffer.strideU]; + stride:frame.strideU]; - [self uploadPlane:buffer.dataV + [self uploadPlane:frame.dataV texture:self.vTexture width:chromaWidth height:chromaHeight - stride:buffer.strideV]; + stride:frame.strideV]; } @end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.m b/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.m index 20a6082a78..e259cee193 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.m +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.m @@ -11,7 +11,6 @@ #import "RTCNV12TextureCache.h" #import "WebRTC/RTCVideoFrame.h" -#import "WebRTC/RTCVideoFrameBuffer.h" @implementation RTCNV12TextureCache { CVOpenGLESTextureCacheRef _textureCache; @@ -74,10 +73,8 @@ } - (BOOL)uploadFrameToTextures:(RTCVideoFrame *)frame { - NSAssert([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]], - @"frame must be CVPixelBuffer backed"); - RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer; - CVPixelBufferRef pixelBuffer = rtcPixelBuffer.pixelBuffer; + CVPixelBufferRef pixelBuffer = frame.nativeHandle; + NSParameterAssert(pixelBuffer); return [self loadTexture:&_yTextureRef pixelBuffer:pixelBuffer planeIndex:0 diff --git a/webrtc/sdk/objc/Framework/Classes/Video/avfoundationvideocapturer.mm b/webrtc/sdk/objc/Framework/Classes/Video/avfoundationvideocapturer.mm index c2e20564b6..a060b51c16 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/avfoundationvideocapturer.mm +++ b/webrtc/sdk/objc/Framework/Classes/Video/avfoundationvideocapturer.mm @@ -15,7 +15,6 @@ #import "RTCAVFoundationVideoCapturerInternal.h" #import "RTCDispatcher+Private.h" #import "WebRTC/RTCLogging.h" -#import "WebRTC/RTCVideoFrameBuffer.h" #include "avfoundationformatmapper.h" @@ -24,7 +23,7 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/thread.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" namespace webrtc { @@ -151,15 +150,12 @@ void AVFoundationVideoCapturer::CaptureSampleBuffer( return; } - RTCCVPixelBuffer* rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:image_buffer - adaptedWidth:adapted_width - adaptedHeight:adapted_height - cropWidth:crop_width - cropHeight:crop_height - cropX:crop_x - cropY:crop_y]; rtc::scoped_refptr buffer = - new rtc::RefCountedObject(rtcPixelBuffer); + new rtc::RefCountedObject( + image_buffer, + adapted_width, adapted_height, + crop_width, crop_height, + crop_x, crop_y); // Applying rotation is only supported for legacy reasons and performance is // not critical here. diff --git a/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h b/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h deleted file mode 100644 index 71099e446b..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJC_FRAME_BUFFER_H_ -#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJC_FRAME_BUFFER_H_ - -#import - -#include "webrtc/common_video/include/video_frame_buffer.h" - -@protocol RTCVideoFrameBuffer; - -namespace webrtc { - -class ObjCFrameBuffer : public VideoFrameBuffer { - public: - explicit ObjCFrameBuffer(id); - ~ObjCFrameBuffer() override; - - Type type() const override; - - int width() const override; - int height() const override; - - rtc::scoped_refptr ToI420() override; - - id wrapped_frame_buffer() const; - - private: - id frame_buffer_; - int width_; - int height_; -}; - -} // namespace webrtc - -#endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJC_FRAME_BUFFER_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.mm b/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.mm deleted file mode 100644 index 74e9c13477..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.mm +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" - -#import "WebRTC/RTCVideoFrameBuffer.h" - -namespace webrtc { - -namespace { - -/** ObjCFrameBuffer that conforms to I420BufferInterface by wrapping RTCI420Buffer */ -class ObjCI420FrameBuffer : public I420BufferInterface { - public: - explicit ObjCI420FrameBuffer(id frame_buffer) - : frame_buffer_(frame_buffer), width_(frame_buffer.width), height_(frame_buffer.height) {} - ~ObjCI420FrameBuffer() override{}; - - int width() const override { return width_; } - - int height() const override { return height_; } - - const uint8_t* DataY() const override { return frame_buffer_.dataY; } - - const uint8_t* DataU() const override { return frame_buffer_.dataU; } - - const uint8_t* DataV() const override { return frame_buffer_.dataV; } - - int StrideY() const override { return frame_buffer_.strideY; } - - int StrideU() const override { return frame_buffer_.strideU; } - - int StrideV() const override { return frame_buffer_.strideV; } - - private: - id frame_buffer_; - int width_; - int height_; -}; - -} // namespace - -ObjCFrameBuffer::ObjCFrameBuffer(id frame_buffer) - : frame_buffer_(frame_buffer), width_(frame_buffer.width), height_(frame_buffer.height) {} - -ObjCFrameBuffer::~ObjCFrameBuffer() {} - -VideoFrameBuffer::Type ObjCFrameBuffer::type() const { - return Type::kNative; -} - -int ObjCFrameBuffer::width() const { - return width_; -} - -int ObjCFrameBuffer::height() const { - return height_; -} - -rtc::scoped_refptr ObjCFrameBuffer::ToI420() { - rtc::scoped_refptr buffer = - new rtc::RefCountedObject([frame_buffer_ toI420]); - - return buffer; -} - -id ObjCFrameBuffer::wrapped_frame_buffer() const { - return frame_buffer_; -} - -} // namespace webrtc diff --git a/webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.mm b/webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.mm index 4200cbc891..387bb1bd13 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.mm +++ b/webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.mm @@ -10,11 +10,10 @@ #include "webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.h" -#import "WebRTC/RTCVideoFrame.h" -#import "WebRTC/RTCVideoFrameBuffer.h" +#import "RTCVideoFrame+Private.h" #include "webrtc/api/video/i420_buffer.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" namespace webrtc { @@ -44,24 +43,18 @@ void ObjcVideoTrackSource::OnCapturedFrame(RTCVideoFrame* frame) { rtc::scoped_refptr buffer; if (adapted_width == frame.width && adapted_height == frame.height) { // No adaption - optimized path. - buffer = new rtc::RefCountedObject(frame.buffer); - } else if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + buffer = frame.videoBuffer; + } else if (frame.nativeHandle) { // Adapted CVPixelBuffer frame. - RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer; - buffer = new rtc::RefCountedObject([[RTCCVPixelBuffer alloc] - initWithPixelBuffer:rtcPixelBuffer.pixelBuffer - adaptedWidth:adapted_width - adaptedHeight:adapted_height - cropWidth:crop_width - cropHeight:crop_height - cropX:crop_x - cropY:crop_y]); + buffer = new rtc::RefCountedObject( + static_cast(frame.nativeHandle), adapted_width, adapted_height, + crop_width, crop_height, crop_x, crop_y); } else { // Adapted I420 frame. // TODO(magjed): Optimize this I420 path. rtc::scoped_refptr i420_buffer = I420Buffer::Create(adapted_width, adapted_height); - buffer = new rtc::RefCountedObject(frame.buffer); - i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height); + i420_buffer->CropAndScaleFrom( + *frame.videoBuffer->ToI420(), crop_x, crop_y, crop_width, crop_height); buffer = i420_buffer; } diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm index c039442eac..1698477e82 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm @@ -18,11 +18,9 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/common_video/include/video_frame.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" -#import "WebRTC/RTCVideoFrameBuffer.h" - #if defined(WEBRTC_IOS) #import "Common/RTCUIApplicationStatusObserver.h" #endif @@ -66,8 +64,8 @@ void VTDecompressionOutputCallback(void* decoder, return; } // TODO(tkchin): Handle CVO properly. - rtc::scoped_refptr buffer = new rtc::RefCountedObject( - [[RTCCVPixelBuffer alloc] initWithPixelBuffer:image_buffer]); + rtc::scoped_refptr buffer = + new rtc::RefCountedObject(image_buffer); VideoFrame decoded_frame(buffer, decode_params->timestamp, CMTimeGetSeconds(timestamp) * kMsPerSec, kVideoRotation_0); diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm index 2bd485fc78..1d37ebaed6 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm @@ -19,12 +19,11 @@ #import "Common/RTCUIApplicationStatusObserver.h" #import "WebRTC/UIDevice+RTCDevice.h" #endif -#import "WebRTC/RTCVideoFrameBuffer.h" #include "libyuv/convert_from.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/common_video/h264/profile_level_id.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" #include "webrtc/system_wrappers/include/clock.h" @@ -412,49 +411,29 @@ int H264VideoToolboxEncoder::Encode( } #endif - CVPixelBufferRef pixel_buffer = nullptr; + CVPixelBufferRef pixel_buffer; if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) { - // Native frame. - rtc::scoped_refptr objc_frame_buffer( - static_cast(frame.video_frame_buffer().get())); - id wrapped_frame_buffer = - (id)objc_frame_buffer->wrapped_frame_buffer(); - - if ([wrapped_frame_buffer isKindOfClass:[RTCCVPixelBuffer class]]) { - RTCCVPixelBuffer* rtc_pixel_buffer = (RTCCVPixelBuffer*)wrapped_frame_buffer; - if (![rtc_pixel_buffer requiresCropping]) { - // This pixel buffer might have a higher resolution than what the - // compression session is configured to. The compression session can - // handle that and will output encoded frames in the configured - // resolution regardless of the input pixel buffer resolution. - pixel_buffer = rtc_pixel_buffer.pixelBuffer; - CVBufferRetain(pixel_buffer); - } else { - // Cropping required, we need to crop and scale to a new pixel buffer. - pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); - if (!pixel_buffer) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - int dst_width = CVPixelBufferGetWidth(pixel_buffer); - int dst_height = CVPixelBufferGetHeight(pixel_buffer); - if ([rtc_pixel_buffer requiresScalingToWidth:dst_width height:dst_height]) { - int size = - [rtc_pixel_buffer bufferSizeForCroppingAndScalingToWidth:dst_width height:dst_height]; - nv12_scale_buffer_.resize(size); - } else { - nv12_scale_buffer_.clear(); - } - nv12_scale_buffer_.shrink_to_fit(); - if (![rtc_pixel_buffer cropAndScaleTo:pixel_buffer - withTempBuffer:nv12_scale_buffer_.data()]) { - return WEBRTC_VIDEO_CODEC_ERROR; - } + rtc::scoped_refptr core_video_frame_buffer( + static_cast(frame.video_frame_buffer().get())); + if (!core_video_frame_buffer->RequiresCropping()) { + pixel_buffer = core_video_frame_buffer->pixel_buffer(); + // This pixel buffer might have a higher resolution than what the + // compression session is configured to. The compression session can + // handle that and will output encoded frames in the configured + // resolution regardless of the input pixel buffer resolution. + CVBufferRetain(pixel_buffer); + } else { + // Cropping required, we need to crop and scale to a new pixel buffer. + pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); + if (!pixel_buffer) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_, + pixel_buffer)) { + return WEBRTC_VIDEO_CODEC_ERROR; } } - } - - if (!pixel_buffer) { - // We did not have a native frame, or the ObjCVideoFrame wrapped a non-native frame + } else { pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); if (!pixel_buffer) { return WEBRTC_VIDEO_CODEC_ERROR; diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h index 78eee5f224..ddf8a6b628 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h @@ -22,8 +22,6 @@ typedef NS_ENUM(NSInteger, RTCVideoRotation) { RTCVideoRotation_270 = 270, }; -@protocol RTCVideoFrameBuffer; - // RTCVideoFrame is an ObjectiveC version of webrtc::VideoFrame. RTC_EXPORT @interface RTCVideoFrame : NSObject @@ -38,35 +36,27 @@ RTC_EXPORT * is null. It is always possible to get such a frame by calling * newI420VideoFrame. */ -@property(nonatomic, readonly, nullable) - const uint8_t *dataY DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); -@property(nonatomic, readonly, nullable) - const uint8_t *dataU DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); -@property(nonatomic, readonly, nullable) - const uint8_t *dataV DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); -@property(nonatomic, readonly) int strideY DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); -@property(nonatomic, readonly) int strideU DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); -@property(nonatomic, readonly) int strideV DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); +@property(nonatomic, readonly, nullable) const uint8_t *dataY; +@property(nonatomic, readonly, nullable) const uint8_t *dataU; +@property(nonatomic, readonly, nullable) const uint8_t *dataV; +@property(nonatomic, readonly) int strideY; +@property(nonatomic, readonly) int strideU; +@property(nonatomic, readonly) int strideV; /** Timestamp in nanoseconds. */ @property(nonatomic, readonly) int64_t timeStampNs; /** The native handle should be a pixel buffer on iOS. */ -@property(nonatomic, readonly) - CVPixelBufferRef nativeHandle DEPRECATED_MSG_ATTRIBUTE("use buffer instead"); - -@property(nonatomic, readonly) id buffer; +@property(nonatomic, readonly) CVPixelBufferRef nativeHandle; - (instancetype)init NS_UNAVAILABLE; - (instancetype)new NS_UNAVAILABLE; /** Initialize an RTCVideoFrame from a pixel buffer, rotation, and timestamp. - * Deprecated - initialize with a RTCCVPixelBuffer instead */ - (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer rotation:(RTCVideoRotation)rotation - timeStampNs:(int64_t)timeStampNs - DEPRECATED_MSG_ATTRIBUTE("use initWithBuffer instead"); + timeStampNs:(int64_t)timeStampNs; /** Initialize an RTCVideoFrame from a pixel buffer combined with cropping and * scaling. Cropping will be applied first on the pixel buffer, followed by @@ -80,14 +70,7 @@ RTC_EXPORT cropX:(int)cropX cropY:(int)cropY rotation:(RTCVideoRotation)rotation - timeStampNs:(int64_t)timeStampNs - DEPRECATED_MSG_ATTRIBUTE("use initWithBuffer instead"); - -/** Initialize an RTCVideoFrame from a frame buffer, rotation, and timestamp. - */ -- (instancetype)initWithBuffer:(id)frameBuffer - rotation:(RTCVideoRotation)rotation - timeStampNs:(int64_t)timeStampNs; + timeStampNs:(int64_t)timeStampNs; /** Return a frame that is guaranteed to be I420, i.e. it is possible to access * the YUV data on it. diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h deleted file mode 100644 index 59986b8da3..0000000000 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import -#import - -NS_ASSUME_NONNULL_BEGIN - -@protocol RTCI420Buffer; - -// RTCVideoFrameBuffer is an ObjectiveC version of webrtc::VideoFrameBuffer. -RTC_EXPORT -@protocol RTCVideoFrameBuffer - -@property(nonatomic, readonly) int width; -@property(nonatomic, readonly) int height; - -- (id)toI420; - -@end - -/** Protocol for RTCVideoFrameBuffers containing YUV planar data. */ -@protocol RTCYUVPlanarBuffer - -@property(nonatomic, readonly) int chromaWidth; -@property(nonatomic, readonly) int chromaHeight; -@property(nonatomic, readonly) const uint8_t *dataY; -@property(nonatomic, readonly) const uint8_t *dataU; -@property(nonatomic, readonly) const uint8_t *dataV; -@property(nonatomic, readonly) int strideY; -@property(nonatomic, readonly) int strideU; -@property(nonatomic, readonly) int strideV; - -- (instancetype)initWithWidth:(int)width height:(int)height; -- (instancetype)initWithWidth:(int)width - height:(int)height - strideY:(int)strideY - strideU:(int)strideU - strideV:(int)strideV; - -@end - -/** Extension of the YUV planar data buffer with mutable data access */ -@protocol RTCMutableYUVPlanarBuffer - -@property(nonatomic, readonly) uint8_t *mutableDataY; -@property(nonatomic, readonly) uint8_t *mutableDataU; -@property(nonatomic, readonly) uint8_t *mutableDataV; - -@end - -/** Protocol for RTCYUVPlanarBuffers containing I420 data */ -@protocol RTCI420Buffer -@end - -/** Extension of the I420 buffer with mutable data access */ -@protocol RTCMutableI420Buffer -@end - -/** RTCVideoFrameBuffer containing a CVPixelBufferRef */ -@interface RTCCVPixelBuffer : NSObject - -@property(nonatomic, readonly) CVPixelBufferRef pixelBuffer; - -- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer; -- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer - adaptedWidth:(int)adaptedWidth - adaptedHeight:(int)adaptedHeight - cropWidth:(int)cropWidth - cropHeight:(int)cropHeight - cropX:(int)cropX - cropY:(int)cropY; - -- (BOOL)requiresCropping; -- (BOOL)requiresScalingToWidth:(int)width height:(int)height; -- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height; -/** The minimum size of the |tmpBuffer| must be the number of bytes returned from the - * bufferSizeForCroppingAndScalingToWidth:height: method. - */ -- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t *)tmpBuffer; - -@end - -/** RTCI420Buffer implements the RTCI420Buffer protocol */ -@interface RTCI420Buffer : NSObject -@end - -/** Mutable version of RTCI420Buffer */ -@interface RTCMutableI420Buffer : RTCI420Buffer -@end - -NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm b/webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm index 3743326cdc..99004ea515 100644 --- a/webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm +++ b/webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm @@ -15,7 +15,6 @@ #include #include -#include // Extension of RTCMTLVideoView for testing purposes. @interface RTCMTLVideoView (Testing) @@ -60,14 +59,12 @@ self.frameMock = nil; } -- (id)frameMockWithCVPixelBuffer:(BOOL)hasCVPixelBuffer { +- (id)frameMockWithNativeHandle:(BOOL)hasNativeHandle { id frameMock = OCMClassMock([RTCVideoFrame class]); - if (hasCVPixelBuffer) { - OCMStub([frameMock buffer]) - .andReturn( - [[RTCCVPixelBuffer alloc] initWithPixelBuffer:(CVPixelBufferRef)[OCMArg anyPointer]]); + if (hasNativeHandle) { + OCMStub([frameMock nativeHandle]).andReturn((CVPixelBufferRef)[OCMArg anyPointer]); } else { - OCMStub([frameMock buffer]).andReturn([[RTCI420Buffer alloc] initWithWidth:200 height:200]); + OCMStub([frameMock nativeHandle]).andReturn((CVPixelBufferRef) nullptr); } return frameMock; } @@ -102,7 +99,7 @@ RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init]; self.frameMock = OCMClassMock([RTCVideoFrame class]); - [[self.frameMock reject] buffer]; + [[self.frameMock reject] nativeHandle]; [[self.classMock reject] createNV12Renderer]; [[self.classMock reject] createI420Renderer]; @@ -119,7 +116,7 @@ // given OCMStub([self.classMock isMetalAvailable]).andReturn(YES); self.rendererI420Mock = [self rendererMockWithSuccessfulSetup:YES]; - self.frameMock = [self frameMockWithCVPixelBuffer:NO]; + self.frameMock = [self frameMockWithNativeHandle:NO]; OCMExpect([self.rendererI420Mock drawFrame:self.frameMock]); OCMExpect([self.classMock createI420Renderer]).andReturn(self.rendererI420Mock); @@ -140,7 +137,7 @@ // given OCMStub([self.classMock isMetalAvailable]).andReturn(YES); self.rendererNV12Mock = [self rendererMockWithSuccessfulSetup:YES]; - self.frameMock = [self frameMockWithCVPixelBuffer:YES]; + self.frameMock = [self frameMockWithNativeHandle:YES]; OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]); OCMExpect([self.classMock createNV12Renderer]).andReturn(self.rendererNV12Mock);