From 13941912b16b522d37c643e0c4413ce378c817f8 Mon Sep 17 00:00:00 2001 From: magjed Date: Tue, 30 May 2017 06:11:58 -0700 Subject: [PATCH] ObjC: Add interface for injecting custom shaders in video views This CL adds a way for external clients to inject their own OpenGL(ES) shaders to RTCEAGLVideoView/RTCNSGLVideoView. The shader interface takes textures as arguments, and not RTCVideoFrame, so that implementations only has to deal with actual OpenGL rendering, and not converting frames into textures. This CL also moves the internal shader code around a bit. The current RTCShader interface with the implementations RTCI420Shader and RTCNativeNV12Shader are removed. RTCEAGLVideoView and RTCNSGLVideoView will be responsible for uploading the frames to textures instead using the helper classes RTCI420TextureCache and RTCNV12TextureCache. They then call the shader implementation with these textures. The rendering code that used to be in RTCI420Shader and RTCNativeNV12Shader have been merged into one RTCDefaultShaderDelegate class. BUG=webrtc:7473 Review-Url: https://codereview.webrtc.org/2869143002 Cr-Commit-Position: refs/heads/master@{#18326} --- webrtc/sdk/BUILD.gn | 7 +- .../Framework/Classes/UI/RTCEAGLVideoView.m | 53 +++-- .../Framework/Classes/UI/RTCNSGLVideoView.m | 38 +++- .../Classes/Video/RTCDefaultShader.h | 23 ++ .../Classes/Video/RTCDefaultShader.mm | 203 ++++++++++++++++++ .../Framework/Classes/Video/RTCI420Shader.mm | 119 ---------- .../Classes/Video/RTCI420TextureCache.h | 3 +- .../Classes/Video/RTCI420TextureCache.mm | 6 +- .../Classes/Video/RTCNV12TextureCache.h | 3 +- .../Classes/Video/RTCNativeNV12Shader.mm | 107 --------- .../Classes/Video/RTCShader+Private.h | 29 --- .../objc/Framework/Classes/Video/RTCShader.h | 41 +--- .../objc/Framework/Classes/Video/RTCShader.mm | 49 +++-- .../Headers/WebRTC/RTCEAGLVideoView.h | 9 +- .../Headers/WebRTC/RTCNSGLVideoView.h | 5 + .../Headers/WebRTC/RTCVideoViewShading.h | 37 ++++ 16 files changed, 388 insertions(+), 344 deletions(-) create mode 100644 webrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.h create mode 100644 webrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.mm delete mode 100644 webrtc/sdk/objc/Framework/Classes/Video/RTCI420Shader.mm delete mode 100644 webrtc/sdk/objc/Framework/Classes/Video/RTCNativeNV12Shader.mm delete mode 100644 webrtc/sdk/objc/Framework/Classes/Video/RTCShader+Private.h create mode 100644 webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoViewShading.h diff --git a/webrtc/sdk/BUILD.gn b/webrtc/sdk/BUILD.gn index c165d60c1f..c4915ca51d 100644 --- a/webrtc/sdk/BUILD.gn +++ b/webrtc/sdk/BUILD.gn @@ -83,11 +83,11 @@ if (is_ios || is_mac) { sources = [ "objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.h", "objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm", - "objc/Framework/Classes/Video/RTCI420Shader.mm", + "objc/Framework/Classes/Video/RTCDefaultShader.h", + "objc/Framework/Classes/Video/RTCDefaultShader.mm", "objc/Framework/Classes/Video/RTCI420TextureCache.h", "objc/Framework/Classes/Video/RTCI420TextureCache.mm", "objc/Framework/Classes/Video/RTCOpenGLDefines.h", - "objc/Framework/Classes/Video/RTCShader+Private.h", "objc/Framework/Classes/Video/RTCShader.h", "objc/Framework/Classes/Video/RTCShader.mm", "objc/Framework/Classes/Video/avfoundationformatmapper.h", @@ -102,7 +102,6 @@ if (is_ios || is_mac) { sources += [ "objc/Framework/Classes/Video/RTCNV12TextureCache.h", "objc/Framework/Classes/Video/RTCNV12TextureCache.m", - "objc/Framework/Classes/Video/RTCNativeNV12Shader.mm", ] libs += [ "GLKit.framework", @@ -289,6 +288,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCVideoRenderer.h", "objc/Framework/Headers/WebRTC/RTCVideoSource.h", "objc/Framework/Headers/WebRTC/RTCVideoTrack.h", + "objc/Framework/Headers/WebRTC/RTCVideoViewShading.h", ] libs = [ "AVFoundation.framework" ] @@ -421,6 +421,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCVideoRenderer.h", "objc/Framework/Headers/WebRTC/RTCVideoSource.h", "objc/Framework/Headers/WebRTC/RTCVideoTrack.h", + "objc/Framework/Headers/WebRTC/RTCVideoViewShading.h", "objc/Framework/Headers/WebRTC/UIDevice+RTCDevice.h", "objc/Framework/Headers/WebRTC/WebRTC.h", ] diff --git a/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m b/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m index 1fb03bc909..5713bf86e3 100644 --- a/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m +++ b/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m @@ -12,7 +12,9 @@ #import -#import "RTCShader+Private.h" +#import "RTCDefaultShader.h" +#import "RTCI420TextureCache.h" +#import "RTCNV12TextureCache.h" #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" @@ -97,8 +99,9 @@ // This flag should only be set and read on the main thread (e.g. by // setNeedsDisplay) BOOL _isDirty; - id _i420Shader; - id _nv12Shader; + id _shader; + RTCNV12TextureCache *_nv12TextureCache; + RTCI420TextureCache *_i420TextureCache; RTCVideoFrame *_lastDrawnFrame; } @@ -107,14 +110,24 @@ @synthesize glkView = _glkView; - (instancetype)initWithFrame:(CGRect)frame { + return [self initWithFrame:frame shader:[[RTCDefaultShader alloc] init]]; +} + +- (instancetype)initWithCoder:(NSCoder *)aDecoder { + return [self initWithCoder:aDecoder shader:[[RTCDefaultShader alloc] init]]; +} + +- (instancetype)initWithFrame:(CGRect)frame shader:(id)shader { if (self = [super initWithFrame:frame]) { + _shader = shader; [self configure]; } return self; } -- (instancetype)initWithCoder:(NSCoder *)aDecoder { +- (instancetype)initWithCoder:(NSCoder *)aDecoder shader:(id)shader { if (self = [super initWithCoder:aDecoder]) { + _shader = shader; [self configure]; } return self; @@ -207,22 +220,26 @@ } [self ensureGLContext]; glClear(GL_COLOR_BUFFER_BIT); - id shader = nil; if (frame.nativeHandle) { - if (!_nv12Shader) { - _nv12Shader = [[RTCNativeNV12Shader alloc] initWithContext:_glContext]; + if (!_nv12TextureCache) { + _nv12TextureCache = [[RTCNV12TextureCache alloc] initWithContext:_glContext]; } - shader = _nv12Shader; - } else { - if (!_i420Shader) { - _i420Shader = [[RTCI420Shader alloc] initWithContext:_glContext]; + if (_nv12TextureCache) { + [_nv12TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithRotation:frame.rotation + yPlane:_nv12TextureCache.yTexture + uvPlane:_nv12TextureCache.uvTexture]; + [_nv12TextureCache releaseTextures]; } - shader = _i420Shader; - } - if (shader && [shader drawFrame:frame]) { - _lastDrawnFrame = frame; } else { - RTCLog(@"Failed to draw frame."); + if (!_i420TextureCache) { + _i420TextureCache = [[RTCI420TextureCache alloc] initWithContext:_glContext]; + } + [_i420TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithRotation:frame.rotation + yPlane:_i420TextureCache.yTexture + uPlane:_i420TextureCache.uTexture + vPlane:_i420TextureCache.vTexture]; } } @@ -275,8 +292,8 @@ _timer.isPaused = YES; [_glkView deleteDrawable]; [self ensureGLContext]; - _i420Shader = nil; - _nv12Shader = nil; + _nv12TextureCache = nil; + _i420TextureCache = nil; } - (void)didBecomeActive { diff --git a/webrtc/sdk/objc/Framework/Classes/UI/RTCNSGLVideoView.m b/webrtc/sdk/objc/Framework/Classes/UI/RTCNSGLVideoView.m index 530d9a7e6a..66145fb506 100644 --- a/webrtc/sdk/objc/Framework/Classes/UI/RTCNSGLVideoView.m +++ b/webrtc/sdk/objc/Framework/Classes/UI/RTCNSGLVideoView.m @@ -18,7 +18,8 @@ #import #import -#import "RTCShader+Private.h" +#import "RTCDefaultShader.h" +#import "RTCI420TextureCache.h" #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" @@ -26,7 +27,7 @@ // |videoFrame| is set when we receive a frame from a worker thread and is read // from the display link callback so atomicity is required. @property(atomic, strong) RTCVideoFrame *videoFrame; -@property(atomic, strong) id i420Shader; +@property(atomic, strong) RTCI420TextureCache *i420TextureCache; - (void)drawFrame; @end @@ -45,11 +46,25 @@ static CVReturn OnDisplayLinkFired(CVDisplayLinkRef displayLink, @implementation RTCNSGLVideoView { CVDisplayLinkRef _displayLink; RTCVideoFrame *_lastDrawnFrame; + id _shader; } @synthesize delegate = _delegate; @synthesize videoFrame = _videoFrame; -@synthesize i420Shader = _i420Shader; +@synthesize i420TextureCache = _i420TextureCache; + +- (instancetype)initWithFrame:(NSRect)frame pixelFormat:(NSOpenGLPixelFormat *)format { + return [self initWithFrame:frame pixelFormat:format shader:[[RTCDefaultShader alloc] init]]; +} + +- (instancetype)initWithFrame:(NSRect)frame + pixelFormat:(NSOpenGLPixelFormat *)format + shader:(id)shader { + if (self = [super initWithFrame:frame pixelFormat:format]) { + _shader = shader; + } + return self; +} - (void)dealloc { [self teardownDisplayLink]; @@ -85,7 +100,7 @@ static CVReturn OnDisplayLinkFired(CVDisplayLinkRef displayLink, - (void)clearGLContext { [self ensureGLContext]; - self.i420Shader = nil; + self.i420TextureCache = nil; [super clearGLContext]; } @@ -118,15 +133,20 @@ static CVReturn OnDisplayLinkFired(CVDisplayLinkRef displayLink, glClear(GL_COLOR_BUFFER_BIT); // Rendering native CVPixelBuffer is not supported on OS X. + // TODO(magjed): Add support for NV12 texture cache on OS X. frame = [frame newI420VideoFrame]; - if (!self.i420Shader) { - self.i420Shader = [[RTCI420Shader alloc] initWithContext:context]; + if (!self.i420TextureCache) { + self.i420TextureCache = [[RTCI420TextureCache alloc] initWithContext:context]; } - if (self.i420Shader && [self.i420Shader drawFrame:frame]) { + RTCI420TextureCache *i420TextureCache = self.i420TextureCache; + if (i420TextureCache) { + [i420TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithRotation:frame.rotation + yPlane:i420TextureCache.yTexture + uPlane:i420TextureCache.uTexture + vPlane:i420TextureCache.vTexture]; [context flushBuffer]; _lastDrawnFrame = frame; - } else { - RTCLog(@"Failed to draw frame."); } CGLUnlockContext([context CGLContextObj]); } diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.h b/webrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.h new file mode 100644 index 0000000000..ed5125e2e8 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.h @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoViewShading.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Default RTCVideoViewShading that will be used in RTCNSGLVideoView and + * RTCEAGLVideoView if no external shader is specified. This shader will render + * the video in a rectangle without any color or geometric transformations. + */ +@interface RTCDefaultShader : NSObject + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.mm new file mode 100644 index 0000000000..95a1eda5cd --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.mm @@ -0,0 +1,203 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDefaultShader.h" + +#if TARGET_OS_IPHONE +#import +#else +#import +#endif + +#import "RTCOpenGLDefines.h" +#import "RTCShader.h" +#import "WebRTC/RTCLogging.h" + +#include "webrtc/base/optional.h" + +static const int kYTextureUnit = 0; +static const int kUTextureUnit = 1; +static const int kVTextureUnit = 2; +static const int kUvTextureUnit = 1; + +// Fragment shader converts YUV values from input textures into a final RGB +// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. +static const char kI420FragmentShaderSource[] = + SHADER_VERSION + "precision highp float;" + FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" + "uniform lowp sampler2D s_textureY;\n" + "uniform lowp sampler2D s_textureU;\n" + "uniform lowp sampler2D s_textureV;\n" + FRAGMENT_SHADER_OUT + "void main() {\n" + " float y, u, v, r, g, b;\n" + " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" + " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" + " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" + " u = u - 0.5;\n" + " v = v - 0.5;\n" + " r = y + 1.403 * v;\n" + " g = y - 0.344 * u - 0.714 * v;\n" + " b = y + 1.770 * u;\n" + " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" + " }\n"; + +static const char kNV12FragmentShaderSource[] = + SHADER_VERSION + "precision mediump float;" + FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" + "uniform lowp sampler2D s_textureY;\n" + "uniform lowp sampler2D s_textureUV;\n" + FRAGMENT_SHADER_OUT + "void main() {\n" + " mediump float y;\n" + " mediump vec2 uv;\n" + " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" + " uv = " FRAGMENT_SHADER_TEXTURE "(s_textureUV, v_texcoord).ra -\n" + " vec2(0.5, 0.5);\n" + " " FRAGMENT_SHADER_COLOR " = vec4(y + 1.403 * uv.y,\n" + " y - 0.344 * uv.x - 0.714 * uv.y,\n" + " y + 1.770 * uv.x,\n" + " 1.0);\n" + " }\n"; + +@implementation RTCDefaultShader { + GLuint _vertexBuffer; + GLuint _vertexArray; + // Store current rotation and only upload new vertex data when rotation changes. + rtc::Optional _currentRotation; + + GLuint _i420Program; + GLuint _nv12Program; +} + +- (void)dealloc { + glDeleteProgram(_i420Program); + glDeleteProgram(_nv12Program); + glDeleteBuffers(1, &_vertexBuffer); + glDeleteVertexArrays(1, &_vertexArray); +} + +- (BOOL)createAndSetupI420Program { + NSAssert(!_i420Program, @"I420 program already created"); + _i420Program = RTCCreateProgramFromFragmentSource(kI420FragmentShaderSource); + if (!_i420Program) { + return NO; + } + GLint ySampler = glGetUniformLocation(_i420Program, "s_textureY"); + GLint uSampler = glGetUniformLocation(_i420Program, "s_textureU"); + GLint vSampler = glGetUniformLocation(_i420Program, "s_textureV"); + + if (ySampler < 0 || uSampler < 0 || vSampler < 0) { + RTCLog(@"Failed to get uniform variable locations in I420 shader"); + glDeleteProgram(_i420Program); + _i420Program = 0; + return NO; + } + + glUseProgram(_i420Program); + glUniform1i(ySampler, kYTextureUnit); + glUniform1i(uSampler, kUTextureUnit); + glUniform1i(vSampler, kVTextureUnit); + + return YES; +} + +- (BOOL)createAndSetupNV12Program { + NSAssert(!_nv12Program, @"NV12 program already created"); + _nv12Program = RTCCreateProgramFromFragmentSource(kNV12FragmentShaderSource); + if (!_nv12Program) { + return NO; + } + GLint ySampler = glGetUniformLocation(_nv12Program, "s_textureY"); + GLint uvSampler = glGetUniformLocation(_nv12Program, "s_textureUV"); + + if (ySampler < 0 || uvSampler < 0) { + RTCLog(@"Failed to get uniform variable locations in NV12 shader"); + glDeleteProgram(_nv12Program); + _nv12Program = 0; + return NO; + } + + glUseProgram(_nv12Program); + glUniform1i(ySampler, kYTextureUnit); + glUniform1i(uvSampler, kUvTextureUnit); + + return YES; +} + +- (BOOL)prepareVertexBufferWithRotation:(RTCVideoRotation)rotation { + if (!_vertexBuffer && !RTCCreateVertexBuffer(&_vertexBuffer, &_vertexArray)) { + RTCLog(@"Failed to setup vertex buffer"); + return NO; + } +#if !TARGET_OS_IPHONE + glBindVertexArray(_vertexArray); +#endif + glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + if (!_currentRotation || rotation != *_currentRotation) { + _currentRotation = rtc::Optional(rotation); + RTCSetVertexData(*_currentRotation); + } + return YES; +} + +- (void)applyShadingForFrameWithRotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uPlane:(GLuint)uPlane + vPlane:(GLuint)vPlane { + if (![self prepareVertexBufferWithRotation:rotation]) { + return; + } + + if (!_i420Program && ![self createAndSetupI420Program]) { + RTCLog(@"Failed to setup I420 program"); + return; + } + + glUseProgram(_i420Program); + + glActiveTexture(static_cast(GL_TEXTURE0 + kYTextureUnit)); + glBindTexture(GL_TEXTURE_2D, yPlane); + + glActiveTexture(static_cast(GL_TEXTURE0 + kUTextureUnit)); + glBindTexture(GL_TEXTURE_2D, uPlane); + + glActiveTexture(static_cast(GL_TEXTURE0 + kVTextureUnit)); + glBindTexture(GL_TEXTURE_2D, vPlane); + + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); +} + +- (void)applyShadingForFrameWithRotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uvPlane:(GLuint)uvPlane { + if (![self prepareVertexBufferWithRotation:rotation]) { + return; + } + + if (!_nv12Program && ![self createAndSetupNV12Program]) { + RTCLog(@"Failed to setup NV12 shader"); + return; + } + + glUseProgram(_nv12Program); + + glActiveTexture(static_cast(GL_TEXTURE0 + kYTextureUnit)); + glBindTexture(GL_TEXTURE_2D, yPlane); + + glActiveTexture(static_cast(GL_TEXTURE0 + kUvTextureUnit)); + glBindTexture(GL_TEXTURE_2D, uvPlane); + + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Shader.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Shader.mm deleted file mode 100644 index 83ecbe4d67..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Shader.mm +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "RTCShader.h" - -#import "RTCI420TextureCache.h" -#import "RTCShader+Private.h" -#import "WebRTC/RTCLogging.h" -#import "WebRTC/RTCVideoFrame.h" - -#include "webrtc/base/optional.h" - -// Fragment shader converts YUV values from input textures into a final RGB -// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. -static const char kI420FragmentShaderSource[] = - SHADER_VERSION - "precision highp float;" - FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" - "uniform lowp sampler2D s_textureY;\n" - "uniform lowp sampler2D s_textureU;\n" - "uniform lowp sampler2D s_textureV;\n" - FRAGMENT_SHADER_OUT - "void main() {\n" - " float y, u, v, r, g, b;\n" - " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" - " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" - " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" - " u = u - 0.5;\n" - " v = v - 0.5;\n" - " r = y + 1.403 * v;\n" - " g = y - 0.344 * u - 0.714 * v;\n" - " b = y + 1.770 * u;\n" - " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" - " }\n"; - -@implementation RTCI420Shader { - RTCI420TextureCache* textureCache; - // Handles for OpenGL constructs. - GLuint _i420Program; - GLuint _vertexArray; - GLuint _vertexBuffer; - GLint _ySampler; - GLint _uSampler; - GLint _vSampler; - // Store current rotation and only upload new vertex data when rotation - // changes. - rtc::Optional _currentRotation; -} - -- (instancetype)initWithContext:(GlContextType *)context { - if (self = [super init]) { - textureCache = [[RTCI420TextureCache alloc] initWithContext:context]; - glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - if (![self setupI420Program] || - !RTCSetupVerticesForProgram(_i420Program, &_vertexBuffer, &_vertexArray)) { - RTCLog(@"Failed to initialize RTCI420Shader."); - self = nil; - } - } - return self; -} - -- (void)dealloc { - glDeleteProgram(_i420Program); - glDeleteBuffers(1, &_vertexBuffer); - glDeleteVertexArrays(1, &_vertexArray); -} - -- (BOOL)setupI420Program { - _i420Program = RTCCreateProgramFromFragmentSource(kI420FragmentShaderSource); - if (!_i420Program) { - return NO; - } - _ySampler = glGetUniformLocation(_i420Program, "s_textureY"); - _uSampler = glGetUniformLocation(_i420Program, "s_textureU"); - _vSampler = glGetUniformLocation(_i420Program, "s_textureV"); - - return (_ySampler >= 0 && _uSampler >= 0 && _vSampler >= 0); -} - -- (BOOL)drawFrame:(RTCVideoFrame*)frame { - glUseProgram(_i420Program); - - [textureCache uploadFrameToTextures:frame]; - -#if !TARGET_OS_IPHONE - glBindVertexArray(_vertexArray); -#endif - - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textureCache.yTexture); - glUniform1i(_ySampler, 0); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, textureCache.uTexture); - glUniform1i(_uSampler, 1); - - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, textureCache.vTexture); - glUniform1i(_vSampler, 2); - - glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); - if (!_currentRotation || frame.rotation != *_currentRotation) { - _currentRotation = rtc::Optional(frame.rotation); - RTCSetVertexData(*_currentRotation); - } - glDrawArrays(GL_TRIANGLE_FAN, 0, 4); - - return YES; -} - -@end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.h b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.h index 5ee657c61f..19f1bc9993 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.h +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.h @@ -17,7 +17,8 @@ @property(nonatomic, readonly) GLuint uTexture; @property(nonatomic, readonly) GLuint vTexture; -- (instancetype)initWithContext:(GlContextType *)context; +- (instancetype)init NS_UNAVAILABLE; +- (instancetype)initWithContext:(GlContextType *)context NS_DESIGNATED_INITIALIZER; - (void)uploadFrameToTextures:(RTCVideoFrame *)frame; diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm index d263c75e93..eeae867bc0 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm @@ -10,7 +10,11 @@ #import "RTCI420TextureCache.h" -#import "RTCShader+Private.h" +#if TARGET_OS_IPHONE +#import +#else +#import +#endif #include diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.h b/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.h index 0aa7a57181..9cba823271 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.h +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.h @@ -19,7 +19,8 @@ NS_ASSUME_NONNULL_BEGIN @property(nonatomic, readonly) GLuint yTexture; @property(nonatomic, readonly) GLuint uvTexture; -- (nullable instancetype)initWithContext:(EAGLContext *)context; +- (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithContext:(EAGLContext *)context NS_DESIGNATED_INITIALIZER; - (BOOL)uploadFrameToTextures:(RTCVideoFrame *)frame; diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCNativeNV12Shader.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCNativeNV12Shader.mm deleted file mode 100644 index c0769c05cd..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCNativeNV12Shader.mm +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "RTCShader.h" - -#import "RTCNV12TextureCache.h" -#import "RTCShader+Private.h" -#import "WebRTC/RTCLogging.h" -#import "WebRTC/RTCVideoFrame.h" - -#include "webrtc/base/checks.h" -#include "webrtc/base/optional.h" - -static const char kNV12FragmentShaderSource[] = - SHADER_VERSION - "precision mediump float;" - FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" - "uniform lowp sampler2D s_textureY;\n" - "uniform lowp sampler2D s_textureUV;\n" - FRAGMENT_SHADER_OUT - "void main() {\n" - " mediump float y;\n" - " mediump vec2 uv;\n" - " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" - " uv = " FRAGMENT_SHADER_TEXTURE "(s_textureUV, v_texcoord).ra -\n" - " vec2(0.5, 0.5);\n" - " " FRAGMENT_SHADER_COLOR " = vec4(y + 1.403 * uv.y,\n" - " y - 0.344 * uv.x - 0.714 * uv.y,\n" - " y + 1.770 * uv.x,\n" - " 1.0);\n" - " }\n"; - -@implementation RTCNativeNV12Shader { - GLuint _vertexBuffer; - GLuint _nv12Program; - GLint _ySampler; - GLint _uvSampler; - RTCNV12TextureCache *_textureCache; - // Store current rotation and only upload new vertex data when rotation - // changes. - rtc::Optional _currentRotation; -} - -- (instancetype)initWithContext:(GlContextType *)context { - if (self = [super init]) { - _textureCache = [[RTCNV12TextureCache alloc] initWithContext:context]; - if (!_textureCache || ![self setupNV12Program] || - !RTCSetupVerticesForProgram(_nv12Program, &_vertexBuffer, nullptr)) { - RTCLog(@"Failed to initialize RTCNativeNV12Shader."); - self = nil; - } - } - return self; -} - -- (void)dealloc { - glDeleteProgram(_nv12Program); - glDeleteBuffers(1, &_vertexBuffer); -} - -- (BOOL)setupNV12Program { - _nv12Program = RTCCreateProgramFromFragmentSource(kNV12FragmentShaderSource); - if (!_nv12Program) { - return NO; - } - _ySampler = glGetUniformLocation(_nv12Program, "s_textureY"); - _uvSampler = glGetUniformLocation(_nv12Program, "s_textureUV"); - - return (_ySampler >= 0 && _uvSampler >= 0); -} - -- (BOOL)drawFrame:(RTCVideoFrame *)frame { - glUseProgram(_nv12Program); - if (![_textureCache uploadFrameToTextures:frame]) { - return NO; - } - - // Y-plane. - glActiveTexture(GL_TEXTURE0); - glUniform1i(_ySampler, 0); - glBindTexture(GL_TEXTURE_2D, _textureCache.yTexture); - - // UV-plane. - glActiveTexture(GL_TEXTURE1); - glUniform1i(_uvSampler, 1); - glBindTexture(GL_TEXTURE_2D, _textureCache.uvTexture); - - glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); - if (!_currentRotation || frame.rotation != *_currentRotation) { - _currentRotation = rtc::Optional(frame.rotation); - RTCSetVertexData(*_currentRotation); - } - glDrawArrays(GL_TRIANGLE_FAN, 0, 4); - - [_textureCache releaseTextures]; - - return YES; -} - -@end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCShader+Private.h b/webrtc/sdk/objc/Framework/Classes/Video/RTCShader+Private.h deleted file mode 100644 index 62da2b89fd..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCShader+Private.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "RTCShader.h" - -#import "WebRTC/RTCMacros.h" -#import "WebRTC/RTCVideoFrame.h" - -#if TARGET_OS_IPHONE -#import -#else -#import -#endif - -RTC_EXTERN const char kRTCVertexShaderSource[]; - -RTC_EXTERN GLuint RTCCreateShader(GLenum type, const GLchar *source); -RTC_EXTERN GLuint RTCCreateProgram(GLuint vertexShader, GLuint fragmentShader); -RTC_EXTERN GLuint RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]); -RTC_EXTERN BOOL RTCSetupVerticesForProgram( - GLuint program, GLuint* vertexBuffer, GLuint* vertexArray); -RTC_EXTERN void RTCSetVertexData(RTCVideoRotation rotation); diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCShader.h b/webrtc/sdk/objc/Framework/Classes/Video/RTCShader.h index c35a528b44..4d92d3dd10 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCShader.h +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCShader.h @@ -8,38 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ -#import +#import "WebRTC/RTCVideoFrame.h" -#import "RTCOpenGLDefines.h" +RTC_EXTERN const char kRTCVertexShaderSource[]; -@class RTCVideoFrame; - -@protocol RTCShader - -- (BOOL)drawFrame:(RTCVideoFrame *)frame; - -@end - -// Shader for non-native I420 frames. -@interface RTCI420Shader : NSObject - -- (instancetype)init NS_UNAVAILABLE; -- (instancetype)initWithContext:(GlContextType *)context NS_DESIGNATED_INITIALIZER; -- (BOOL)drawFrame:(RTCVideoFrame *)frame; - -@end - -// Native CVPixelBufferRef rendering is only supported on iPhone because it -// depends on CVOpenGLESTextureCacheCreate. -#if TARGET_OS_IPHONE - -// Shader for native NV12 frames. -@interface RTCNativeNV12Shader : NSObject - -- (instancetype)init NS_UNAVAILABLE; -- (instancetype)initWithContext:(GlContextType *)context NS_DESIGNATED_INITIALIZER; -- (BOOL)drawFrame:(RTCVideoFrame *)frame; - -@end - -#endif // TARGET_OS_IPHONE +RTC_EXTERN GLuint RTCCreateShader(GLenum type, const GLchar *source); +RTC_EXTERN GLuint RTCCreateProgram(GLuint vertexShader, GLuint fragmentShader); +RTC_EXTERN GLuint RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]); +RTC_EXTERN BOOL RTCCreateVertexBuffer(GLuint* vertexBuffer, + GLuint* vertexArray); +RTC_EXTERN void RTCSetVertexData(RTCVideoRotation rotation); diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCShader.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCShader.mm index 155a0165a1..3c3e6abb27 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCShader.mm +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCShader.mm @@ -10,11 +10,17 @@ #import "RTCShader.h" +#if TARGET_OS_IPHONE +#import +#else +#import +#endif + #include #include #include -#import "RTCShader+Private.h" +#import "RTCOpenGLDefines.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" @@ -95,17 +101,30 @@ GLuint RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]) { if (fragmentShader) { glDeleteShader(fragmentShader); } - return program; -} -// Set vertex shader variables 'position' and 'texcoord' in |program| to use -// |vertexBuffer| and |vertexArray| to store the vertex data. -BOOL RTCSetupVerticesForProgram(GLuint program, GLuint* vertexBuffer, GLuint* vertexArray) { + // Set vertex shader variables 'position' and 'texcoord' in program. GLint position = glGetAttribLocation(program, "position"); GLint texcoord = glGetAttribLocation(program, "texcoord"); if (position < 0 || texcoord < 0) { - return NO; + glDeleteProgram(program); + return 0; } + + // Read position attribute with size of 2 and stride of 4 beginning at the start of the array. The + // last argument indicates offset of data within the vertex buffer. + glVertexAttribPointer(position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)0); + glEnableVertexAttribArray(position); + + // Read texcoord attribute with size of 2 and stride of 4 beginning at the first texcoord in the + // array. The last argument indicates offset of data within the vertex buffer. + glVertexAttribPointer( + texcoord, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)(2 * sizeof(GLfloat))); + glEnableVertexAttribArray(texcoord); + + return program; +} + +BOOL RTCCreateVertexBuffer(GLuint *vertexBuffer, GLuint *vertexArray) { #if !TARGET_OS_IPHONE glGenVertexArrays(1, vertexArray); if (*vertexArray == 0) { @@ -115,25 +134,11 @@ BOOL RTCSetupVerticesForProgram(GLuint program, GLuint* vertexBuffer, GLuint* ve #endif glGenBuffers(1, vertexBuffer); if (*vertexBuffer == 0) { + glDeleteVertexArrays(1, vertexArray); return NO; } glBindBuffer(GL_ARRAY_BUFFER, *vertexBuffer); glBufferData(GL_ARRAY_BUFFER, 4 * 4 * sizeof(GLfloat), NULL, GL_DYNAMIC_DRAW); - - // Read position attribute with size of 2 and stride of 4 beginning at the - // start of the array. The last argument indicates offset of data within the - // vertex buffer. - glVertexAttribPointer(position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), - (void *)0); - glEnableVertexAttribArray(position); - - // Read texcoord attribute from |gVertices| with size of 2 and stride of 4 - // beginning at the first texcoord in the array. The last argument indicates - // offset of data within |gVertices| as supplied to the vertex buffer. - glVertexAttribPointer(texcoord, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), - (void *)(2 * sizeof(GLfloat))); - glEnableVertexAttribArray(texcoord); - return YES; } diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCEAGLVideoView.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCEAGLVideoView.h index 5f8d73bc8d..60c624e0d2 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCEAGLVideoView.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCEAGLVideoView.h @@ -13,6 +13,7 @@ #import #import +#import NS_ASSUME_NONNULL_BEGIN @@ -26,13 +27,19 @@ RTC_EXPORT /** * RTCEAGLVideoView is an RTCVideoRenderer which renders video frames in its - * bounds using OpenGLES 2.0. + * bounds using OpenGLES 2.0 or OpenGLES 3.0. */ RTC_EXPORT @interface RTCEAGLVideoView : UIView @property(nonatomic, weak) id delegate; +- (instancetype)initWithFrame:(CGRect)frame + shader:(id)shader NS_DESIGNATED_INITIALIZER; + +- (instancetype)initWithCoder:(NSCoder *)aDecoder + shader:(id)shader NS_DESIGNATED_INITIALIZER; + @end NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCNSGLVideoView.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCNSGLVideoView.h index 34588dbe4c..9e6a9ce948 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCNSGLVideoView.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCNSGLVideoView.h @@ -15,6 +15,7 @@ #import #import +#import NS_ASSUME_NONNULL_BEGIN @@ -29,6 +30,10 @@ NS_ASSUME_NONNULL_BEGIN @property(nonatomic, weak) id delegate; +- (instancetype)initWithFrame:(NSRect)frameRect + pixelFormat:(NSOpenGLPixelFormat *)format + shader:(id)shader NS_DESIGNATED_INITIALIZER; + @end NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoViewShading.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoViewShading.h new file mode 100644 index 0000000000..48d4e0670a --- /dev/null +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoViewShading.h @@ -0,0 +1,37 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * RTCVideoViewShading provides a way for apps to customize the OpenGL(ES) shaders used in + * rendering for the RTCEAGLVideoView/RTCNSGLVideoView. + */ +RTC_EXPORT +@protocol RTCVideoViewShading + +/** Callback for I420 frames. Each plane is given as a texture. */ +- (void)applyShadingForFrameWithRotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uPlane:(GLuint)uPlane + vPlane:(GLuint)vPlane; + +/** Callback for NV12 frames. Each plane is given as a texture. */ +- (void)applyShadingForFrameWithRotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uvPlane:(GLuint)uvPlane; + +@end + +NS_ASSUME_NONNULL_END