From fb372f0074a0cb21fa36976316c0b5fe3b4ae7bc Mon Sep 17 00:00:00 2001 From: magjed Date: Wed, 10 Aug 2016 07:58:29 -0700 Subject: [PATCH] iOS render: Handle frame rotation in OpenGL This CL handles frame rotation by updating the OpenGL vertex data in RTCOpenGLVideoRenderer, instead of calling the expensive libyuv::I420Rotate that will rotate the actual memory. Also, we can handle rotated native frames instead of falling back to NativeToI420Buffer. Review-Url: https://codereview.webrtc.org/2176623002 Cr-Commit-Position: refs/heads/master@{#13715} --- webrtc/sdk/DEPS | 1 + .../objc/Framework/Classes/RTCEAGLVideoView.m | 2 +- .../objc/Framework/Classes/RTCI420Shader.mm | 11 +++ .../Framework/Classes/RTCNativeNV12Shader.mm | 10 +++ .../Framework/Classes/RTCShader+Private.h | 3 + .../sdk/objc/Framework/Classes/RTCShader.mm | 76 +++++++++++++------ .../Framework/Classes/RTCVideoFrame+Private.h | 6 +- .../objc/Framework/Classes/RTCVideoFrame.mm | 42 +++++----- .../Classes/RTCVideoRendererAdapter.mm | 29 ++----- .../Framework/Headers/WebRTC/RTCVideoFrame.h | 3 +- 10 files changed, 119 insertions(+), 64 deletions(-) diff --git a/webrtc/sdk/DEPS b/webrtc/sdk/DEPS index d6e53d504d..16ef0f6a06 100644 --- a/webrtc/sdk/DEPS +++ b/webrtc/sdk/DEPS @@ -2,6 +2,7 @@ include_rules = [ "+WebRTC", "+webrtc/api", "+webrtc/common_video/include", + "+webrtc/common_video/rotation.h", "+webrtc/media", "+webrtc/system_wrappers", ] diff --git a/webrtc/sdk/objc/Framework/Classes/RTCEAGLVideoView.m b/webrtc/sdk/objc/Framework/Classes/RTCEAGLVideoView.m index 2d5355d249..01c839a3bb 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCEAGLVideoView.m +++ b/webrtc/sdk/objc/Framework/Classes/RTCEAGLVideoView.m @@ -13,7 +13,7 @@ #import #import "RTCOpenGLVideoRenderer.h" -#import "WebRTC//RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrame.h" // RTCDisplayLinkTimer wraps a CADisplayLink and is set to fire every two screen // refreshes, which should be 30fps. We wrap the display link in order to avoid diff --git a/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm b/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm index 0618c1a530..e0c9642269 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm @@ -15,6 +15,9 @@ #import "RTCShader+Private.h" #import "WebRTC/RTCVideoFrame.h" +#include "webrtc/base/optional.h" +#include "webrtc/common_video/rotation.h" + // |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets // of 3 textures are used here, one for each of the Y, U and V planes. Having // two sets alleviates CPU blockage in the event that the GPU is asked to render @@ -57,6 +60,9 @@ static const char kI420FragmentShaderSource[] = GLint _ySampler; GLint _uSampler; GLint _vSampler; + // Store current rotation and only upload new vertex data when rotation + // changes. + rtc::Optional _currentRotation; // Used to create a non-padded plane for GPU upload when we receive padded // frames. std::vector _planeBuffer; @@ -119,6 +125,11 @@ static const char kI420FragmentShaderSource[] = glBindVertexArray(_vertexArray); #endif glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + if (!_currentRotation || frame.rotation != *_currentRotation) { + _currentRotation = rtc::Optional( + static_cast(frame.rotation)); + RTCSetVertexData(*_currentRotation); + } glDrawArrays(GL_TRIANGLE_FAN, 0, 4); return YES; diff --git a/webrtc/sdk/objc/Framework/Classes/RTCNativeNV12Shader.mm b/webrtc/sdk/objc/Framework/Classes/RTCNativeNV12Shader.mm index cc979434e9..75a575b075 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCNativeNV12Shader.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCNativeNV12Shader.mm @@ -20,6 +20,8 @@ #import "WebRTC/RTCVideoFrame.h" #include "webrtc/base/checks.h" +#include "webrtc/base/optional.h" +#include "webrtc/common_video/rotation.h" static const char kNV12FragmentShaderSource[] = SHADER_VERSION @@ -46,6 +48,9 @@ static const char kNV12FragmentShaderSource[] = GLint _ySampler; GLint _uvSampler; CVOpenGLESTextureCacheRef _textureCache; + // Store current rotation and only upload new vertex data when rotation + // changes. + rtc::Optional _currentRotation; } - (instancetype)initWithContext:(GlContextType *)context { @@ -149,6 +154,11 @@ static const char kNV12FragmentShaderSource[] = glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + if (!_currentRotation || frame.rotation != *_currentRotation) { + _currentRotation = rtc::Optional( + static_cast(frame.rotation)); + RTCSetVertexData(*_currentRotation); + } glDrawArrays(GL_TRIANGLE_FAN, 0, 4); CFRelease(chromaTexture); diff --git a/webrtc/sdk/objc/Framework/Classes/RTCShader+Private.h b/webrtc/sdk/objc/Framework/Classes/RTCShader+Private.h index 0f277d4524..547d26235d 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCShader+Private.h +++ b/webrtc/sdk/objc/Framework/Classes/RTCShader+Private.h @@ -18,6 +18,8 @@ #import #endif +#include "webrtc/common_video/rotation.h" + RTC_EXTERN const char kRTCVertexShaderSource[]; RTC_EXTERN GLuint RTCCreateShader(GLenum type, const GLchar *source); @@ -25,3 +27,4 @@ RTC_EXTERN GLuint RTCCreateProgram(GLuint vertexShader, GLuint fragmentShader); RTC_EXTERN GLuint RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]); RTC_EXTERN BOOL RTCSetupVerticesForProgram( GLuint program, GLuint* vertexBuffer, GLuint* vertexArray); +RTC_EXTERN void RTCSetVertexData(webrtc::VideoRotation rotation); diff --git a/webrtc/sdk/objc/Framework/Classes/RTCShader.mm b/webrtc/sdk/objc/Framework/Classes/RTCShader.mm index 88082064df..26dc64f92b 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCShader.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCShader.mm @@ -10,6 +10,8 @@ #import "RTCShader.h" +#include +#include #include #import "RTCShader+Private.h" @@ -28,21 +30,6 @@ const char kRTCVertexShaderSource[] = " v_texcoord = texcoord;\n" "}\n"; -// When modelview and projection matrices are identity (default) the world is -// contained in the square around origin with unit size 2. Drawing to these -// coordinates is equivalent to drawing to the entire screen. The texture is -// stretched over that square using texture coordinates (u, v) that range -// from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically -// here because the incoming frame has origin in upper left hand corner but -// OpenGL expects origin in bottom left corner. -static const GLfloat gVertices[] = { - // X, Y, U, V. - -1, -1, 0, 1, // Bottom left. - 1, -1, 1, 1, // Bottom right. - 1, 1, 1, 0, // Top right. - -1, 1, 0, 0, // Top left. -}; - // Compiles a shader of the given |type| with GLSL source |source| and returns // the shader handle or 0 on error. GLuint RTCCreateShader(GLenum type, const GLchar *source) { @@ -111,9 +98,8 @@ GLuint RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]) { return program; } -// Set vertex shader variables 'position' and 'texcoord' in |program| to the -// |gVertices| array above. It will use |vertexBuffer| and |vertexArray| to -// store the vertex data. +// Set vertex shader variables 'position' and 'texcoord' in |program| to use +// |vertexBuffer| and |vertexArray| to store the vertex data. BOOL RTCSetupVerticesForProgram(GLuint program, GLuint* vertexBuffer, GLuint* vertexArray) { GLint position = glGetAttribLocation(program, "position"); GLint texcoord = glGetAttribLocation(program, "texcoord"); @@ -132,11 +118,11 @@ BOOL RTCSetupVerticesForProgram(GLuint program, GLuint* vertexBuffer, GLuint* ve return NO; } glBindBuffer(GL_ARRAY_BUFFER, *vertexBuffer); - glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW); + glBufferData(GL_ARRAY_BUFFER, 4 * 4 * sizeof(GLfloat), NULL, GL_DYNAMIC_DRAW); - // Read position attribute from |gVertices| with size of 2 and stride of 4 - // beginning at the start of the array. The last argument indicates offset - // of data within |gVertices| as supplied to the vertex buffer. + // Read position attribute with size of 2 and stride of 4 beginning at the + // start of the array. The last argument indicates offset of data within the + // vertex buffer. glVertexAttribPointer(position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)0); glEnableVertexAttribArray(position); @@ -150,3 +136,49 @@ BOOL RTCSetupVerticesForProgram(GLuint program, GLuint* vertexBuffer, GLuint* ve return YES; } + +// Set vertex data to the currently bound vertex buffer. +void RTCSetVertexData(webrtc::VideoRotation rotation) { + // When modelview and projection matrices are identity (default) the world is + // contained in the square around origin with unit size 2. Drawing to these + // coordinates is equivalent to drawing to the entire screen. The texture is + // stretched over that square using texture coordinates (u, v) that range + // from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically + // here because the incoming frame has origin in upper left hand corner but + // OpenGL expects origin in bottom left corner. + std::array, 4> UVCoords = {{ + {{0, 1}}, // Lower left. + {{1, 1}}, // Lower right. + {{1, 0}}, // Upper right. + {{0, 0}}, // Upper left. + }}; + + // Rotate the UV coordinates. + int rotation_offset; + switch (rotation) { + case webrtc::kVideoRotation_0: + rotation_offset = 0; + break; + case webrtc::kVideoRotation_90: + rotation_offset = 1; + break; + case webrtc::kVideoRotation_180: + rotation_offset = 2; + break; + case webrtc::kVideoRotation_270: + rotation_offset = 3; + break; + } + std::rotate(UVCoords.begin(), UVCoords.begin() + rotation_offset, + UVCoords.end()); + + const GLfloat gVertices[] = { + // X, Y, U, V. + -1, -1, UVCoords[0][0], UVCoords[0][1], + 1, -1, UVCoords[1][0], UVCoords[1][1], + 1, 1, UVCoords[2][0], UVCoords[2][1], + -1, 1, UVCoords[3][0], UVCoords[3][1], + }; + + glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(gVertices), gVertices); +} diff --git a/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h b/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h index a4807127ca..83ae21ba68 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h +++ b/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h @@ -10,6 +10,7 @@ #import "WebRTC/RTCVideoFrame.h" +#include "webrtc/common_video/rotation.h" #include "webrtc/media/base/videoframe.h" NS_ASSUME_NONNULL_BEGIN @@ -19,7 +20,10 @@ NS_ASSUME_NONNULL_BEGIN @property(nonatomic, readonly) rtc::scoped_refptr i420Buffer; -- (instancetype)initWithNativeFrame:(const cricket::VideoFrame *)nativeFrame +- (instancetype)initWithVideoBuffer: + (rtc::scoped_refptr)videoBuffer + rotation:(webrtc::VideoRotation)rotation + timeStampNs:(int64_t)timeStampNs NS_DESIGNATED_INITIALIZER; @end diff --git a/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm b/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm index 5b2d2586b1..4b2b75415f 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm @@ -12,17 +12,25 @@ #include +#include "webrtc/common_video/rotation.h" + @implementation RTCVideoFrame { - std::unique_ptr _videoFrame; + rtc::scoped_refptr _videoBuffer; + webrtc::VideoRotation _rotation; + int64_t _timeStampNs; rtc::scoped_refptr _i420Buffer; } - (size_t)width { - return _videoFrame->width(); + return _videoBuffer->width(); } - (size_t)height { - return _videoFrame->height(); + return _videoBuffer->height(); +} + +- (int)rotation { + return static_cast(_rotation); } // TODO(nisse): chromaWidth and chromaHeight are used only in @@ -78,34 +86,32 @@ return self.i420Buffer->StrideV(); } -- (int64_t)timeStamp { - return _videoFrame->GetTimeStamp(); +- (int64_t)timeStampNs { + return _timeStampNs; } - (CVPixelBufferRef)nativeHandle { - return static_cast( - _videoFrame->video_frame_buffer()->native_handle()); + return static_cast(_videoBuffer->native_handle()); } - (void)convertBufferIfNeeded { if (!_i420Buffer) { - if (_videoFrame->video_frame_buffer()->native_handle()) { - // Convert to I420. - _i420Buffer = _videoFrame->video_frame_buffer()->NativeToI420Buffer(); - } else { - // Should already be I420. - _i420Buffer = _videoFrame->video_frame_buffer(); - } + _i420Buffer = _videoBuffer->native_handle() + ? _videoBuffer->NativeToI420Buffer() + : _videoBuffer; } } #pragma mark - Private -- (instancetype)initWithNativeFrame:(const cricket::VideoFrame *)nativeFrame { +- (instancetype)initWithVideoBuffer: + (rtc::scoped_refptr)videoBuffer + rotation:(webrtc::VideoRotation)rotation + timeStampNs:(int64_t)timeStampNs { if (self = [super init]) { - // Keep a shallow copy of the video frame. The underlying frame buffer is - // not copied. - _videoFrame.reset(nativeFrame->Copy()); + _videoBuffer = videoBuffer; + _rotation = rotation; + _timeStampNs = timeStampNs; } return self; } diff --git a/webrtc/sdk/objc/Framework/Classes/RTCVideoRendererAdapter.mm b/webrtc/sdk/objc/Framework/Classes/RTCVideoRendererAdapter.mm index a130d0e7bf..73682d3eaf 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCVideoRendererAdapter.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCVideoRendererAdapter.mm @@ -27,27 +27,14 @@ class VideoRendererAdapter } void OnFrame(const cricket::VideoFrame& nativeVideoFrame) override { - RTCVideoFrame *videoFrame = nil; - // Rotation of native handles is unsupported right now. Convert to CPU - // I420 buffer for rotation before calling the rotation method otherwise - // it will hit a DCHECK. - if (nativeVideoFrame.rotation() != webrtc::kVideoRotation_0 && - nativeVideoFrame.video_frame_buffer()->native_handle()) { - rtc::scoped_refptr i420Buffer = - nativeVideoFrame.video_frame_buffer()->NativeToI420Buffer(); - std::unique_ptr cpuFrame( - new cricket::WebRtcVideoFrame(i420Buffer, nativeVideoFrame.rotation(), - nativeVideoFrame.timestamp_us(), - nativeVideoFrame.transport_frame_id())); - const cricket::VideoFrame *rotatedFrame = - cpuFrame->GetCopyWithRotationApplied(); - videoFrame = [[RTCVideoFrame alloc] initWithNativeFrame:rotatedFrame]; - } else { - const cricket::VideoFrame *rotatedFrame = - nativeVideoFrame.GetCopyWithRotationApplied(); - videoFrame = [[RTCVideoFrame alloc] initWithNativeFrame:rotatedFrame]; - } - CGSize current_size = CGSizeMake(videoFrame.width, videoFrame.height); + RTCVideoFrame* videoFrame = [[RTCVideoFrame alloc] + initWithVideoBuffer:nativeVideoFrame.video_frame_buffer() + rotation:nativeVideoFrame.rotation() + timeStampNs:nativeVideoFrame.GetTimeStamp()]; + CGSize current_size = (videoFrame.rotation % 180 == 0) + ? CGSizeMake(videoFrame.width, videoFrame.height) + : CGSizeMake(videoFrame.height, videoFrame.width); + if (!CGSizeEqualToSize(size_, current_size)) { size_ = current_size; [adapter_.videoRenderer setSize:size_]; diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h index efb666e4bf..ffbbbc6906 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h @@ -24,6 +24,7 @@ RTC_EXPORT /** Height without rotation applied. */ @property(nonatomic, readonly) size_t height; +@property(nonatomic, readonly) int rotation; @property(nonatomic, readonly) size_t chromaWidth; @property(nonatomic, readonly) size_t chromaHeight; // These can return NULL if the object is not backed by a buffer. @@ -35,7 +36,7 @@ RTC_EXPORT @property(nonatomic, readonly) int32_t vPitch; /** Timestamp in nanoseconds. */ -@property(nonatomic, readonly) int64_t timeStamp; +@property(nonatomic, readonly) int64_t timeStampNs; /** The native handle should be a pixel buffer on iOS. */ @property(nonatomic, readonly) CVPixelBufferRef nativeHandle;