diff --git a/webrtc/sdk/android/BUILD.gn b/webrtc/sdk/android/BUILD.gn index 339e47caaf..ab9bfb6699 100644 --- a/webrtc/sdk/android/BUILD.gn +++ b/webrtc/sdk/android/BUILD.gn @@ -424,6 +424,7 @@ android_library("libjingle_peerconnection_java") { "api/org/webrtc/VideoEncoderFactory.java", "api/org/webrtc/VideoFileRenderer.java", "api/org/webrtc/VideoFrame.java", + "api/org/webrtc/VideoFrameDrawer.java", "api/org/webrtc/VideoRenderer.java", "api/org/webrtc/VideoSink.java", "api/org/webrtc/VideoSource.java", diff --git a/webrtc/sdk/android/api/org/webrtc/EglRenderer.java b/webrtc/sdk/android/api/org/webrtc/EglRenderer.java index d3c91512c5..16e89f4d21 100644 --- a/webrtc/sdk/android/api/org/webrtc/EglRenderer.java +++ b/webrtc/sdk/android/api/org/webrtc/EglRenderer.java @@ -96,8 +96,9 @@ public class EglRenderer implements VideoRenderer.Callbacks, VideoSink { // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed // from the render thread. private EglBase eglBase; - private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader(); + private final VideoFrameDrawer frameDrawer = new VideoFrameDrawer(); private RendererCommon.GlDrawer drawer; + private final Matrix drawMatrix = new Matrix(); // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|. private final Object frameLock = new Object(); @@ -227,7 +228,7 @@ public class EglRenderer implements VideoRenderer.Callbacks, VideoSink { drawer.release(); drawer = null; } - yuvUploader.release(); + frameDrawer.release(); if (bitmapTextureFramebuffer != null) { bitmapTextureFramebuffer.release(); bitmapTextureFramebuffer = null; @@ -560,35 +561,6 @@ public class EglRenderer implements VideoRenderer.Callbacks, VideoSink { drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio; } - VideoFrame.Buffer buffer = frame.getBuffer(); - final boolean isYuvBuffer; - if (buffer instanceof VideoFrame.TextureBuffer) { - isYuvBuffer = false; - } else { - isYuvBuffer = true; - VideoFrame.Buffer oldBuffer = buffer; - buffer = buffer.toI420(); - oldBuffer.release(); - } - boolean shouldUploadYuvTextures = false; - if (isYuvBuffer) { - shouldUploadYuvTextures = shouldRenderFrame; - // Check if there are frame listeners that we want to render a bitmap for regardless of if the - // frame was rendered. This is the case when there are frameListeners with scale != 0f. - if (!shouldUploadYuvTextures) { - for (FrameListenerAndParams listenerAndParams : frameListeners) { - if (listenerAndParams.scale != 0f - && (shouldRenderFrame || !listenerAndParams.applyFpsReduction)) { - shouldUploadYuvTextures = true; - break; - } - } - } - } - final int[] yuvTextures = shouldUploadYuvTextures - ? yuvUploader.uploadFromBuffer((VideoFrame.I420Buffer) buffer) - : null; - final float scaleX; final float scaleY; @@ -600,14 +572,8 @@ public class EglRenderer implements VideoRenderer.Callbacks, VideoSink { scaleY = frameAspectRatio / drawnAspectRatio; } - final int drawnFrameWidth = (int) (scaleX * frame.getRotatedWidth()); - final int drawnFrameHeight = (int) (scaleY * frame.getRotatedHeight()); - - final Matrix drawMatrix = new Matrix(); + drawMatrix.reset(); drawMatrix.preTranslate(0.5f, 0.5f); - if (isYuvBuffer) - drawMatrix.preScale(1f, -1f); // I420-frames are upside down - drawMatrix.preRotate(frame.getRotation()); if (mirror) drawMatrix.preScale(-1f, 1f); drawMatrix.preScale(scaleX, scaleY); @@ -616,15 +582,8 @@ public class EglRenderer implements VideoRenderer.Callbacks, VideoSink { if (shouldRenderFrame) { GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - if (isYuvBuffer) { - drawer.drawYuv(yuvTextures, - RendererCommon.convertMatrixFromAndroidGraphicsMatrix(drawMatrix), drawnFrameWidth, - drawnFrameHeight, 0, 0, eglBase.surfaceWidth(), eglBase.surfaceHeight()); - } else { - VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) buffer; - RendererCommon.drawTexture(drawer, textureBuffer, drawMatrix, drawnFrameWidth, - drawnFrameHeight, 0, 0, eglBase.surfaceWidth(), eglBase.surfaceHeight()); - } + frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */, + eglBase.surfaceWidth(), eglBase.surfaceHeight()); final long swapBuffersStartTimeNs = System.nanoTime(); eglBase.swapBuffers(); @@ -637,20 +596,16 @@ public class EglRenderer implements VideoRenderer.Callbacks, VideoSink { } } - notifyCallbacks(frame, isYuvBuffer, yuvTextures, shouldRenderFrame); - buffer.release(); + notifyCallbacks(frame, shouldRenderFrame); + frame.release(); } - private void notifyCallbacks( - VideoFrame frame, boolean isYuvBuffer, int[] yuvTextures, boolean wasRendered) { + private void notifyCallbacks(VideoFrame frame, boolean wasRendered) { if (frameListeners.isEmpty()) return; - final Matrix drawMatrix = new Matrix(); + drawMatrix.reset(); drawMatrix.preTranslate(0.5f, 0.5f); - if (isYuvBuffer) - drawMatrix.preScale(1f, -1f); // I420-frames are upside down - drawMatrix.preRotate(frame.getRotation()); if (mirror) drawMatrix.preScale(-1f, 1f); drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap. @@ -683,15 +638,8 @@ public class EglRenderer implements VideoRenderer.Callbacks, VideoSink { GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - if (isYuvBuffer) { - listenerAndParams.drawer.drawYuv(yuvTextures, - RendererCommon.convertMatrixFromAndroidGraphicsMatrix(drawMatrix), - frame.getRotatedWidth(), frame.getRotatedHeight(), 0, 0, scaledWidth, scaledHeight); - } else { - VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) frame.getBuffer(); - RendererCommon.drawTexture(listenerAndParams.drawer, textureBuffer, drawMatrix, - frame.getRotatedWidth(), frame.getRotatedHeight(), 0, 0, scaledWidth, scaledHeight); - } + frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */, + 0 /* viewportY */, scaledWidth, scaledHeight); final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4); GLES20.glViewport(0, 0, scaledWidth, scaledHeight); diff --git a/webrtc/sdk/android/api/org/webrtc/MediaCodecVideoEncoder.java b/webrtc/sdk/android/api/org/webrtc/MediaCodecVideoEncoder.java index a45e70920c..227c5a40c5 100644 --- a/webrtc/sdk/android/api/org/webrtc/MediaCodecVideoEncoder.java +++ b/webrtc/sdk/android/api/org/webrtc/MediaCodecVideoEncoder.java @@ -616,7 +616,7 @@ public class MediaCodecVideoEncoder { // TODO(perkj): glClear() shouldn't be necessary since every pixel is covered anyway, // but it's a workaround for bug webrtc:5147. GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - RendererCommon.drawTexture(drawer, textureBuffer, new Matrix() /* renderMatrix */, width, + VideoFrameDrawer.drawTexture(drawer, textureBuffer, new Matrix() /* renderMatrix */, width, height, 0 /* viewportX */, 0 /* viewportY */, width, height); eglBase.swapBuffers(frame.getTimestampNs()); } else { diff --git a/webrtc/sdk/android/api/org/webrtc/RendererCommon.java b/webrtc/sdk/android/api/org/webrtc/RendererCommon.java index 18d96c22e5..0554f119c3 100644 --- a/webrtc/sdk/android/api/org/webrtc/RendererCommon.java +++ b/webrtc/sdk/android/api/org/webrtc/RendererCommon.java @@ -11,10 +11,8 @@ package org.webrtc; import android.graphics.Point; -import android.opengl.GLES20; import android.opengl.Matrix; import android.view.View; -import java.nio.ByteBuffer; /** * Static helper functions for renderer implementations. @@ -53,109 +51,6 @@ public class RendererCommon { void release(); } - /** - * Draws a VideoFrame.TextureBuffer. Calls either drawer.drawOes or drawer.drawRgb - * depending on the type of the buffer. You can supply an additional render matrix. This is - * used multiplied together with the transformation matrix of the frame. (M = renderMatrix * - * transformationMatrix) - */ - static void drawTexture(GlDrawer drawer, VideoFrame.TextureBuffer buffer, - android.graphics.Matrix renderMatrix, int frameWidth, int frameHeight, int viewportX, - int viewportY, int viewportWidth, int viewportHeight) { - android.graphics.Matrix finalMatrix = new android.graphics.Matrix(buffer.getTransformMatrix()); - finalMatrix.preConcat(renderMatrix); - float[] finalGlMatrix = convertMatrixFromAndroidGraphicsMatrix(finalMatrix); - switch (buffer.getType()) { - case OES: - drawer.drawOes(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX, - viewportY, viewportWidth, viewportHeight); - break; - case RGB: - drawer.drawRgb(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX, - viewportY, viewportWidth, viewportHeight); - break; - default: - throw new RuntimeException("Unknown texture type."); - } - } - - /** - * Helper class for uploading YUV bytebuffer frames to textures that handles stride > width. This - * class keeps an internal ByteBuffer to avoid unnecessary allocations for intermediate copies. - */ - public static class YuvUploader { - // Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width. - // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader - // that handles stride and compare performance with intermediate copy. - private ByteBuffer copyBuffer; - private int[] yuvTextures; - - /** - * Upload |planes| into OpenGL textures, taking stride into consideration. - * - * @return Array of three texture indices corresponding to Y-, U-, and V-plane respectively. - */ - public int[] uploadYuvData(int width, int height, int[] strides, ByteBuffer[] planes) { - final int[] planeWidths = new int[] {width, width / 2, width / 2}; - final int[] planeHeights = new int[] {height, height / 2, height / 2}; - // Make a first pass to see if we need a temporary copy buffer. - int copyCapacityNeeded = 0; - for (int i = 0; i < 3; ++i) { - if (strides[i] > planeWidths[i]) { - copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidths[i] * planeHeights[i]); - } - } - // Allocate copy buffer if necessary. - if (copyCapacityNeeded > 0 - && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) { - copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded); - } - // Make sure YUV textures are allocated. - if (yuvTextures == null) { - yuvTextures = new int[3]; - for (int i = 0; i < 3; i++) { - yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D); - } - } - // Upload each plane. - for (int i = 0; i < 3; ++i) { - GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); - // GLES only accepts packed data, i.e. stride == planeWidth. - final ByteBuffer packedByteBuffer; - if (strides[i] == planeWidths[i]) { - // Input is packed already. - packedByteBuffer = planes[i]; - } else { - VideoRenderer.nativeCopyPlane( - planes[i], planeWidths[i], planeHeights[i], strides[i], copyBuffer, planeWidths[i]); - packedByteBuffer = copyBuffer; - } - GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidths[i], - planeHeights[i], 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer); - } - return yuvTextures; - } - - public int[] uploadFromBuffer(VideoFrame.I420Buffer buffer) { - int[] strides = {buffer.getStrideY(), buffer.getStrideU(), buffer.getStrideV()}; - ByteBuffer[] planes = {buffer.getDataY(), buffer.getDataU(), buffer.getDataV()}; - return uploadYuvData(buffer.getWidth(), buffer.getHeight(), strides, planes); - } - - /** - * Releases cached resources. Uploader can still be used and the resources will be reallocated - * on first use. - */ - public void release() { - copyBuffer = null; - if (yuvTextures != null) { - GLES20.glDeleteTextures(3, yuvTextures, 0); - yuvTextures = null; - } - } - } - /** * Helper class for determining layout size based on layout requirements, scaling type, and video * aspect ratio. diff --git a/webrtc/sdk/android/api/org/webrtc/VideoFrameDrawer.java b/webrtc/sdk/android/api/org/webrtc/VideoFrameDrawer.java new file mode 100644 index 0000000000..491fd054c2 --- /dev/null +++ b/webrtc/sdk/android/api/org/webrtc/VideoFrameDrawer.java @@ -0,0 +1,227 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Matrix; +import android.graphics.Point; +import android.opengl.GLES20; +import java.nio.ByteBuffer; + +/** + * Helper class to draw VideoFrames. Calls either drawer.drawOes, drawer.drawRgb, or + * drawer.drawYuv depending on the type of the buffer. The frame will be rendered with rotation + * taken into account. You can supply an additional render matrix for custom transformations. + */ +public class VideoFrameDrawer { + /** + * Draws a VideoFrame.TextureBuffer. Calls either drawer.drawOes or drawer.drawRgb + * depending on the type of the buffer. You can supply an additional render matrix. This is + * used multiplied together with the transformation matrix of the frame. (M = renderMatrix * + * transformationMatrix) + */ + static void drawTexture(RendererCommon.GlDrawer drawer, VideoFrame.TextureBuffer buffer, + Matrix renderMatrix, int frameWidth, int frameHeight, int viewportX, int viewportY, + int viewportWidth, int viewportHeight) { + Matrix finalMatrix = new Matrix(buffer.getTransformMatrix()); + finalMatrix.preConcat(renderMatrix); + float[] finalGlMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(finalMatrix); + switch (buffer.getType()) { + case OES: + drawer.drawOes(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX, + viewportY, viewportWidth, viewportHeight); + break; + case RGB: + drawer.drawRgb(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX, + viewportY, viewportWidth, viewportHeight); + break; + default: + throw new RuntimeException("Unknown texture type."); + } + } + + /** + * Helper class for uploading YUV bytebuffer frames to textures that handles stride > width. This + * class keeps an internal ByteBuffer to avoid unnecessary allocations for intermediate copies. + */ + private static class YuvUploader { + // Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width. + // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader + // that handles stride and compare performance with intermediate copy. + private ByteBuffer copyBuffer; + private int[] yuvTextures; + + /** + * Upload |planes| into OpenGL textures, taking stride into consideration. + * + * @return Array of three texture indices corresponding to Y-, U-, and V-plane respectively. + */ + public int[] uploadYuvData(int width, int height, int[] strides, ByteBuffer[] planes) { + final int[] planeWidths = new int[] {width, width / 2, width / 2}; + final int[] planeHeights = new int[] {height, height / 2, height / 2}; + // Make a first pass to see if we need a temporary copy buffer. + int copyCapacityNeeded = 0; + for (int i = 0; i < 3; ++i) { + if (strides[i] > planeWidths[i]) { + copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidths[i] * planeHeights[i]); + } + } + // Allocate copy buffer if necessary. + if (copyCapacityNeeded > 0 + && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) { + copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded); + } + // Make sure YUV textures are allocated. + if (yuvTextures == null) { + yuvTextures = new int[3]; + for (int i = 0; i < 3; i++) { + yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D); + } + } + // Upload each plane. + for (int i = 0; i < 3; ++i) { + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); + // GLES only accepts packed data, i.e. stride == planeWidth. + final ByteBuffer packedByteBuffer; + if (strides[i] == planeWidths[i]) { + // Input is packed already. + packedByteBuffer = planes[i]; + } else { + VideoRenderer.nativeCopyPlane( + planes[i], planeWidths[i], planeHeights[i], strides[i], copyBuffer, planeWidths[i]); + packedByteBuffer = copyBuffer; + } + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidths[i], + planeHeights[i], 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer); + } + return yuvTextures; + } + + public int[] uploadFromBuffer(VideoFrame.I420Buffer buffer) { + int[] strides = {buffer.getStrideY(), buffer.getStrideU(), buffer.getStrideV()}; + ByteBuffer[] planes = {buffer.getDataY(), buffer.getDataU(), buffer.getDataV()}; + return uploadYuvData(buffer.getWidth(), buffer.getHeight(), strides, planes); + } + + public int[] getYuvTextures() { + return yuvTextures; + } + + /** + * Releases cached resources. Uploader can still be used and the resources will be reallocated + * on first use. + */ + public void release() { + copyBuffer = null; + if (yuvTextures != null) { + GLES20.glDeleteTextures(3, yuvTextures, 0); + yuvTextures = null; + } + } + } + + private static int distance(float x0, float y0, float x1, float y1) { + return (int) Math.round(Math.hypot(x1 - x0, y1 - y0)); + } + + // These points are used to calculate the size of the part of the frame we are rendering. + final static float[] srcPoints = + new float[] {0f /* x0 */, 0f /* y0 */, 1f /* x1 */, 0f /* y1 */, 0f /* x2 */, 1f /* y2 */}; + private final float[] dstPoints = new float[6]; + private final Point renderSize = new Point(); + private int renderWidth; + private int renderHeight; + + // Calculate the frame size after |renderMatrix| is applied. Stores the output in member variables + // |renderWidth| and |renderHeight| to avoid allocations since this function is called for every + // frame. + private void calculateTransformedRenderSize( + int frameWidth, int frameHeight, Matrix renderMatrix) { + if (renderMatrix == null) { + renderWidth = frameWidth; + renderHeight = frameHeight; + return; + } + // Transform the texture coordinates (in the range [0, 1]) according to |renderMatrix|. + renderMatrix.mapPoints(dstPoints, srcPoints); + + // Multiply with the width and height to get the positions in terms of pixels. + for (int i = 0; i < 3; ++i) { + dstPoints[i * 2 + 0] *= frameWidth; + dstPoints[i * 2 + 1] *= frameHeight; + } + + // Get the length of the sides of the transformed rectangle in terms of pixels. + renderWidth = distance(dstPoints[0], dstPoints[1], dstPoints[2], dstPoints[3]); + renderHeight = distance(dstPoints[0], dstPoints[1], dstPoints[4], dstPoints[5]); + } + + private final YuvUploader yuvUploader = new YuvUploader(); + // This variable will only be used for checking reference equality and is used for caching I420 + // textures. + private VideoFrame lastI420Frame; + private final Matrix renderMatrix = new Matrix(); + + public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer) { + drawFrame(frame, drawer, null /* additionalRenderMatrix */); + } + + public void drawFrame( + VideoFrame frame, RendererCommon.GlDrawer drawer, Matrix additionalRenderMatrix) { + drawFrame(frame, drawer, additionalRenderMatrix, 0 /* viewportX */, 0 /* viewportY */, + frame.getRotatedWidth(), frame.getRotatedHeight()); + } + + public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer, + Matrix additionalRenderMatrix, int viewportX, int viewportY, int viewportWidth, + int viewportHeight) { + final int width = frame.getRotatedWidth(); + final int height = frame.getRotatedHeight(); + + calculateTransformedRenderSize(width, height, additionalRenderMatrix); + + final boolean isTextureFrame = frame.getBuffer() instanceof VideoFrame.TextureBuffer; + renderMatrix.reset(); + renderMatrix.preTranslate(0.5f, 0.5f); + if (!isTextureFrame) { + renderMatrix.preScale(1f, -1f); // I420-frames are upside down + } + renderMatrix.preRotate(frame.getRotation()); + renderMatrix.preTranslate(-0.5f, -0.5f); + if (additionalRenderMatrix != null) { + renderMatrix.preConcat(additionalRenderMatrix); + } + + if (isTextureFrame) { + lastI420Frame = null; + drawTexture(drawer, (VideoFrame.TextureBuffer) frame.getBuffer(), renderMatrix, renderWidth, + renderHeight, viewportX, viewportY, viewportWidth, viewportHeight); + } else { + // Only upload the I420 data to textures once per frame, if we are called multiple times + // with the same frame. + if (frame != lastI420Frame) { + lastI420Frame = frame; + final VideoFrame.I420Buffer i420Buffer = frame.getBuffer().toI420(); + yuvUploader.uploadFromBuffer(i420Buffer); + i420Buffer.release(); + } + + drawer.drawYuv(yuvUploader.getYuvTextures(), + RendererCommon.convertMatrixFromAndroidGraphicsMatrix(renderMatrix), renderWidth, + renderHeight, viewportX, viewportY, viewportWidth, viewportHeight); + } + } + + public void release() { + yuvUploader.release(); + lastI420Frame = null; + } +}