diff --git a/webrtc/sdk/android/api/org/webrtc/EglRenderer.java b/webrtc/sdk/android/api/org/webrtc/EglRenderer.java index f5c1fe1773..0e54145063 100644 --- a/webrtc/sdk/android/api/org/webrtc/EglRenderer.java +++ b/webrtc/sdk/android/api/org/webrtc/EglRenderer.java @@ -95,8 +95,6 @@ public class EglRenderer implements VideoRenderer.Callbacks { private EglBase eglBase; private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader(); private RendererCommon.GlDrawer drawer; - // Texture ids for YUV frames. Allocated on first arrival of a YUV frame. - private int[] yuvTextures = null; // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|. private final Object frameLock = new Object(); @@ -239,10 +237,7 @@ public class EglRenderer implements VideoRenderer.Callbacks { drawer.release(); drawer = null; } - if (yuvTextures != null) { - GLES20.glDeleteTextures(3, yuvTextures, 0); - yuvTextures = null; - } + yuvUploader.release(); if (bitmapTextureFramebuffer != null) { bitmapTextureFramebuffer.release(); bitmapTextureFramebuffer = null; @@ -364,33 +359,31 @@ public class EglRenderer implements VideoRenderer.Callbacks { * Register a callback to be invoked when a new video frame has been received. This version uses * the drawer of the EglRenderer that was passed in init. * - * @param listener The callback to be invoked. + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is * required. */ public void addFrameListener(final FrameListener listener, final float scale) { - postToRenderThread(new Runnable() { - @Override - public void run() { - frameListeners.add(new FrameListenerAndParams(listener, scale, drawer)); - } - }); + addFrameListener(listener, scale, null); } /** * Register a callback to be invoked when a new video frame has been received. * - * @param listener The callback to be invoked. + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is * required. - * @param drawer Custom drawer to use for this frame listener. + * @param drawer Custom drawer to use for this frame listener or null to use the default one. */ public void addFrameListener( - final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawer) { + final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) { postToRenderThread(new Runnable() { @Override public void run() { - frameListeners.add(new FrameListenerAndParams(listener, scale, drawer)); + final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam; + frameListeners.add(new FrameListenerAndParams(listener, scale, listenerDrawer)); } }); } @@ -403,6 +396,9 @@ public class EglRenderer implements VideoRenderer.Callbacks { * @param runnable The callback to remove. */ public void removeFrameListener(final FrameListener listener) { + if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) { + throw new RuntimeException("removeFrameListener must not be called on the render thread."); + } final CountDownLatch latch = new CountDownLatch(1); postToRenderThread(new Runnable() { @Override @@ -432,20 +428,6 @@ public class EglRenderer implements VideoRenderer.Callbacks { VideoRenderer.renderFrameDone(frame); return; } - // Check if fps reduction is active. - synchronized (fpsReductionLock) { - if (minRenderPeriodNs > 0) { - final long currentTimeNs = System.nanoTime(); - if (currentTimeNs < nextFrameTimeNs) { - logD("Dropping frame - fps reduction is active."); - VideoRenderer.renderFrameDone(frame); - return; - } - nextFrameTimeNs += minRenderPeriodNs; - // The time for the next frame should always be in the future. - nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs); - } - } synchronized (frameLock) { dropOldFrame = (pendingFrame != null); if (dropOldFrame) { @@ -543,6 +525,28 @@ public class EglRenderer implements VideoRenderer.Callbacks { VideoRenderer.renderFrameDone(frame); return; } + // Check if fps reduction is active. + final boolean shouldRenderFrame; + synchronized (fpsReductionLock) { + if (minRenderPeriodNs == Long.MAX_VALUE) { + // Rendering is paused. + shouldRenderFrame = false; + } else if (minRenderPeriodNs <= 0) { + // FPS reduction is disabled. + shouldRenderFrame = true; + } else { + final long currentTimeNs = System.nanoTime(); + if (currentTimeNs < nextFrameTimeNs) { + logD("Skipping frame rendering - fps reduction is active."); + shouldRenderFrame = false; + } else { + nextFrameTimeNs += minRenderPeriodNs; + // The time for the next frame should always be in the future. + nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs); + shouldRenderFrame = true; + } + } + } final long startTimeNs = System.nanoTime(); final float[] texMatrix = @@ -575,55 +579,61 @@ public class EglRenderer implements VideoRenderer.Callbacks { drawMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix); } - GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + boolean shouldUploadYuvTextures = false; if (frame.yuvFrame) { - // Make sure YUV textures are allocated. - if (yuvTextures == null) { - yuvTextures = new int[3]; - for (int i = 0; i < 3; i++) { - yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D); + shouldUploadYuvTextures = shouldRenderFrame; + // Check if there are frame listeners that we want to render a bitmap for regardless of if the + // frame was rendered. This is the case when there are frameListeners with scale != 0f. + if (!shouldUploadYuvTextures) { + for (FrameListenerAndParams listenerAndParams : frameListeners) { + if (listenerAndParams.scale != 0f) { + shouldUploadYuvTextures = true; + break; + } } } + } + final int[] yuvTextures = shouldUploadYuvTextures + ? yuvUploader.uploadYuvData(frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes) + : null; - yuvUploader.uploadYuvData( - yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes); - drawer.drawYuv(yuvTextures, drawMatrix, drawnFrameWidth, drawnFrameHeight, 0, 0, - eglBase.surfaceWidth(), eglBase.surfaceHeight()); - } else { - drawer.drawOes(frame.textureId, drawMatrix, drawnFrameWidth, drawnFrameHeight, 0, 0, - eglBase.surfaceWidth(), eglBase.surfaceHeight()); + if (shouldRenderFrame) { + GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + if (frame.yuvFrame) { + drawer.drawYuv(yuvTextures, drawMatrix, drawnFrameWidth, drawnFrameHeight, 0, 0, + eglBase.surfaceWidth(), eglBase.surfaceHeight()); + } else { + drawer.drawOes(frame.textureId, drawMatrix, drawnFrameWidth, drawnFrameHeight, 0, 0, + eglBase.surfaceWidth(), eglBase.surfaceHeight()); + } + + final long swapBuffersStartTimeNs = System.nanoTime(); + eglBase.swapBuffers(); + + final long currentTimeNs = System.nanoTime(); + synchronized (statisticsLock) { + ++framesRendered; + renderTimeNs += (currentTimeNs - startTimeNs); + renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs); + } } - final long swapBuffersStartTimeNs = System.nanoTime(); - eglBase.swapBuffers(); - - final long currentTimeNs = System.nanoTime(); - synchronized (statisticsLock) { - ++framesRendered; - renderTimeNs += (currentTimeNs - startTimeNs); - renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs); - } - - notifyCallbacks(frame, texMatrix); + notifyCallbacks(frame, yuvTextures, texMatrix); VideoRenderer.renderFrameDone(frame); } - private void notifyCallbacks(VideoRenderer.I420Frame frame, float[] texMatrix) { - // Make temporary copy of callback list to avoid ConcurrentModificationException, in case - // callbacks call addFramelistener or removeFrameListener. - final ArrayList tmpList; + private void notifyCallbacks( + VideoRenderer.I420Frame frame, int[] yuvTextures, float[] texMatrix) { if (frameListeners.isEmpty()) return; - tmpList = new ArrayList<>(frameListeners); - frameListeners.clear(); final float[] bitmapMatrix = RendererCommon.multiplyMatrices( RendererCommon.multiplyMatrices(texMatrix, mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix()), RendererCommon.verticalFlipMatrix()); - for (FrameListenerAndParams listenerAndParams : tmpList) { + for (FrameListenerAndParams listenerAndParams : frameListeners) { final int scaledWidth = (int) (listenerAndParams.scale * frame.rotatedWidth()); final int scaledHeight = (int) (listenerAndParams.scale * frame.rotatedHeight()); @@ -663,6 +673,7 @@ public class EglRenderer implements VideoRenderer.Callbacks { bitmap.copyPixelsFromBuffer(bitmapBuffer); listenerAndParams.listener.onFrame(bitmap); } + frameListeners.clear(); } private String averageTimeAsString(long sumTimeNs, int count) { diff --git a/webrtc/sdk/android/api/org/webrtc/RendererCommon.java b/webrtc/sdk/android/api/org/webrtc/RendererCommon.java index a82cea1d25..63ffb8e552 100644 --- a/webrtc/sdk/android/api/org/webrtc/RendererCommon.java +++ b/webrtc/sdk/android/api/org/webrtc/RendererCommon.java @@ -14,7 +14,6 @@ import android.graphics.Point; import android.opengl.GLES20; import android.opengl.Matrix; import android.view.View; - import java.nio.ByteBuffer; /** @@ -63,13 +62,14 @@ public class RendererCommon { // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader // that handles stride and compare performance with intermediate copy. private ByteBuffer copyBuffer; + private int[] yuvTextures; /** - * Upload |planes| into |outputYuvTextures|, taking stride into consideration. - * |outputYuvTextures| must have been generated in advance. + * Upload |planes| into OpenGL textures, taking stride into consideration. + * + * @return Array of three texture indices corresponding to Y-, U-, and V-plane respectively. */ - public void uploadYuvData( - int[] outputYuvTextures, int width, int height, int[] strides, ByteBuffer[] planes) { + public int[] uploadYuvData(int width, int height, int[] strides, ByteBuffer[] planes) { final int[] planeWidths = new int[] {width, width / 2, width / 2}; final int[] planeHeights = new int[] {height, height / 2, height / 2}; // Make a first pass to see if we need a temporary copy buffer. @@ -84,10 +84,17 @@ public class RendererCommon { && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) { copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded); } + // Make sure YUV textures are allocated. + if (yuvTextures == null) { + yuvTextures = new int[3]; + for (int i = 0; i < 3; i++) { + yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D); + } + } // Upload each plane. for (int i = 0; i < 3; ++i) { GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, outputYuvTextures[i]); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); // GLES only accepts packed data, i.e. stride == planeWidth. final ByteBuffer packedByteBuffer; if (strides[i] == planeWidths[i]) { @@ -101,6 +108,19 @@ public class RendererCommon { GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidths[i], planeHeights[i], 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer); } + return yuvTextures; + } + + /** + * Releases cached resources. Uploader can still be used and the resources will be reallocated + * on first use. + */ + public void release() { + copyBuffer = null; + if (yuvTextures != null) { + GLES20.glDeleteTextures(3, yuvTextures, 0); + yuvTextures = null; + } } } diff --git a/webrtc/sdk/android/api/org/webrtc/SurfaceViewRenderer.java b/webrtc/sdk/android/api/org/webrtc/SurfaceViewRenderer.java index 866ca59d68..2a9d04f046 100644 --- a/webrtc/sdk/android/api/org/webrtc/SurfaceViewRenderer.java +++ b/webrtc/sdk/android/api/org/webrtc/SurfaceViewRenderer.java @@ -111,21 +111,23 @@ public class SurfaceViewRenderer /** * Register a callback to be invoked when a new video frame has been received. * - * @param listener The callback to be invoked. + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is * required. * @param drawer Custom drawer to use for this frame listener. */ public void addFrameListener( - EglRenderer.FrameListener listener, float scale, final RendererCommon.GlDrawer drawer) { - eglRenderer.addFrameListener(listener, scale, drawer); + EglRenderer.FrameListener listener, float scale, RendererCommon.GlDrawer drawerParam) { + eglRenderer.addFrameListener(listener, scale, drawerParam); } /** * Register a callback to be invoked when a new video frame has been received. This version uses * the drawer of the EglRenderer that was passed in init. * - * @param listener The callback to be invoked. + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is * required. */ diff --git a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/EglRendererTest.java b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/EglRendererTest.java index 7bbdb5fc73..e8d5e5bca3 100644 --- a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/EglRendererTest.java +++ b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/EglRendererTest.java @@ -307,4 +307,15 @@ public class EglRendererTest { // Check the frame listener hasn't triggered. assertFalse(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); } + + @Test + @SmallTest + public void testFrameListenersWhilePaused() throws Exception { + // Test that frame listeners receive frames while renderer is paused. + eglRenderer.pauseVideo(); + eglRenderer.addFrameListener(testFrameListener, 1f /* scaleFactor */); + feedFrame(0); + assertTrue(testFrameListener.waitForBitmap(RENDER_WAIT_MS)); + checkBitmapContent(testFrameListener.resetAndGetBitmap(), 0); + } }