From 066b42fa67e2206d2083ebf44ca794d5538f3bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sami=20Kalliom=C3=A4ki?= Date: Fri, 30 Aug 2019 11:20:42 +0200 Subject: [PATCH] Interface for monitoring ref counts of texture buffers created by SurfaceTextureHelper. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: b/139745386 Change-Id: I095d6b2862dac55044af5852098fb1c38e8738cf Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/150649 Commit-Queue: Sami Kalliomäki Reviewed-by: Alex Glaznev Cr-Commit-Position: refs/heads/master@{#29024} --- .../api/org/webrtc/SurfaceTextureHelper.java | 87 +++++++++++++++---- .../api/org/webrtc/TextureBufferImpl.java | 60 ++++++++++--- 2 files changed, 119 insertions(+), 28 deletions(-) diff --git a/sdk/android/api/org/webrtc/SurfaceTextureHelper.java b/sdk/android/api/org/webrtc/SurfaceTextureHelper.java index b8f5624d89..3522a87487 100644 --- a/sdk/android/api/org/webrtc/SurfaceTextureHelper.java +++ b/sdk/android/api/org/webrtc/SurfaceTextureHelper.java @@ -11,7 +11,6 @@ package org.webrtc; import android.annotation.TargetApi; -import android.graphics.Matrix; import android.graphics.SurfaceTexture; import android.opengl.GLES11Ext; import android.opengl.GLES20; @@ -19,9 +18,9 @@ import android.os.Build; import android.os.Handler; import android.os.HandlerThread; import android.support.annotation.Nullable; -import java.nio.ByteBuffer; import java.util.concurrent.Callable; -import org.webrtc.EglBase; +import org.webrtc.EglBase.Context; +import org.webrtc.TextureBufferImpl.RefCountMonitor; import org.webrtc.VideoFrame.TextureBuffer; /** @@ -32,6 +31,21 @@ import org.webrtc.VideoFrame.TextureBuffer; * resources once the texture frame is released. */ public class SurfaceTextureHelper { + /** + * Interface for monitoring texture buffers created from this SurfaceTexture. Since only one + * texture buffer can exist at a time, this can be used to monitor for stuck frames. + */ + public interface FrameRefMonitor { + /** A new frame was created. New frames start with ref count of 1. */ + void onNewBuffer(TextureBuffer textureBuffer); + /** Ref count of the frame was incremented by the calling thread. */ + void onRetainBuffer(TextureBuffer textureBuffer); + /** Ref count of the frame was decremented by the calling thread. */ + void onReleaseBuffer(TextureBuffer textureBuffer); + /** Frame was destroyed (ref count reached 0). */ + void onDestroyBuffer(TextureBuffer textureBuffer); + } + private static final String TAG = "SurfaceTextureHelper"; /** * Construct a new SurfaceTextureHelper sharing OpenGL resources with |sharedContext|. A dedicated @@ -43,8 +57,8 @@ public class SurfaceTextureHelper { * closer to actual creation time. */ public static SurfaceTextureHelper create(final String threadName, - final EglBase.Context sharedContext, boolean alignTimestamps, - final YuvConverter yuvConverter) { + final EglBase.Context sharedContext, boolean alignTimestamps, final YuvConverter yuvConverter, + FrameRefMonitor frameRefMonitor) { final HandlerThread thread = new HandlerThread(threadName); thread.start(); final Handler handler = new Handler(thread.getLooper()); @@ -58,7 +72,8 @@ public class SurfaceTextureHelper { @Override public SurfaceTextureHelper call() { try { - return new SurfaceTextureHelper(sharedContext, handler, alignTimestamps, yuvConverter); + return new SurfaceTextureHelper( + sharedContext, handler, alignTimestamps, yuvConverter, frameRefMonitor); } catch (RuntimeException e) { Logging.e(TAG, threadName + " create failure", e); return null; @@ -70,29 +85,67 @@ public class SurfaceTextureHelper { /** * Same as above with alignTimestamps set to false and yuvConverter set to new YuvConverter. * - * @see #create(String, EglBase.Context, boolean, YuvConverter) + * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor) */ public static SurfaceTextureHelper create( final String threadName, final EglBase.Context sharedContext) { - return create(threadName, sharedContext, /* alignTimestamps= */ false, new YuvConverter()); + return create(threadName, sharedContext, /* alignTimestamps= */ false, new YuvConverter(), + /*frameRefMonitor=*/null); } /** * Same as above with yuvConverter set to new YuvConverter. * - * @see #create(String, EglBase.Context, boolean, YuvConverter) + * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor) */ public static SurfaceTextureHelper create( final String threadName, final EglBase.Context sharedContext, boolean alignTimestamps) { - return create(threadName, sharedContext, alignTimestamps, new YuvConverter()); + return create( + threadName, sharedContext, alignTimestamps, new YuvConverter(), /*frameRefMonitor=*/null); } + /** + * Create a SurfaceTextureHelper without frame ref monitor. + * + * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor) + */ + public static SurfaceTextureHelper create(final String threadName, + final EglBase.Context sharedContext, boolean alignTimestamps, YuvConverter yuvConverter) { + return create( + threadName, sharedContext, alignTimestamps, yuvConverter, /*frameRefMonitor=*/null); + } + + private final RefCountMonitor textureRefCountMonitor = new RefCountMonitor() { + @Override + public void onRetain(TextureBufferImpl textureBuffer) { + if (frameRefMonitor != null) { + frameRefMonitor.onRetainBuffer(textureBuffer); + } + } + + @Override + public void onRelease(TextureBufferImpl textureBuffer) { + if (frameRefMonitor != null) { + frameRefMonitor.onReleaseBuffer(textureBuffer); + } + } + + @Override + public void onDestroy(TextureBufferImpl textureBuffer) { + returnTextureFrame(); + if (frameRefMonitor != null) { + frameRefMonitor.onDestroyBuffer(textureBuffer); + } + } + }; + private final Handler handler; private final EglBase eglBase; private final SurfaceTexture surfaceTexture; private final int oesTextureId; private final YuvConverter yuvConverter; @Nullable private final TimestampAligner timestampAligner; + private final FrameRefMonitor frameRefMonitor; // These variables are only accessed from the |handler| thread. @Nullable private VideoSink listener; @@ -121,14 +174,15 @@ public class SurfaceTextureHelper { } }; - private SurfaceTextureHelper(EglBase.Context sharedContext, Handler handler, - boolean alignTimestamps, YuvConverter yuvConverter) { + private SurfaceTextureHelper(Context sharedContext, Handler handler, boolean alignTimestamps, + YuvConverter yuvConverter, FrameRefMonitor frameRefMonitor) { if (handler.getLooper().getThread() != Thread.currentThread()) { throw new IllegalStateException("SurfaceTextureHelper must be created on the handler thread"); } this.handler = handler; this.timestampAligner = alignTimestamps ? new TimestampAligner() : null; this.yuvConverter = yuvConverter; + this.frameRefMonitor = frameRefMonitor; eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); try { @@ -304,12 +358,15 @@ public class SurfaceTextureHelper { if (timestampAligner != null) { timestampNs = timestampAligner.translateTimestamp(timestampNs); } - final VideoFrame.Buffer buffer = + final VideoFrame.TextureBuffer buffer = new TextureBufferImpl(textureWidth, textureHeight, TextureBuffer.Type.OES, oesTextureId, RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix), handler, - yuvConverter, this ::returnTextureFrame); + yuvConverter, textureRefCountMonitor); + if (frameRefMonitor != null) { + frameRefMonitor.onNewBuffer(buffer); + } final VideoFrame frame = new VideoFrame(buffer, frameRotation, timestampNs); - ((VideoSink) listener).onFrame(frame); + listener.onFrame(frame); frame.release(); } diff --git a/sdk/android/api/org/webrtc/TextureBufferImpl.java b/sdk/android/api/org/webrtc/TextureBufferImpl.java index a24f284790..3d3bbabf91 100644 --- a/sdk/android/api/org/webrtc/TextureBufferImpl.java +++ b/sdk/android/api/org/webrtc/TextureBufferImpl.java @@ -19,6 +19,12 @@ import android.support.annotation.Nullable; * release callback. ToI420() is implemented by providing a Handler and a YuvConverter. */ public class TextureBufferImpl implements VideoFrame.TextureBuffer { + interface RefCountMonitor { + void onRetain(TextureBufferImpl textureBuffer); + void onRelease(TextureBufferImpl textureBuffer); + void onDestroy(TextureBufferImpl textureBuffer); + } + // This is the full resolution the texture has in memory after applying the transformation matrix // that might include cropping. This resolution is useful to know when sampling the texture to // avoid downscaling artifacts. @@ -33,24 +39,34 @@ public class TextureBufferImpl implements VideoFrame.TextureBuffer { private final Handler toI420Handler; private final YuvConverter yuvConverter; private final RefCountDelegate refCountDelegate; + private final @Nullable RefCountMonitor refCountMonitor; public TextureBufferImpl(int width, int height, Type type, int id, Matrix transformMatrix, Handler toI420Handler, YuvConverter yuvConverter, @Nullable Runnable releaseCallback) { - this.unscaledWidth = width; - this.unscaledHeight = height; - this.width = width; - this.height = height; - this.type = type; - this.id = id; - this.transformMatrix = transformMatrix; - this.toI420Handler = toI420Handler; - this.yuvConverter = yuvConverter; - this.refCountDelegate = new RefCountDelegate(releaseCallback); + this(width, height, width, height, type, id, transformMatrix, toI420Handler, yuvConverter, + new RefCountMonitor() { + @Override + public void onRetain(TextureBufferImpl textureBuffer) {} + + @Override + public void onRelease(TextureBufferImpl textureBuffer) {} + + @Override + public void onDestroy(TextureBufferImpl textureBuffer) { + releaseCallback.run(); + } + }); + } + + TextureBufferImpl(int width, int height, Type type, int id, Matrix transformMatrix, + Handler toI420Handler, YuvConverter yuvConverter, RefCountMonitor refCountMonitor) { + this(width, height, width, height, type, id, transformMatrix, toI420Handler, yuvConverter, + refCountMonitor); } private TextureBufferImpl(int unscaledWidth, int unscaledHeight, int width, int height, Type type, int id, Matrix transformMatrix, Handler toI420Handler, YuvConverter yuvConverter, - @Nullable Runnable releaseCallback) { + RefCountMonitor refCountMonitor) { this.unscaledWidth = unscaledWidth; this.unscaledHeight = unscaledHeight; this.width = width; @@ -60,7 +76,8 @@ public class TextureBufferImpl implements VideoFrame.TextureBuffer { this.transformMatrix = transformMatrix; this.toI420Handler = toI420Handler; this.yuvConverter = yuvConverter; - this.refCountDelegate = new RefCountDelegate(releaseCallback); + this.refCountDelegate = new RefCountDelegate(() -> refCountMonitor.onDestroy(this)); + this.refCountMonitor = refCountMonitor; } @Override @@ -96,11 +113,13 @@ public class TextureBufferImpl implements VideoFrame.TextureBuffer { @Override public void retain() { + refCountMonitor.onRetain(this); refCountDelegate.retain(); } @Override public void release() { + refCountMonitor.onRelease(this); refCountDelegate.release(); } @@ -161,6 +180,21 @@ public class TextureBufferImpl implements VideoFrame.TextureBuffer { newMatrix.preConcat(transformMatrix); retain(); return new TextureBufferImpl(unscaledWidth, unscaledHeight, scaledWidth, scaledHeight, type, id, - newMatrix, toI420Handler, yuvConverter, this ::release); + newMatrix, toI420Handler, yuvConverter, new RefCountMonitor() { + @Override + public void onRetain(TextureBufferImpl textureBuffer) { + refCountMonitor.onRetain(TextureBufferImpl.this); + } + + @Override + public void onRelease(TextureBufferImpl textureBuffer) { + refCountMonitor.onRelease(TextureBufferImpl.this); + } + + @Override + public void onDestroy(TextureBufferImpl textureBuffer) { + release(); + } + }); } }