admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Android: Split out EGL rendering from SurfaceViewRenderer to separate class

Browse Source 
The purpose is to prepare for a TextureViewRenderer that will share the
EGL rendering code.

Two functional changes are also included:
* The implementation of SurfaceHolder.Callback.surfaceDestroyed will now
  block until the EGL surface is released. This is done in order to
  comply with the documentation that says: "If you have a rendering
  thread that directly accesses the surface, you must ensure that thread
  is no longer touching the Surface before returning from this function."
* We will no longer try to hide render glitches during layout changes.
  This was a lost cause anyway.

BUG=webrtc:6407

Review-Url: https://codereview.webrtc.org/2399463006
Cr-Commit-Position: refs/heads/master@{#14570}
This commit is contained in:
magjed 2016-10-07 05:32:35 -07:00 committed by Commit bot
parent 36542514f6
commit df494b0908
4 changed files with 462 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
webrtc/api/BUILD.gn
View File

@ -261,6 +261,7 @@ if (is_android && !build_with_chromium) {
"android/java/src/org/webrtc/EglBase.java",
"android/java/src/org/webrtc/EglBase10.java",
"android/java/src/org/webrtc/EglBase14.java",
"android/java/src/org/webrtc/EglRenderer.java",
"android/java/src/org/webrtc/GlRectDrawer.java",
"android/java/src/org/webrtc/GlShader.java",
"android/java/src/org/webrtc/GlTextureFrameBuffer.java",

413
webrtc/api/android/java/src/org/webrtc/EglRenderer.java Normal file
View File

@ -0,0 +1,413 @@
/*
* Copyright 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Looper;
import android.view.Surface;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
* Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on an EGL Surface.
* This class is intended to be used as a helper class for rendering on SurfaceViews and
* TextureViews.
*/
public class EglRenderer implements VideoRenderer.Callbacks {
private static final String TAG = "EglRenderer";
private static final int MAX_SURFACE_CLEAR_COUNT = 3;
private class EglSurfaceCreation implements Runnable {
private Surface surface;
public synchronized void setSurface(Surface surface) {
this.surface = surface;
}
@Override
public synchronized void run() {
if (surface != null && eglBase != null && !eglBase.hasSurface()) {
eglBase.createSurface((Surface) surface);
eglBase.makeCurrent();
// Necessary for YUV frames with odd width.
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
}
}
}
private final String name;
// |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
// on |handlerLock|.
private final Object handlerLock = new Object();
private Handler renderThreadHandler;
// EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
// from the render thread.
private EglBase eglBase;
private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader();
private RendererCommon.GlDrawer drawer;
// Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
private int[] yuvTextures = null;
// Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
private final Object frameLock = new Object();
private VideoRenderer.I420Frame pendingFrame;
// These variables are synchronized on |layoutLock|.
private final Object layoutLock = new Object();
private int surfaceWidth;
private int surfaceHeight;
private float layoutAspectRatio;
// If true, mirrors the video stream horizontally.
private boolean mirror;
// These variables are synchronized on |statisticsLock|.
private final Object statisticsLock = new Object();
// Total number of video frames received in renderFrame() call.
private int framesReceived;
// Number of video frames dropped by renderFrame() because previous frame has not been rendered
// yet.
private int framesDropped;
// Number of rendered video frames.
private int framesRendered;
// Time in ns when the first video frame was rendered.
private long firstFrameTimeNs;
// Time in ns spent in renderFrameOnRenderThread() function.
private long renderTimeNs;
// Runnable for posting frames to render thread.
private final Runnable renderFrameRunnable = new Runnable() {
@Override
public void run() {
renderFrameOnRenderThread();
}
};
private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation();
/**
* Standard constructor. The name will be used for the render thread name and included when
* logging. In order to render something, you must first call init() and createEglSurface.
*/
public EglRenderer(String name) {
this.name = name;
}
/**
* Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used
* for drawing frames on the EGLSurface. This class is responsible for calling release() on
* |drawer|. It is allowed to call init() to reinitialize the renderer after a previous
* init()/release() cycle.
*/
public void init(final EglBase.Context sharedContext, final int[] configAttributes,
RendererCommon.GlDrawer drawer) {
synchronized (handlerLock) {
if (renderThreadHandler != null) {
throw new IllegalStateException(name + "Already initialized");
}
logD("Initializing EglRenderer");
this.drawer = drawer;
final HandlerThread renderThread = new HandlerThread(name + "EglRenderer");
renderThread.start();
renderThreadHandler = new Handler(renderThread.getLooper());
// Create EGL context on the newly created render thread. It should be possibly to create the
// context on this thread and make it current on the render thread, but this causes failure on
// some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, new Runnable() {
@Override
public void run() {
eglBase = EglBase.create(sharedContext, configAttributes);
}
});
}
}
public void createEglSurface(Surface surface) {
eglSurfaceCreationRunnable.setSurface(surface);
runOnRenderThread(eglSurfaceCreationRunnable);
}
/**
* Block until any pending frame is returned and all GL resources released, even if an interrupt
* occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function
* should be called before the Activity is destroyed and the EGLContext is still valid. If you
* don't call this function, the GL resources might leak.
*/
public void release() {
final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
synchronized (handlerLock) {
if (renderThreadHandler == null) {
logD("Already released");
return;
}
// Release EGL and GL resources on render thread.
renderThreadHandler.postAtFrontOfQueue(new Runnable() {
@Override
public void run() {
if (drawer != null) {
drawer.release();
drawer = null;
}
if (yuvTextures != null) {
GLES20.glDeleteTextures(3, yuvTextures, 0);
yuvTextures = null;
}
if (eglBase != null) {
logD("eglBase detach and release.");
eglBase.detachCurrent();
eglBase.release();
eglBase = null;
}
eglCleanupBarrier.countDown();
}
});
final Looper renderLooper = renderThreadHandler.getLooper();
// TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18.
renderThreadHandler.post(new Runnable() {
@Override
public void run() {
logD("Quitting render thread.");
renderLooper.quit();
}
});
// Don't accept any more frames or messages to the render thread.
renderThreadHandler = null;
}
// Make sure the EGL/GL cleanup posted above is executed.
ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
synchronized (frameLock) {
if (pendingFrame != null) {
VideoRenderer.renderFrameDone(pendingFrame);
pendingFrame = null;
}
}
resetStatistics();
logD("Releasing done.");
}
/**
* Reset statistics. This will reset the logged statistics in logStatistics(), and
* RendererEvents.onFirstFrameRendered() will be called for the next frame.
*/
public void resetStatistics() {
synchronized (statisticsLock) {
framesReceived = 0;
framesDropped = 0;
framesRendered = 0;
firstFrameTimeNs = 0;
renderTimeNs = 0;
}
}
/**
* Set if the video stream should be mirrored or not.
*/
public void setMirror(final boolean mirror) {
logD("setMirror: " + mirror);
synchronized (layoutLock) {
this.mirror = mirror;
}
}
/**
* Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video.
* Set this to 0 to disable cropping.
*/
public void setLayoutAspectRatio(float layoutAspectRatio) {
logD("setLayoutAspectRatio: " + layoutAspectRatio);
synchronized (layoutLock) {
this.layoutAspectRatio = layoutAspectRatio;
}
}
// VideoRenderer.Callbacks interface.
@Override
public void renderFrame(VideoRenderer.I420Frame frame) {
synchronized (statisticsLock) {
++framesReceived;
}
synchronized (handlerLock) {
if (renderThreadHandler == null) {
logD("Dropping frame - Not initialized or already released.");
VideoRenderer.renderFrameDone(frame);
return;
}
synchronized (frameLock) {
if (pendingFrame != null) {
// Drop old frame.
synchronized (statisticsLock) {
++framesDropped;
}
VideoRenderer.renderFrameDone(pendingFrame);
}
pendingFrame = frame;
renderThreadHandler.post(renderFrameRunnable);
}
}
}
/**
* Release EGL surface. This function will block until the EGL surface is released.
*/
public void releaseEglSurface() {
// Ensure that the render thread is no longer touching the Surface before returning from this
// function.
eglSurfaceCreationRunnable.setSurface(null /* surface */);
synchronized (handlerLock) {
if (renderThreadHandler != null) {
renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable);
ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, new Runnable() {
@Override
public void run() {
if (eglBase != null) {
eglBase.detachCurrent();
eglBase.releaseSurface();
}
}
});
}
}
}
/**
* Notify that the surface size has changed.
*/
public void surfaceSizeChanged(int surfaceWidth, int surfaceHeight) {
logD("Surface size changed: " + surfaceWidth + "x" + surfaceHeight);
synchronized (layoutLock) {
this.surfaceWidth = surfaceWidth;
this.surfaceHeight = surfaceHeight;
}
}
/**
* Private helper function to post tasks safely.
*/
private void runOnRenderThread(Runnable runnable) {
synchronized (handlerLock) {
if (renderThreadHandler != null) {
renderThreadHandler.post(runnable);
}
}
}
private void makeBlack() {
if (eglBase != null && eglBase.hasSurface()) {
logD("clearSurface");
GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
eglBase.swapBuffers();
}
}
/**
* Renders and releases |pendingFrame|.
*/
private void renderFrameOnRenderThread() {
// Fetch and render |pendingFrame|.
final VideoRenderer.I420Frame frame;
synchronized (frameLock) {
if (pendingFrame == null) {
return;
}
frame = pendingFrame;
pendingFrame = null;
}
if (eglBase == null || !eglBase.hasSurface()) {
logD("Dropping frame - No surface");
VideoRenderer.renderFrameDone(frame);
return;
}
final long startTimeNs = System.nanoTime();
float[] texMatrix =
RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
// After a surface size change, the EGLSurface might still have a buffer of the old size in the
// pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
// changed. Such a buffer will be rendered incorrectly, so flush it with a black frame.
synchronized (layoutLock) {
int surfaceClearCount = 0;
while (eglBase.surfaceWidth() != surfaceWidth || eglBase.surfaceHeight() != surfaceHeight) {
++surfaceClearCount;
if (surfaceClearCount > MAX_SURFACE_CLEAR_COUNT) {
logD("Failed to get surface of expected size - dropping frame.");
VideoRenderer.renderFrameDone(frame);
return;
}
logD("Surface size mismatch - clearing surface.");
makeBlack();
}
final float[] layoutMatrix;
if (layoutAspectRatio > 0) {
layoutMatrix = RendererCommon.getLayoutMatrix(
mirror, frame.rotatedWidth() / (float) frame.rotatedHeight(), layoutAspectRatio);
} else {
layoutMatrix =
mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix();
}
texMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
}
GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
if (frame.yuvFrame) {
// Make sure YUV textures are allocated.
if (yuvTextures == null) {
yuvTextures = new int[3];
for (int i = 0; i < 3; i++) {
yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
}
}
yuvUploader.uploadYuvData(
yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
drawer.drawYuv(yuvTextures, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
surfaceWidth, surfaceHeight);
} else {
drawer.drawOes(frame.textureId, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
surfaceWidth, surfaceHeight);
}
eglBase.swapBuffers();
VideoRenderer.renderFrameDone(frame);
synchronized (statisticsLock) {
if (framesRendered == 0) {
firstFrameTimeNs = startTimeNs;
}
++framesRendered;
renderTimeNs += (System.nanoTime() - startTimeNs);
if (framesRendered % 300 == 0) {
logStatistics();
}
}
}
private void logStatistics() {
synchronized (statisticsLock) {
logD("Frames received: " + framesReceived + ". Dropped: " + framesDropped + ". Rendered: "
+ framesRendered);
if (framesReceived > 0 && framesRendered > 0) {
final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs;
logD("Duration: " + (int) (timeSinceFirstFrameNs / 1e6) + " ms. FPS: "
+ framesRendered * 1e9 / timeSinceFirstFrameNs);
logD("Average render time: " + (int) (renderTimeNs / (1000 * framesRendered)) + " us.");
}
}
}
private void logD(String string) {
Logging.d(TAG, name + string);
}
}

5
webrtc/api/android/java/src/org/webrtc/RendererCommon.java
View File

@ -137,11 +137,10 @@ public class RendererCommon {
// and maximum layout size.
final float frameAspect = frameWidth / (float) frameHeight;
final float displayAspect = maxWidth / (float) maxHeight;
final RendererCommon.ScalingType scalingType = (frameAspect > 1.0f) == (displayAspect > 1.0f)
final ScalingType scalingType = (frameAspect > 1.0f) == (displayAspect > 1.0f)
? scalingTypeMatchOrientation
: scalingTypeMismatchOrientation;
final Point layoutSize =
RendererCommon.getDisplaySize(scalingType, frameAspect, maxWidth, maxHeight);
final Point layoutSize = getDisplaySize(scalingType, frameAspect, maxWidth, maxHeight);
// If the measure specification is forcing a specific size - yield.
if (View.MeasureSpec.getMode(widthSpec) == View.MeasureSpec.EXACTLY) {

439
webrtc/api/android/java/src/org/webrtc/SurfaceViewRenderer.java
View File

@ -13,17 +13,10 @@ package org.webrtc;
import android.content.Context;
import android.content.res.Resources.NotFoundException;
import android.graphics.Point;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.HandlerThread;
import android.util.AttributeSet;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import java.util.concurrent.CountDownLatch;
import javax.microedition.khronos.egl.EGLContext;
/**
* Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on a SurfaceView.
* renderFrame() is asynchronous to avoid blocking the calling thread.
@ -37,91 +30,28 @@ public class SurfaceViewRenderer
extends SurfaceView implements SurfaceHolder.Callback, VideoRenderer.Callbacks {
private static final String TAG = "SurfaceViewRenderer";
// Dedicated render thread.
private HandlerThread renderThread;
// |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
// on |handlerLock|.
private final Object handlerLock = new Object();
private Handler renderThreadHandler;
// Cached resource name.
private final String resourceName;
private final RendererCommon.VideoLayoutMeasure videoLayoutMeasure =
new RendererCommon.VideoLayoutMeasure();
private final EglRenderer eglRenderer;
// EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
// from the render thread.
private EglBase eglBase;
private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader();
private RendererCommon.GlDrawer drawer;
// Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
private int[] yuvTextures = null;
// Callback for reporting renderer events. Read-only after initilization so no lock required.
private RendererCommon.RendererEvents rendererEvents;
// Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
private final Object frameLock = new Object();
private VideoRenderer.I420Frame pendingFrame;
// These variables are synchronized on |layoutLock|.
private final Object layoutLock = new Object();
// These dimension values are used to keep track of the state in these functions: onMeasure(),
// onLayout(), and surfaceChanged(). A new layout is triggered with requestLayout(). This happens
// internally when the incoming frame size changes. requestLayout() can also be triggered
// externally. The layout change is a two pass process: first onMeasure() is called in a top-down
// traversal of the View tree, followed by an onLayout() pass that is also top-down. During the
// onLayout() pass, each parent is responsible for positioning its children using the sizes
// computed in the measure pass.
// |desiredLayoutsize| is the layout size we have requested in onMeasure() and are waiting for to
// take effect.
private Point desiredLayoutSize = new Point();
// |layoutSize|/|surfaceSize| is the actual current layout/surface size. They are updated in
// onLayout() and surfaceChanged() respectively.
private final Point layoutSize = new Point();
// TODO(magjed): Enable hardware scaler with SurfaceHolder.setFixedSize(). This will decouple
// layout and surface size.
private final Point surfaceSize = new Point();
// |isSurfaceCreated| keeps track of the current status in surfaceCreated()/surfaceDestroyed().
private boolean isSurfaceCreated;
// Last rendered frame dimensions, or 0 if no frame has been rendered yet.
private boolean isFirstFrameRendered;
private int rotatedFrameWidth;
private int rotatedFrameHeight;
private int frameRotation;
private final RendererCommon.VideoLayoutMeasure videoLayoutMeasure =
new RendererCommon.VideoLayoutMeasure();
// If true, mirrors the video stream horizontally.
private boolean mirror;
// Callback for reporting renderer events.
private RendererCommon.RendererEvents rendererEvents;
// These variables are synchronized on |statisticsLock|.
private final Object statisticsLock = new Object();
// Total number of video frames received in renderFrame() call.
private int framesReceived;
// Number of video frames dropped by renderFrame() because previous frame has not been rendered
// yet.
private int framesDropped;
// Number of rendered video frames.
private int framesRendered;
// Time in ns when the first video frame was rendered.
private long firstFrameTimeNs;
// Time in ns spent in renderFrameOnRenderThread() function.
private long renderTimeNs;
// Runnable for posting frames to render thread.
private final Runnable renderFrameRunnable = new Runnable() {
@Override
public void run() {
renderFrameOnRenderThread();
}
};
// Runnable for clearing Surface to black.
private final Runnable makeBlackRunnable = new Runnable() {
@Override
public void run() {
makeBlack();
}
};
/**
* Standard View constructor. In order to render something, you must first call init().
*/
public SurfaceViewRenderer(Context context) {
super(context);
this.resourceName = getResourceName();
eglRenderer = new EglRenderer(resourceName);
getHolder().addCallback(this);
}
@ -130,6 +60,8 @@ public class SurfaceViewRenderer
*/
public SurfaceViewRenderer(Context context, AttributeSet attrs) {
super(context, attrs);
this.resourceName = getResourceName();
eglRenderer = new EglRenderer(resourceName);
getHolder().addCallback(this);
}
@ -150,48 +82,14 @@ public class SurfaceViewRenderer
public void init(final EglBase.Context sharedContext,
RendererCommon.RendererEvents rendererEvents, final int[] configAttributes,
RendererCommon.GlDrawer drawer) {
synchronized (handlerLock) {
if (renderThreadHandler != null) {
throw new IllegalStateException(getResourceName() + "Already initialized");
}
Logging.d(TAG, getResourceName() + "Initializing.");
this.rendererEvents = rendererEvents;
this.drawer = drawer;
renderThread = new HandlerThread(TAG);
renderThread.start();
renderThreadHandler = new Handler(renderThread.getLooper());
// Create EGL context on the newly created render thread. It should be possibly to create the
// context on this thread and make it current on the render thread, but this causes failure on
// some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, new Runnable() {
@Override
public void run() {
eglBase = EglBase.create(sharedContext, configAttributes);
}
});
ThreadUtils.checkIsOnMainThread();
this.rendererEvents = rendererEvents;
synchronized (layoutLock) {
rotatedFrameWidth = 0;
rotatedFrameHeight = 0;
frameRotation = 0;
}
tryCreateEglSurface();
}
/**
* Create and make an EGLSurface current if both init() and surfaceCreated() have been called.
*/
public void tryCreateEglSurface() {
// |renderThreadHandler| is only created after |eglBase| is created in init(), so the
// following code will only execute if eglBase != null.
runOnRenderThread(new Runnable() {
@Override
public void run() {
synchronized (layoutLock) {
if (eglBase != null && isSurfaceCreated && !eglBase.hasSurface()) {
eglBase.createSurface(getHolder().getSurface());
eglBase.makeCurrent();
// Necessary for YUV frames with odd width.
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
}
}
}
});
eglRenderer.init(sharedContext, configAttributes, drawer);
}
/**
@ -201,78 +99,14 @@ public class SurfaceViewRenderer
* don't call this function, the GL resources might leak.
*/
public void release() {
final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
synchronized (handlerLock) {
if (renderThreadHandler == null) {
Logging.d(TAG, getResourceName() + "Already released");
return;
}
// Release EGL and GL resources on render thread.
// TODO(magjed): This might not be necessary - all OpenGL resources are automatically deleted
// when the EGL context is lost. It might be dangerous to delete them manually in
// Activity.onDestroy().
renderThreadHandler.postAtFrontOfQueue(new Runnable() {
@Override
public void run() {
drawer.release();
drawer = null;
if (yuvTextures != null) {
GLES20.glDeleteTextures(3, yuvTextures, 0);
yuvTextures = null;
}
// Clear last rendered image to black.
makeBlack();
eglBase.release();
eglBase = null;
eglCleanupBarrier.countDown();
}
});
// Don't accept any more frames or messages to the render thread.
renderThreadHandler = null;
}
// Make sure the EGL/GL cleanup posted above is executed.
ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
renderThread.quit();
synchronized (frameLock) {
if (pendingFrame != null) {
VideoRenderer.renderFrameDone(pendingFrame);
pendingFrame = null;
}
}
// The |renderThread| cleanup is not safe to cancel and we need to wait until it's done.
ThreadUtils.joinUninterruptibly(renderThread);
renderThread = null;
// Reset statistics and event reporting.
synchronized (layoutLock) {
rotatedFrameWidth = 0;
rotatedFrameHeight = 0;
frameRotation = 0;
rendererEvents = null;
}
resetStatistics();
}
/**
* Reset statistics. This will reset the logged statistics in logStatistics(), and
* RendererEvents.onFirstFrameRendered() will be called for the next frame.
*/
public void resetStatistics() {
synchronized (statisticsLock) {
framesReceived = 0;
framesDropped = 0;
framesRendered = 0;
firstFrameTimeNs = 0;
renderTimeNs = 0;
}
eglRenderer.release();
}
/**
* Set if the video stream should be mirrored or not.
*/
public void setMirror(final boolean mirror) {
synchronized (layoutLock) {
this.mirror = mirror;
}
eglRenderer.setMirror(mirror);
}
/**
@ -292,112 +126,46 @@ public class SurfaceViewRenderer
// VideoRenderer.Callbacks interface.
@Override
public void renderFrame(VideoRenderer.I420Frame frame) {
synchronized (statisticsLock) {
++framesReceived;
}
synchronized (handlerLock) {
if (renderThreadHandler == null) {
Logging.d(TAG, getResourceName() + "Dropping frame - Not initialized or already released.");
VideoRenderer.renderFrameDone(frame);
return;
}
synchronized (frameLock) {
if (pendingFrame != null) {
// Drop old frame.
synchronized (statisticsLock) {
++framesDropped;
}
VideoRenderer.renderFrameDone(pendingFrame);
}
pendingFrame = frame;
renderThreadHandler.post(renderFrameRunnable);
}
}
updateFrameDimensionsAndReportEvents(frame);
eglRenderer.renderFrame(frame);
}
// View layout interface.
@Override
protected void onMeasure(int widthSpec, int heightSpec) {
ThreadUtils.checkIsOnMainThread();
final boolean isNewSize;
final Point size;
synchronized (layoutLock) {
desiredLayoutSize =
size =
videoLayoutMeasure.measure(widthSpec, heightSpec, rotatedFrameWidth, rotatedFrameHeight);
isNewSize =
(desiredLayoutSize.x != getMeasuredWidth() || desiredLayoutSize.y != getMeasuredHeight());
setMeasuredDimension(desiredLayoutSize.x, desiredLayoutSize.y);
}
if (isNewSize) {
// Clear the surface asap before the layout change to avoid stretched video and other
// render artifacs. Don't wait for it to finish because the IO thread should never be
// blocked, so it's a best-effort attempt.
synchronized (handlerLock) {
if (renderThreadHandler != null) {
renderThreadHandler.postAtFrontOfQueue(makeBlackRunnable);
}
}
}
setMeasuredDimension(size.x, size.y);
logD("onMeasure(). New size: " + size.x + "x" + size.y);
}
@Override
protected void onLayout(boolean changed, int left, int top, int right, int bottom) {
synchronized (layoutLock) {
layoutSize.x = right - left;
layoutSize.y = bottom - top;
}
// Might have a pending frame waiting for a layout of correct size.
runOnRenderThread(renderFrameRunnable);
ThreadUtils.checkIsOnMainThread();
eglRenderer.setLayoutAspectRatio((right - left) / (float) (bottom - top));
}
// SurfaceHolder.Callback interface.
@Override
public void surfaceCreated(final SurfaceHolder holder) {
Logging.d(TAG, getResourceName() + "Surface created.");
synchronized (layoutLock) {
isSurfaceCreated = true;
}
tryCreateEglSurface();
ThreadUtils.checkIsOnMainThread();
eglRenderer.createEglSurface(holder.getSurface());
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
Logging.d(TAG, getResourceName() + "Surface destroyed.");
synchronized (layoutLock) {
isSurfaceCreated = false;
surfaceSize.x = 0;
surfaceSize.y = 0;
}
runOnRenderThread(new Runnable() {
@Override
public void run() {
if (eglBase != null) {
eglBase.detachCurrent();
eglBase.releaseSurface();
}
}
});
ThreadUtils.checkIsOnMainThread();
eglRenderer.releaseEglSurface();
}
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
Logging.d(TAG, getResourceName() + "Surface changed: " + width + "x" + height);
synchronized (layoutLock) {
surfaceSize.x = width;
surfaceSize.y = height;
}
// Might have a pending frame waiting for a surface of correct size.
runOnRenderThread(renderFrameRunnable);
}
/**
* Private helper function to post tasks safely.
*/
private void runOnRenderThread(Runnable runnable) {
synchronized (handlerLock) {
if (renderThreadHandler != null) {
renderThreadHandler.post(runnable);
}
}
ThreadUtils.checkIsOnMainThread();
eglRenderer.surfaceSizeChanged(width, height);
}
private String getResourceName() {
@ -408,125 +176,20 @@ public class SurfaceViewRenderer
}
}
private void makeBlack() {
if (Thread.currentThread() != renderThread) {
throw new IllegalStateException(getResourceName() + "Wrong thread.");
}
if (eglBase != null && eglBase.hasSurface()) {
GLES20.glClearColor(0, 0, 0, 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
eglBase.swapBuffers();
}
}
/**
* Requests new layout if necessary. Returns true if layout and surface size are consistent.
*/
private boolean checkConsistentLayout() {
if (Thread.currentThread() != renderThread) {
throw new IllegalStateException(getResourceName() + "Wrong thread.");
}
synchronized (layoutLock) {
// Return false while we are in the middle of a layout change.
return layoutSize.equals(desiredLayoutSize) && surfaceSize.equals(layoutSize);
}
}
/**
* Renders and releases |pendingFrame|.
*/
private void renderFrameOnRenderThread() {
if (Thread.currentThread() != renderThread) {
throw new IllegalStateException(getResourceName() + "Wrong thread.");
}
// Fetch and render |pendingFrame|.
final VideoRenderer.I420Frame frame;
synchronized (frameLock) {
if (pendingFrame == null) {
return;
}
frame = pendingFrame;
pendingFrame = null;
}
updateFrameDimensionsAndReportEvents(frame);
if (eglBase == null || !eglBase.hasSurface()) {
Logging.d(TAG, getResourceName() + "No surface to draw on");
VideoRenderer.renderFrameDone(frame);
return;
}
if (!checkConsistentLayout()) {
// Output intermediate black frames while the layout is updated.
makeBlack();
VideoRenderer.renderFrameDone(frame);
return;
}
// After a surface size change, the EGLSurface might still have a buffer of the old size in the
// pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
// changed. Such a buffer will be rendered incorrectly, so flush it with a black frame.
synchronized (layoutLock) {
if (eglBase.surfaceWidth() != surfaceSize.x || eglBase.surfaceHeight() != surfaceSize.y) {
makeBlack();
}
}
final long startTimeNs = System.nanoTime();
final float[] texMatrix;
synchronized (layoutLock) {
final float[] rotatedSamplingMatrix =
RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
final float[] layoutMatrix = RendererCommon.getLayoutMatrix(mirror,
frame.rotatedWidth() / (float) frame.rotatedHeight(),
layoutSize.x / (float) layoutSize.y);
texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix);
}
// TODO(magjed): glClear() shouldn't be necessary since every pixel is covered anyway, but it's
// a workaround for bug 5147. Performance will be slightly worse.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
if (frame.yuvFrame) {
// Make sure YUV textures are allocated.
if (yuvTextures == null) {
yuvTextures = new int[3];
for (int i = 0; i < 3; i++) {
yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
}
}
yuvUploader.uploadYuvData(
yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
drawer.drawYuv(yuvTextures, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
surfaceSize.x, surfaceSize.y);
} else {
drawer.drawOes(frame.textureId, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
surfaceSize.x, surfaceSize.y);
}
eglBase.swapBuffers();
VideoRenderer.renderFrameDone(frame);
synchronized (statisticsLock) {
if (framesRendered == 0) {
firstFrameTimeNs = startTimeNs;
synchronized (layoutLock) {
Logging.d(TAG, getResourceName() + "Reporting first rendered frame.");
if (rendererEvents != null) {
rendererEvents.onFirstFrameRendered();
}
}
}
++framesRendered;
renderTimeNs += (System.nanoTime() - startTimeNs);
if (framesRendered % 300 == 0) {
logStatistics();
}
}
}
// Update frame dimensions and report any changes to |rendererEvents|.
private void updateFrameDimensionsAndReportEvents(VideoRenderer.I420Frame frame) {
synchronized (layoutLock) {
if (!isFirstFrameRendered) {
isFirstFrameRendered = true;
logD("Reporting first rendered frame.");
if (rendererEvents != null) {
rendererEvents.onFirstFrameRendered();
}
}
if (rotatedFrameWidth != frame.rotatedWidth() || rotatedFrameHeight != frame.rotatedHeight()
|| frameRotation != frame.rotationDegree) {
Logging.d(TAG, getResourceName() + "Reporting frame resolution changed to " + frame.width
+ "x" + frame.height + " with rotation " + frame.rotationDegree);
logD("Reporting frame resolution changed to " + frame.width + "x" + frame.height
+ " with rotation " + frame.rotationDegree);
if (rendererEvents != null) {
rendererEvents.onFrameResolutionChanged(frame.width, frame.height, frame.rotationDegree);
}
@ -543,17 +206,7 @@ public class SurfaceViewRenderer
}
}
private void logStatistics() {
synchronized (statisticsLock) {
Logging.d(TAG, getResourceName() + "Frames received: " + framesReceived + ". Dropped: "
+ framesDropped + ". Rendered: " + framesRendered);
if (framesReceived > 0 && framesRendered > 0) {
final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs;
Logging.d(TAG, getResourceName() + "Duration: " + (int) (timeSinceFirstFrameNs / 1e6)
+ " ms. FPS: " + framesRendered * 1e9 / timeSinceFirstFrameNs);
Logging.d(TAG, getResourceName() + "Average render time: "
+ (int) (renderTimeNs / (1000 * framesRendered)) + " us.");
}
}
private void logD(String string) {
Logging.d(TAG, resourceName + string);
}
}