diff --git a/AUTHORS b/AUTHORS index 3bf6671fb2..285cde44f6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -53,6 +53,7 @@ Vladimir Beloborodov Vicken Simonian Victor Costan Xiaohong Xu +Xiaolei Yu Hans Knoechel Korniltsev Anatoly diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 19eb00b637..f30c7aef9d 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -438,6 +438,7 @@ android_library("libjingle_peerconnection_java") { "api/org/webrtc/StatsObserver.java", "api/org/webrtc/StatsReport.java", "api/org/webrtc/SurfaceTextureHelper.java", + "api/org/webrtc/SurfaceEglRenderer.java", "api/org/webrtc/SurfaceViewRenderer.java", "api/org/webrtc/TurnCustomizer.java", "api/org/webrtc/VideoCapturer.java", diff --git a/sdk/android/api/org/webrtc/EglRenderer.java b/sdk/android/api/org/webrtc/EglRenderer.java index 16e89f4d21..22e1a4cff0 100644 --- a/sdk/android/api/org/webrtc/EglRenderer.java +++ b/sdk/android/api/org/webrtc/EglRenderer.java @@ -76,7 +76,7 @@ public class EglRenderer implements VideoRenderer.Callbacks, VideoSink { } } - private final String name; + protected final String name; // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized // on |handlerLock|. diff --git a/sdk/android/api/org/webrtc/SurfaceEglRenderer.java b/sdk/android/api/org/webrtc/SurfaceEglRenderer.java new file mode 100644 index 0000000000..338871c0a1 --- /dev/null +++ b/sdk/android/api/org/webrtc/SurfaceEglRenderer.java @@ -0,0 +1,194 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.view.SurfaceHolder; +import java.util.concurrent.CountDownLatch; + +/** + * Display the video stream on a Surface. + * renderFrame() is asynchronous to avoid blocking the calling thread. + * This class is thread safe and handles access from potentially three different threads: + * Interaction from the main app in init, release and setMirror. + * Interaction from C++ rtc::VideoSinkInterface in renderFrame. + * Interaction from SurfaceHolder lifecycle in surfaceCreated, surfaceChanged, and surfaceDestroyed. + */ +public class SurfaceEglRenderer extends EglRenderer implements SurfaceHolder.Callback { + private static final String TAG = "SurfaceEglRenderer"; + + // Callback for reporting renderer events. Read-only after initilization so no lock required. + private RendererCommon.RendererEvents rendererEvents; + + private final Object layoutLock = new Object(); + private boolean isRenderingPaused = false; + private boolean isFirstFrameRendered; + private int rotatedFrameWidth; + private int rotatedFrameHeight; + private int frameRotation; + + /** + * In order to render something, you must first call init(). + */ + public SurfaceEglRenderer(String name) { + super(name); + } + + /** + * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used + * for drawing frames on the EGLSurface. This class is responsible for calling release() on + * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous + * init()/release() cycle. + */ + public void init(final EglBase.Context sharedContext, + RendererCommon.RendererEvents rendererEvents, final int[] configAttributes, + RendererCommon.GlDrawer drawer) { + ThreadUtils.checkIsOnMainThread(); + this.rendererEvents = rendererEvents; + synchronized (layoutLock) { + isFirstFrameRendered = false; + rotatedFrameWidth = 0; + rotatedFrameHeight = 0; + frameRotation = 0; + } + super.init(sharedContext, configAttributes, drawer); + } + + @Override + public void init(final EglBase.Context sharedContext, final int[] configAttributes, + RendererCommon.GlDrawer drawer) { + init(sharedContext, null /* rendererEvents */, configAttributes, drawer); + } + + /** + * Limit render framerate. + * + * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps + * reduction. + */ + @Override + public void setFpsReduction(float fps) { + synchronized (layoutLock) { + isRenderingPaused = fps == 0f; + } + super.setFpsReduction(fps); + } + + @Override + public void disableFpsReduction() { + synchronized (layoutLock) { + isRenderingPaused = false; + } + super.disableFpsReduction(); + } + + @Override + public void pauseVideo() { + synchronized (layoutLock) { + isRenderingPaused = true; + } + super.pauseVideo(); + } + + // VideoRenderer.Callbacks interface. + @Override + public void renderFrame(VideoRenderer.I420Frame frame) { + updateFrameDimensionsAndReportEvents(frame); + super.renderFrame(frame); + } + + // VideoSink interface. + @Override + public void onFrame(VideoFrame frame) { + updateFrameDimensionsAndReportEvents(frame); + super.onFrame(frame); + } + + // SurfaceHolder.Callback interface. + @Override + public void surfaceCreated(final SurfaceHolder holder) { + ThreadUtils.checkIsOnMainThread(); + createEglSurface(holder.getSurface()); + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + ThreadUtils.checkIsOnMainThread(); + final CountDownLatch completionLatch = new CountDownLatch(1); + releaseEglSurface(completionLatch::countDown); + ThreadUtils.awaitUninterruptibly(completionLatch); + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + ThreadUtils.checkIsOnMainThread(); + logD("surfaceChanged: format: " + format + " size: " + width + "x" + height); + } + + // Update frame dimensions and report any changes to |rendererEvents|. + private void updateFrameDimensionsAndReportEvents(VideoRenderer.I420Frame frame) { + synchronized (layoutLock) { + if (isRenderingPaused) { + return; + } + if (!isFirstFrameRendered) { + isFirstFrameRendered = true; + logD("Reporting first rendered frame."); + if (rendererEvents != null) { + rendererEvents.onFirstFrameRendered(); + } + } + if (rotatedFrameWidth != frame.rotatedWidth() || rotatedFrameHeight != frame.rotatedHeight() + || frameRotation != frame.rotationDegree) { + logD("Reporting frame resolution changed to " + frame.width + "x" + frame.height + + " with rotation " + frame.rotationDegree); + if (rendererEvents != null) { + rendererEvents.onFrameResolutionChanged(frame.width, frame.height, frame.rotationDegree); + } + rotatedFrameWidth = frame.rotatedWidth(); + rotatedFrameHeight = frame.rotatedHeight(); + frameRotation = frame.rotationDegree; + } + } + } + + // Update frame dimensions and report any changes to |rendererEvents|. + private void updateFrameDimensionsAndReportEvents(VideoFrame frame) { + synchronized (layoutLock) { + if (isRenderingPaused) { + return; + } + if (!isFirstFrameRendered) { + isFirstFrameRendered = true; + logD("Reporting first rendered frame."); + if (rendererEvents != null) { + rendererEvents.onFirstFrameRendered(); + } + } + if (rotatedFrameWidth != frame.getRotatedWidth() + || rotatedFrameHeight != frame.getRotatedHeight() + || frameRotation != frame.getRotation()) { + logD("Reporting frame resolution changed to " + frame.getBuffer().getWidth() + "x" + + frame.getBuffer().getHeight() + " with rotation " + frame.getRotation()); + if (rendererEvents != null) { + rendererEvents.onFrameResolutionChanged( + frame.getBuffer().getWidth(), frame.getBuffer().getHeight(), frame.getRotation()); + } + rotatedFrameWidth = frame.getRotatedWidth(); + rotatedFrameHeight = frame.getRotatedHeight(); + frameRotation = frame.getRotation(); + } + } + } + + private void logD(String string) { + Logging.d(TAG, name + ": " + string); + } +} diff --git a/sdk/android/api/org/webrtc/SurfaceViewRenderer.java b/sdk/android/api/org/webrtc/SurfaceViewRenderer.java index f664b014e1..a8eb57b8e3 100644 --- a/sdk/android/api/org/webrtc/SurfaceViewRenderer.java +++ b/sdk/android/api/org/webrtc/SurfaceViewRenderer.java @@ -13,41 +13,31 @@ package org.webrtc; import android.content.Context; import android.content.res.Resources.NotFoundException; import android.graphics.Point; +import android.os.Looper; import android.util.AttributeSet; import android.view.SurfaceHolder; import android.view.SurfaceView; -import java.util.concurrent.CountDownLatch; /** - * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on a SurfaceView. - * renderFrame() is asynchronous to avoid blocking the calling thread. - * This class is thread safe and handles access from potentially four different threads: - * Interaction from the main app in init, release, setMirror, and setScalingtype. - * Interaction from C++ rtc::VideoSinkInterface in renderFrame. - * Interaction from the Activity lifecycle in surfaceCreated, surfaceChanged, and surfaceDestroyed. - * Interaction with the layout framework in onMeasure and onSizeChanged. + * Display the video stream on a SurfaceView. */ -public class SurfaceViewRenderer - extends SurfaceView implements SurfaceHolder.Callback, VideoRenderer.Callbacks, VideoSink { +public class SurfaceViewRenderer extends SurfaceView implements SurfaceHolder.Callback, + VideoRenderer.Callbacks, VideoSink, + RendererCommon.RendererEvents { private static final String TAG = "SurfaceViewRenderer"; // Cached resource name. private final String resourceName; private final RendererCommon.VideoLayoutMeasure videoLayoutMeasure = new RendererCommon.VideoLayoutMeasure(); - private final EglRenderer eglRenderer; + private final SurfaceEglRenderer eglRenderer; // Callback for reporting renderer events. Read-only after initilization so no lock required. private RendererCommon.RendererEvents rendererEvents; - private final Object layoutLock = new Object(); - private boolean isRenderingPaused = false; - private boolean isFirstFrameRendered; + // Accessed only on the main thread. private int rotatedFrameWidth; private int rotatedFrameHeight; - private int frameRotation; - - // Accessed only on the main thread. private boolean enableFixedSize; private int surfaceWidth; private int surfaceHeight; @@ -58,8 +48,9 @@ public class SurfaceViewRenderer public SurfaceViewRenderer(Context context) { super(context); this.resourceName = getResourceName(); - eglRenderer = new EglRenderer(resourceName); + eglRenderer = new SurfaceEglRenderer(resourceName); getHolder().addCallback(this); + getHolder().addCallback(eglRenderer); } /** @@ -68,8 +59,9 @@ public class SurfaceViewRenderer public SurfaceViewRenderer(Context context, AttributeSet attrs) { super(context, attrs); this.resourceName = getResourceName(); - eglRenderer = new EglRenderer(resourceName); + eglRenderer = new SurfaceEglRenderer(resourceName); getHolder().addCallback(this); + getHolder().addCallback(eglRenderer); } /** @@ -91,13 +83,9 @@ public class SurfaceViewRenderer RendererCommon.GlDrawer drawer) { ThreadUtils.checkIsOnMainThread(); this.rendererEvents = rendererEvents; - synchronized (layoutLock) { - isFirstFrameRendered = false; - rotatedFrameWidth = 0; - rotatedFrameHeight = 0; - frameRotation = 0; - } - eglRenderer.init(sharedContext, configAttributes, drawer); + rotatedFrameWidth = 0; + rotatedFrameHeight = 0; + eglRenderer.init(sharedContext, this /* rendererEvents */, configAttributes, drawer); } /** @@ -181,37 +169,26 @@ public class SurfaceViewRenderer * reduction. */ public void setFpsReduction(float fps) { - synchronized (layoutLock) { - isRenderingPaused = fps == 0f; - } eglRenderer.setFpsReduction(fps); } public void disableFpsReduction() { - synchronized (layoutLock) { - isRenderingPaused = false; - } eglRenderer.disableFpsReduction(); } public void pauseVideo() { - synchronized (layoutLock) { - isRenderingPaused = true; - } eglRenderer.pauseVideo(); } // VideoRenderer.Callbacks interface. @Override public void renderFrame(VideoRenderer.I420Frame frame) { - updateFrameDimensionsAndReportEvents(frame); eglRenderer.renderFrame(frame); } // VideoSink interface. @Override public void onFrame(VideoFrame frame) { - updateFrameDimensionsAndReportEvents(frame); eglRenderer.onFrame(frame); } @@ -219,11 +196,8 @@ public class SurfaceViewRenderer @Override protected void onMeasure(int widthSpec, int heightSpec) { ThreadUtils.checkIsOnMainThread(); - final Point size; - synchronized (layoutLock) { - size = - videoLayoutMeasure.measure(widthSpec, heightSpec, rotatedFrameWidth, rotatedFrameHeight); - } + Point size = + videoLayoutMeasure.measure(widthSpec, heightSpec, rotatedFrameWidth, rotatedFrameHeight); setMeasuredDimension(size.x, size.y); logD("onMeasure(). New size: " + size.x + "x" + size.y); } @@ -237,35 +211,33 @@ public class SurfaceViewRenderer private void updateSurfaceSize() { ThreadUtils.checkIsOnMainThread(); - synchronized (layoutLock) { - if (enableFixedSize && rotatedFrameWidth != 0 && rotatedFrameHeight != 0 && getWidth() != 0 - && getHeight() != 0) { - final float layoutAspectRatio = getWidth() / (float) getHeight(); - final float frameAspectRatio = rotatedFrameWidth / (float) rotatedFrameHeight; - final int drawnFrameWidth; - final int drawnFrameHeight; - if (frameAspectRatio > layoutAspectRatio) { - drawnFrameWidth = (int) (rotatedFrameHeight * layoutAspectRatio); - drawnFrameHeight = rotatedFrameHeight; - } else { - drawnFrameWidth = rotatedFrameWidth; - drawnFrameHeight = (int) (rotatedFrameWidth / layoutAspectRatio); - } - // Aspect ratio of the drawn frame and the view is the same. - final int width = Math.min(getWidth(), drawnFrameWidth); - final int height = Math.min(getHeight(), drawnFrameHeight); - logD("updateSurfaceSize. Layout size: " + getWidth() + "x" + getHeight() + ", frame size: " - + rotatedFrameWidth + "x" + rotatedFrameHeight + ", requested surface size: " + width - + "x" + height + ", old surface size: " + surfaceWidth + "x" + surfaceHeight); - if (width != surfaceWidth || height != surfaceHeight) { - surfaceWidth = width; - surfaceHeight = height; - getHolder().setFixedSize(width, height); - } + if (enableFixedSize && rotatedFrameWidth != 0 && rotatedFrameHeight != 0 && getWidth() != 0 + && getHeight() != 0) { + final float layoutAspectRatio = getWidth() / (float) getHeight(); + final float frameAspectRatio = rotatedFrameWidth / (float) rotatedFrameHeight; + final int drawnFrameWidth; + final int drawnFrameHeight; + if (frameAspectRatio > layoutAspectRatio) { + drawnFrameWidth = (int) (rotatedFrameHeight * layoutAspectRatio); + drawnFrameHeight = rotatedFrameHeight; } else { - surfaceWidth = surfaceHeight = 0; - getHolder().setSizeFromLayout(); + drawnFrameWidth = rotatedFrameWidth; + drawnFrameHeight = (int) (rotatedFrameWidth / layoutAspectRatio); } + // Aspect ratio of the drawn frame and the view is the same. + final int width = Math.min(getWidth(), drawnFrameWidth); + final int height = Math.min(getHeight(), drawnFrameHeight); + logD("updateSurfaceSize. Layout size: " + getWidth() + "x" + getHeight() + ", frame size: " + + rotatedFrameWidth + "x" + rotatedFrameHeight + ", requested surface size: " + width + + "x" + height + ", old surface size: " + surfaceWidth + "x" + surfaceHeight); + if (width != surfaceWidth || height != surfaceHeight) { + surfaceWidth = width; + surfaceHeight = height; + getHolder().setFixedSize(width, height); + } + } else { + surfaceWidth = surfaceHeight = 0; + getHolder().setSizeFromLayout(); } } @@ -273,33 +245,19 @@ public class SurfaceViewRenderer @Override public void surfaceCreated(final SurfaceHolder holder) { ThreadUtils.checkIsOnMainThread(); - eglRenderer.createEglSurface(holder.getSurface()); surfaceWidth = surfaceHeight = 0; updateSurfaceSize(); } @Override - public void surfaceDestroyed(SurfaceHolder holder) { - ThreadUtils.checkIsOnMainThread(); - final CountDownLatch completionLatch = new CountDownLatch(1); - eglRenderer.releaseEglSurface(new Runnable() { - @Override - public void run() { - completionLatch.countDown(); - } - }); - ThreadUtils.awaitUninterruptibly(completionLatch); - } + public void surfaceDestroyed(SurfaceHolder holder) {} @Override - public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { - ThreadUtils.checkIsOnMainThread(); - logD("surfaceChanged: format: " + format + " size: " + width + "x" + height); - } + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {} private String getResourceName() { try { - return getResources().getResourceEntryName(getId()) + ": "; + return getResources().getResourceEntryName(getId()); } catch (NotFoundException e) { return ""; } @@ -312,74 +270,38 @@ public class SurfaceViewRenderer eglRenderer.clearImage(); } - // Update frame dimensions and report any changes to |rendererEvents|. - private void updateFrameDimensionsAndReportEvents(VideoRenderer.I420Frame frame) { - synchronized (layoutLock) { - if (isRenderingPaused) { - return; - } - if (!isFirstFrameRendered) { - isFirstFrameRendered = true; - logD("Reporting first rendered frame."); - if (rendererEvents != null) { - rendererEvents.onFirstFrameRendered(); - } - } - if (rotatedFrameWidth != frame.rotatedWidth() || rotatedFrameHeight != frame.rotatedHeight() - || frameRotation != frame.rotationDegree) { - logD("Reporting frame resolution changed to " + frame.width + "x" + frame.height - + " with rotation " + frame.rotationDegree); - if (rendererEvents != null) { - rendererEvents.onFrameResolutionChanged(frame.width, frame.height, frame.rotationDegree); - } - rotatedFrameWidth = frame.rotatedWidth(); - rotatedFrameHeight = frame.rotatedHeight(); - frameRotation = frame.rotationDegree; - post(new Runnable() { - @Override - public void run() { - updateSurfaceSize(); - requestLayout(); - } - }); - } + @Override + public void onFirstFrameRendered() { + if (rendererEvents != null) { + rendererEvents.onFirstFrameRendered(); } } - // Update frame dimensions and report any changes to |rendererEvents|. - private void updateFrameDimensionsAndReportEvents(VideoFrame frame) { - synchronized (layoutLock) { - if (isRenderingPaused) { - return; - } - if (!isFirstFrameRendered) { - isFirstFrameRendered = true; - logD("Reporting first rendered frame."); - if (rendererEvents != null) { - rendererEvents.onFirstFrameRendered(); - } - } - if (rotatedFrameWidth != frame.getRotatedWidth() - || rotatedFrameHeight != frame.getRotatedHeight() - || frameRotation != frame.getRotation()) { - logD("Reporting frame resolution changed to " + frame.getBuffer().getWidth() + "x" - + frame.getBuffer().getHeight() + " with rotation " + frame.getRotation()); - if (rendererEvents != null) { - rendererEvents.onFrameResolutionChanged( - frame.getBuffer().getWidth(), frame.getBuffer().getHeight(), frame.getRotation()); - } - rotatedFrameWidth = frame.getRotatedWidth(); - rotatedFrameHeight = frame.getRotatedHeight(); - frameRotation = frame.getRotation(); - post(() -> { - updateSurfaceSize(); - requestLayout(); - }); - } + @Override + public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation) { + if (rendererEvents != null) { + rendererEvents.onFrameResolutionChanged(videoWidth, videoHeight, rotation); + } + int rotatedWidth = rotation == 0 || rotation == 180 ? videoWidth : videoHeight; + int rotatedHeight = rotation == 0 || rotation == 180 ? videoHeight : videoWidth; + // run immediately if possible for ui thread tests + postOrRun(() -> { + rotatedFrameWidth = rotatedWidth; + rotatedFrameHeight = rotatedHeight; + updateSurfaceSize(); + requestLayout(); + }); + } + + private void postOrRun(Runnable r) { + if (Thread.currentThread() == Looper.getMainLooper().getThread()) { + r.run(); + } else { + post(r); } } private void logD(String string) { - Logging.d(TAG, resourceName + string); + Logging.d(TAG, resourceName + ": " + string); } }