From dd0c7f0ea4a432135d06beef40e20560533cb600 Mon Sep 17 00:00:00 2001 From: sakal Date: Mon, 19 Sep 2016 04:37:16 -0700 Subject: [PATCH] Events refactor in Android session based capturing. In the new implementation session reports all events through CameraSession.Events interface. CameraCapturer passes these events forward. BUG=webrtc:6325 Review-Url: https://codereview.webrtc.org/2331343010 Cr-Commit-Position: refs/heads/master@{#14286} --- .../java/src/org/webrtc/Camera2Capturer.java | 21 +--- .../java/src/org/webrtc/Camera2Session.java | 74 +++++------- .../java/src/org/webrtc/CameraCapturer.java | 107 +++++++++++++++++- .../java/src/org/webrtc/CameraSession.java | 12 ++ .../src/org/webrtc/CameraVideoCapturer.java | 1 - 5 files changed, 150 insertions(+), 65 deletions(-) diff --git a/webrtc/api/android/java/src/org/webrtc/Camera2Capturer.java b/webrtc/api/android/java/src/org/webrtc/Camera2Capturer.java index b055f8bb83..78d1f04a43 100644 --- a/webrtc/api/android/java/src/org/webrtc/Camera2Capturer.java +++ b/webrtc/api/android/java/src/org/webrtc/Camera2Capturer.java @@ -10,14 +10,9 @@ package org.webrtc; -import org.webrtc.CameraEnumerationAndroid.CaptureFormat; - import android.annotation.TargetApi; import android.content.Context; import android.hardware.camera2.CameraManager; -import android.os.Handler; - -import java.util.List; @TargetApi(21) public class Camera2Capturer extends CameraCapturer { @@ -33,17 +28,13 @@ public class Camera2Capturer extends CameraCapturer { @Override protected void createCameraSession( - CameraSession.CreateSessionCallback createSessionCallback, - CameraEventsHandler eventsHandler, Context applicationContext, - CameraVideoCapturer.CapturerObserver capturerObserver, - SurfaceTextureHelper surfaceTextureHelper, + CameraSession.CreateSessionCallback createSessionCallback, CameraSession.Events events, + Context applicationContext, SurfaceTextureHelper surfaceTextureHelper, String cameraName, int width, int height, int framerate) { Camera2Session.create( - cameraManager, - createSessionCallback, - eventsHandler, applicationContext, - capturerObserver, - surfaceTextureHelper, - cameraName, width, height, framerate); + createSessionCallback, events, + applicationContext, cameraManager, + surfaceTextureHelper, + cameraName, width, height, framerate); } } diff --git a/webrtc/api/android/java/src/org/webrtc/Camera2Session.java b/webrtc/api/android/java/src/org/webrtc/Camera2Session.java index 1125204358..35a4862d13 100644 --- a/webrtc/api/android/java/src/org/webrtc/Camera2Session.java +++ b/webrtc/api/android/java/src/org/webrtc/Camera2Session.java @@ -46,11 +46,10 @@ public class Camera2Session implements CameraSession { private static enum SessionState { RUNNING, STOPPED }; private final Handler cameraThreadHandler; - private final CameraManager cameraManager; private final CreateSessionCallback callback; - private final CameraVideoCapturer.CameraEventsHandler eventsHandler; + private final Events events; private final Context applicationContext; - private final CameraVideoCapturer.CapturerObserver capturerObserver; + private final CameraManager cameraManager; private final SurfaceTextureHelper surfaceTextureHelper; private final String cameraId; private final int width; @@ -70,7 +69,6 @@ public class Camera2Session implements CameraSession { // Initialized when capture session is created private CameraCaptureSession captureSession; - private CameraVideoCapturer.CameraStatistics cameraStatistics; // State private SessionState state = SessionState.RUNNING; @@ -134,7 +132,7 @@ public class Camera2Session implements CameraSession { checkIsOnCameraThread(); Logging.d(TAG, "Camera device closed."); - eventsHandler.onCameraClosed(); + events.onCameraClosed(Camera2Session.this); } } @@ -192,7 +190,6 @@ public class Camera2Session implements CameraSession { } if (!firstFrameReported) { - eventsHandler.onFirstFrameAvailable(); firstFrameReported = true; final int startTimeMs = (int) TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - constructionTimeNs); @@ -211,14 +208,10 @@ public class Camera2Session implements CameraSession { transformMatrix = RendererCommon.rotateTextureMatrix( transformMatrix, -cameraOrientation); - cameraStatistics.addFrame(); - capturerObserver.onTextureFrameCaptured(captureFormat.width, captureFormat.height, - oesTextureId, transformMatrix, rotation, timestampNs); + events.onTextureFrameCaptured(Camera2Session.this, captureFormat.width, + captureFormat.height, oesTextureId, transformMatrix, rotation, timestampNs); } }); - capturerObserver.onCapturerStarted(true /* success */); - cameraStatistics = new CameraVideoCapturer.CameraStatistics( - surfaceTextureHelper, eventsHandler); Logging.d(TAG, "Camera device successfully started."); callback.onDone(Camera2Session.this); } @@ -266,23 +259,20 @@ public class Camera2Session implements CameraSession { } public static void create( - CameraManager cameraManager, CreateSessionCallback callback, - CameraVideoCapturer.CameraEventsHandler eventsHandler, Context applicationContext, - CameraVideoCapturer.CapturerObserver capturerObserver, + CreateSessionCallback callback, Events events, + Context applicationContext, CameraManager cameraManager, SurfaceTextureHelper surfaceTextureHelper, String cameraId, int width, int height, int framerate) { new Camera2Session( - cameraManager, callback, - eventsHandler, applicationContext, - capturerObserver, + callback, events, + applicationContext, cameraManager, surfaceTextureHelper, cameraId, width, height, framerate); } private Camera2Session( - CameraManager cameraManager, CreateSessionCallback callback, - CameraVideoCapturer.CameraEventsHandler eventsHandler, Context applicationContext, - CameraVideoCapturer.CapturerObserver capturerObserver, + CreateSessionCallback callback, Events events, + Context applicationContext, CameraManager cameraManager, SurfaceTextureHelper surfaceTextureHelper, String cameraId, int width, int height, int framerate) { Logging.d(TAG, "Create new camera2 session on camera " + cameraId); @@ -290,11 +280,10 @@ public class Camera2Session implements CameraSession { constructionTimeNs = System.nanoTime(); this.cameraThreadHandler = new Handler(); - this.cameraManager = cameraManager; this.callback = callback; - this.eventsHandler = eventsHandler; + this.events = events; this.applicationContext = applicationContext; - this.capturerObserver = capturerObserver; + this.cameraManager = cameraManager; this.surfaceTextureHelper = surfaceTextureHelper; this.cameraId = cameraId; this.width = width; @@ -351,7 +340,7 @@ public class Camera2Session implements CameraSession { checkIsOnCameraThread(); Logging.d(TAG, "Opening camera " + cameraId); - eventsHandler.onCameraOpening(cameraId); + events.onCameraOpening(); try { cameraManager.openCamera(cameraId, new CameraStateCallback(), cameraThreadHandler); @@ -367,7 +356,6 @@ public class Camera2Session implements CameraSession { if (Thread.currentThread() == cameraThreadHandler.getLooper().getThread()) { if (state != SessionState.STOPPED) { state = SessionState.STOPPED; - capturerObserver.onCapturerStopped(); // Post the stopInternal to return earlier. cameraThreadHandler.post(new Runnable() { @Override @@ -387,7 +375,6 @@ public class Camera2Session implements CameraSession { public void run() { if (state != SessionState.STOPPED) { state = SessionState.STOPPED; - capturerObserver.onCapturerStopped(); stopLatch.countDown(); stopInternal(); final int stopTimeMs = @@ -406,14 +393,19 @@ public class Camera2Session implements CameraSession { checkIsOnCameraThread(); surfaceTextureHelper.stopListening(); - cameraStatistics.release(); - captureSession.close(); - captureSession = null; - surface.release(); - surface = null; - cameraDevice.close(); - cameraDevice = null; + if (captureSession != null) { + captureSession.close(); + captureSession = null; + } + if (surface != null) { + surface.release(); + surface = null; + } + if (cameraDevice != null) { + cameraDevice.close(); + cameraDevice = null; + } Logging.d(TAG, "Stop done"); } @@ -422,17 +414,13 @@ public class Camera2Session implements CameraSession { checkIsOnCameraThread(); Logging.e(TAG, "Error: " + error); - if (captureSession == null) { - if (cameraDevice != null) { - cameraDevice.close(); - cameraDevice = null; - } - - state = SessionState.STOPPED; + final boolean startFailure = (captureSession == null); + state = SessionState.STOPPED; + stopInternal(); + if (startFailure) { callback.onFailure(error); - capturerObserver.onCapturerStarted(false /* success */); } else { - eventsHandler.onCameraError(error); + events.onCameraError(this, error); } } diff --git a/webrtc/api/android/java/src/org/webrtc/CameraCapturer.java b/webrtc/api/android/java/src/org/webrtc/CameraCapturer.java index 8724da0785..be57a9bd7f 100644 --- a/webrtc/api/android/java/src/org/webrtc/CameraCapturer.java +++ b/webrtc/api/android/java/src/org/webrtc/CameraCapturer.java @@ -37,8 +37,10 @@ public abstract class CameraCapturer implements CameraVideoCapturer { new CameraSession.CreateSessionCallback() { @Override public void onDone(CameraSession session) { + checkIsOnCameraThread(); Logging.d(TAG, "Create session done"); uiThreadHandler.removeCallbacks(openCameraTimeoutRunnable); + capturerObserver.onCapturerStarted(true /* success */); synchronized (stateLock) { sessionOpening = false; currentSession = session; @@ -55,12 +57,17 @@ public abstract class CameraCapturer implements CameraVideoCapturer { switchState = SwitchState.IDLE; switchCameraInternal(switchEventsHandler); } + + cameraStatistics = new CameraStatistics(surfaceHelper, eventsHandler); + firstFrameObserved = false; } } @Override public void onFailure(String error) { + checkIsOnCameraThread(); uiThreadHandler.removeCallbacks(openCameraTimeoutRunnable); + capturerObserver.onCapturerStarted(false /* success */); synchronized (stateLock) { openAttemptsRemaining--; @@ -87,6 +94,84 @@ public abstract class CameraCapturer implements CameraVideoCapturer { } }; + private final CameraSession.Events cameraSessionEventsHandler = new CameraSession.Events() { + @Override + public void onCameraOpening() { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (currentSession != null) { + Logging.w(TAG, "onCameraOpening while session was open."); + return; + } + eventsHandler.onCameraOpening(cameraName); + } + } + + @Override + public void onCameraError(CameraSession session, String error) { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (session != currentSession) { + Logging.w(TAG, "onCameraError from another session: " + error); + return; + } + eventsHandler.onCameraError(error); + stopCapture(); + } + } + + @Override + public void onCameraClosed(CameraSession session) { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (session != currentSession && currentSession != null) { + Logging.d(TAG, "onCameraClosed from another session."); + return; + } + eventsHandler.onCameraClosed(); + } + } + + @Override + public void onByteBufferFrameCaptured( + CameraSession session, byte[] data, int width, int height, int rotation, + long timestamp) { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (session != currentSession) { + Logging.w(TAG, "onByteBufferFrameCaptured from another session."); + return; + } + if (!firstFrameObserved) { + eventsHandler.onFirstFrameAvailable(); + firstFrameObserved = true; + } + cameraStatistics.addFrame(); + capturerObserver.onByteBufferFrameCaptured(data, width, height, rotation, timestamp); + } + } + + @Override + public void onTextureFrameCaptured( + CameraSession session, int width, int height, int oesTextureId, float[] transformMatrix, + int rotation, long timestamp) { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (session != currentSession) { + Logging.w(TAG, "onTextureFrameCaptured from another session."); + return; + } + if (!firstFrameObserved) { + eventsHandler.onFirstFrameAvailable(); + firstFrameObserved = true; + } + cameraStatistics.addFrame(); + capturerObserver.onTextureFrameCaptured( + width, height, oesTextureId, transformMatrix, rotation, timestamp); + } + } + }; + private final Runnable openCameraTimeoutRunnable = new Runnable() { @Override public void run() { @@ -111,6 +196,9 @@ public abstract class CameraCapturer implements CameraVideoCapturer { private int openAttemptsRemaining; /* guarded by stateLock */ private SwitchState switchState = SwitchState.IDLE; /* guarded by stateLock */ private CameraSwitchHandler switchEventsHandler; /* guarded by stateLock */ + // Valid from onDone call until stopCapture, otherwise null. + private CameraStatistics cameraStatistics; /* guarded by stateLock */ + private boolean firstFrameObserved; /* guarded by stateLock */ public CameraCapturer( String cameraName, CameraEventsHandler eventsHandler, CameraEnumerator cameraEnumerator) { @@ -181,8 +269,7 @@ public abstract class CameraCapturer implements CameraVideoCapturer { @Override public void run() { createCameraSession( - createSessionCallback, - eventsHandler, applicationContext, capturerObserver, surfaceHelper, + createSessionCallback, cameraSessionEventsHandler, applicationContext, surfaceHelper, cameraName, width, height, framerate); } }, delayMs); @@ -200,8 +287,11 @@ public abstract class CameraCapturer implements CameraVideoCapturer { if (currentSession != null) { Logging.d(TAG, "Stop capture: Stopping session"); + cameraStatistics.release(); + cameraStatistics = null; currentSession.stop(); currentSession = null; + capturerObserver.onCapturerStopped(); } else { Logging.d(TAG, "Stop capture: No session open"); } @@ -319,6 +409,13 @@ public abstract class CameraCapturer implements CameraVideoCapturer { Logging.d(TAG, "switchCamera done"); } + private void checkIsOnCameraThread() { + if (Thread.currentThread() != cameraThreadHandler.getLooper().getThread()) { + Logging.e(TAG, "Check is on camera thread failed."); + throw new RuntimeException("Not on camera thread."); + } + } + protected String getCameraName() { synchronized (stateLock) { return cameraName; @@ -326,9 +423,7 @@ public abstract class CameraCapturer implements CameraVideoCapturer { } abstract protected void createCameraSession( - CameraSession.CreateSessionCallback createSessionCallback, - CameraEventsHandler eventsHandler, Context applicationContext, - CameraVideoCapturer.CapturerObserver capturerObserver, - SurfaceTextureHelper surfaceTextureHelper, + CameraSession.CreateSessionCallback createSessionCallback, CameraSession.Events events, + Context applicationContext, SurfaceTextureHelper surfaceTextureHelper, String cameraName, int width, int height, int framerate); } diff --git a/webrtc/api/android/java/src/org/webrtc/CameraSession.java b/webrtc/api/android/java/src/org/webrtc/CameraSession.java index c6b73a8bf1..bad623fc1d 100644 --- a/webrtc/api/android/java/src/org/webrtc/CameraSession.java +++ b/webrtc/api/android/java/src/org/webrtc/CameraSession.java @@ -11,11 +11,23 @@ package org.webrtc; public interface CameraSession { + // Callbacks are fired on the camera thread. public interface CreateSessionCallback { void onDone(CameraSession session); void onFailure(String error); } + // Events are fired on the camera thread. + public interface Events { + void onCameraOpening(); + void onCameraError(CameraSession session, String error); + void onCameraClosed(CameraSession session); + void onByteBufferFrameCaptured(CameraSession session, byte[] data, int width, int height, + int rotation, long timestamp); + void onTextureFrameCaptured(CameraSession session, int width, int height, int oesTextureId, + float[] transformMatrix, int rotation, long timestamp); + } + /** * Stops the capture. Waits until no more calls to capture observer will be made. * If waitCameraStop is true, also waits for the camera to stop. diff --git a/webrtc/api/android/java/src/org/webrtc/CameraVideoCapturer.java b/webrtc/api/android/java/src/org/webrtc/CameraVideoCapturer.java index 43513055fa..a53935621f 100644 --- a/webrtc/api/android/java/src/org/webrtc/CameraVideoCapturer.java +++ b/webrtc/api/android/java/src/org/webrtc/CameraVideoCapturer.java @@ -121,7 +121,6 @@ public interface CameraVideoCapturer extends VideoCapturer { } public void release() { - checkThread(); surfaceTextureHelper.getHandler().removeCallbacks(cameraObserver); } }