diff --git a/talk/app/webrtc/androidtests/AndroidManifest.xml b/talk/app/webrtc/androidtests/AndroidManifest.xml index 3bcd99b425..a22611f292 100644 --- a/talk/app/webrtc/androidtests/AndroidManifest.xml +++ b/talk/app/webrtc/androidtests/AndroidManifest.xml @@ -7,7 +7,7 @@ - + diff --git a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java index 0fe827d1f9..e0eccb1f76 100644 --- a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java +++ b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java @@ -26,148 +26,18 @@ */ package org.webrtc; -import android.hardware.Camera; +import android.opengl.EGL14; import android.test.ActivityTestCase; -import android.test.suitebuilder.annotation.SmallTest; import android.test.suitebuilder.annotation.MediumTest; +import android.test.suitebuilder.annotation.SmallTest; import android.util.Size; import org.webrtc.CameraEnumerationAndroid.CaptureFormat; -import org.webrtc.VideoRenderer.I420Frame; - -import java.util.ArrayList; import java.util.HashSet; -import java.util.List; import java.util.Set; -import java.util.concurrent.CountDownLatch; @SuppressWarnings("deprecation") public class VideoCapturerAndroidTest extends ActivityTestCase { - static class RendererCallbacks implements VideoRenderer.Callbacks { - private int framesRendered = 0; - private Object frameLock = 0; - - @Override - public void renderFrame(I420Frame frame) { - synchronized (frameLock) { - ++framesRendered; - frameLock.notify(); - } - VideoRenderer.renderFrameDone(frame); - } - - public int WaitForNextFrameToRender() throws InterruptedException { - synchronized (frameLock) { - frameLock.wait(); - return framesRendered; - } - } - } - - static class FakeAsyncRenderer implements VideoRenderer.Callbacks { - private final List pendingFrames = new ArrayList(); - - @Override - public void renderFrame(I420Frame frame) { - synchronized (pendingFrames) { - pendingFrames.add(frame); - pendingFrames.notifyAll(); - } - } - - // Wait until at least one frame have been received, before returning them. - public List waitForPendingFrames() throws InterruptedException { - synchronized (pendingFrames) { - while (pendingFrames.isEmpty()) { - pendingFrames.wait(); - } - return new ArrayList(pendingFrames); - } - } - } - - static class FakeCapturerObserver implements - VideoCapturerAndroid.CapturerObserver { - private int framesCaptured = 0; - private int frameSize = 0; - private Object frameLock = 0; - private Object capturerStartLock = 0; - private boolean captureStartResult = false; - private List timestamps = new ArrayList(); - - @Override - public void OnCapturerStarted(boolean success) { - synchronized (capturerStartLock) { - captureStartResult = success; - capturerStartLock.notify(); - } - } - - @Override - public void OnFrameCaptured(byte[] frame, int length, int width, int height, - int rotation, long timeStamp) { - synchronized (frameLock) { - ++framesCaptured; - frameSize = length; - timestamps.add(timeStamp); - frameLock.notify(); - } - } - - @Override - public void OnOutputFormatRequest(int width, int height, int fps) {} - - public boolean WaitForCapturerToStart() throws InterruptedException { - synchronized (capturerStartLock) { - capturerStartLock.wait(); - return captureStartResult; - } - } - - public int WaitForNextCapturedFrame() throws InterruptedException { - synchronized (frameLock) { - frameLock.wait(); - return framesCaptured; - } - } - - int frameSize() { - synchronized (frameLock) { - return frameSize; - } - } - - List getCopyAndResetListOftimeStamps() { - synchronized (frameLock) { - ArrayList list = new ArrayList(timestamps); - timestamps.clear(); - return list; - } - } - } - - // Return true if the device under test have at least two cameras. - @SuppressWarnings("deprecation") - boolean HaveTwoCameras() { - return (Camera.getNumberOfCameras() >= 2); - } - - void startCapturerAndRender(String deviceName) throws InterruptedException { - PeerConnectionFactory factory = new PeerConnectionFactory(); - VideoCapturerAndroid capturer = - VideoCapturerAndroid.create(deviceName, null); - VideoSource source = - factory.createVideoSource(capturer, new MediaConstraints()); - VideoTrack track = factory.createVideoTrack("dummy", source); - RendererCallbacks callbacks = new RendererCallbacks(); - track.addRenderer(new VideoRenderer(callbacks)); - assertTrue(callbacks.WaitForNextFrameToRender() > 0); - track.dispose(); - source.dispose(); - factory.dispose(); - assertTrue(capturer.isReleased()); - } - @Override protected void setUp() { assertTrue(PeerConnectionFactory.initializeAndroidGlobals( @@ -206,15 +76,18 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { } @SmallTest - public void testCreateAndRelease() throws Exception { - VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null); - assertNotNull(capturer); - capturer.dispose(); - assertTrue(capturer.isReleased()); + public void testCreateAndRelease() { + VideoCapturerAndroidTestFixtures.release(VideoCapturerAndroid.create("", null)); } @SmallTest - public void testCreateNonExistingCamera() throws Exception { + public void testCreateAndReleaseUsingTextures() { + VideoCapturerAndroidTestFixtures.release( + VideoCapturerAndroid.create("", null, EGL14.EGL_NO_CONTEXT)); + } + + @SmallTest + public void testCreateNonExistingCamera() { VideoCapturerAndroid capturer = VideoCapturerAndroid.create( "non-existing camera", null); assertNull(capturer); @@ -224,195 +97,135 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { // This test that the camera can be started and that the frames are forwarded // to a Java video renderer using a "default" capturer. // It tests both the Java and the C++ layer. - public void testStartVideoCapturer() throws Exception { - startCapturerAndRender(""); + public void testStartVideoCapturer() throws InterruptedException { + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create("", null); + VideoCapturerAndroidTestFixtures.startCapturerAndRender(capturer); } + /* TODO(perkj): Enable once VideoCapture to texture support has landed in C++. + @SmallTest + public void testStartVideoCapturerUsingTextures() throws InterruptedException { + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create("", null, EGL14.EGL_NO_CONTEXT); + VideoCapturerAndroidTestFixtures.startCapturerAndRender(capturer); + }*/ + @SmallTest // This test that the camera can be started and that the frames are forwarded // to a Java video renderer using the front facing video capturer. // It tests both the Java and the C++ layer. - public void testStartFrontFacingVideoCapturer() throws Exception { - startCapturerAndRender(CameraEnumerationAndroid.getNameOfFrontFacingDevice()); + public void testStartFrontFacingVideoCapturer() throws InterruptedException { + String deviceName = CameraEnumerationAndroid.getNameOfFrontFacingDevice(); + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create(deviceName, null); + VideoCapturerAndroidTestFixtures.startCapturerAndRender(capturer); } @SmallTest // This test that the camera can be started and that the frames are forwarded // to a Java video renderer using the back facing video capturer. // It tests both the Java and the C++ layer. - public void testStartBackFacingVideoCapturer() throws Exception { - if (!HaveTwoCameras()) { + public void testStartBackFacingVideoCapturer() throws InterruptedException { + if (!VideoCapturerAndroidTestFixtures.HaveTwoCameras()) { return; } - startCapturerAndRender(CameraEnumerationAndroid.getNameOfBackFacingDevice()); + + String deviceName = CameraEnumerationAndroid.getNameOfBackFacingDevice(); + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create(deviceName, null); + VideoCapturerAndroidTestFixtures.startCapturerAndRender(capturer); } @SmallTest // This test that the default camera can be started and that the camera can // later be switched to another camera. // It tests both the Java and the C++ layer. - public void testSwitchVideoCapturer() throws Exception { - PeerConnectionFactory factory = new PeerConnectionFactory(); + public void testSwitchVideoCapturer() throws InterruptedException { VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null); - VideoSource source = - factory.createVideoSource(capturer, new MediaConstraints()); - VideoTrack track = factory.createVideoTrack("dummy", source); - - // Array with one element to avoid final problem in nested classes. - final boolean[] cameraSwitchSuccessful = new boolean[1]; - final CountDownLatch barrier = new CountDownLatch(1); - capturer.switchCamera(new VideoCapturerAndroid.CameraSwitchHandler() { - @Override - public void onCameraSwitchDone(boolean isFrontCamera) { - cameraSwitchSuccessful[0] = true; - barrier.countDown(); - } - @Override - public void onCameraSwitchError(String errorDescription) { - cameraSwitchSuccessful[0] = false; - barrier.countDown(); - } - }); - // Wait until the camera has been switched. - barrier.await(); - - // Check result. - if (HaveTwoCameras()) { - assertTrue(cameraSwitchSuccessful[0]); - } else { - assertFalse(cameraSwitchSuccessful[0]); - } - // Ensure that frames are received. - RendererCallbacks callbacks = new RendererCallbacks(); - track.addRenderer(new VideoRenderer(callbacks)); - assertTrue(callbacks.WaitForNextFrameToRender() > 0); - track.dispose(); - source.dispose(); - factory.dispose(); - assertTrue(capturer.isReleased()); + VideoCapturerAndroidTestFixtures.switchCamera(capturer); } + /* TODO(perkj): Enable once VideoCapture to texture support has landed in C++. + @SmallTest + public void testSwitchVideoCapturerUsingTextures() throws InterruptedException { + VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null, EGL14.EGL_NO_CONTEXT); + VideoCapturerAndroidTestFixtures.switchCamera(capturer); + }*/ + @MediumTest // Test what happens when attempting to call e.g. switchCamera() after camera has been stopped. public void testCameraCallsAfterStop() throws InterruptedException { final String deviceName = CameraEnumerationAndroid.getDeviceName(0); final VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName, null); - final List formats = CameraEnumerationAndroid.getSupportedFormats(0); - final CameraEnumerationAndroid.CaptureFormat format = formats.get(0); - final FakeCapturerObserver observer = new FakeCapturerObserver(); - capturer.startCapture(format.width, format.height, format.maxFramerate, - getInstrumentation().getContext(), observer); - // Make sure camera is started and then stop it. - assertTrue(observer.WaitForCapturerToStart()); - capturer.stopCapture(); - for (long timeStamp : observer.getCopyAndResetListOftimeStamps()) { - capturer.returnBuffer(timeStamp); - } - // We can't change |capturer| at this point, but we should not crash. - capturer.switchCamera(null); - capturer.onOutputFormatRequest(640, 480, 15); - capturer.changeCaptureFormat(640, 480, 15); + VideoCapturerAndroidTestFixtures.cameraCallsAfterStop(capturer, + getInstrumentation().getContext()); + } - capturer.dispose(); - assertTrue(capturer.isReleased()); + @MediumTest + public void testCameraCallsAfterStopUsingTextures() throws InterruptedException { + final String deviceName = CameraEnumerationAndroid.getDeviceName(0); + final VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName, null, + EGL14.EGL_NO_CONTEXT); + + VideoCapturerAndroidTestFixtures.cameraCallsAfterStop(capturer, + getInstrumentation().getContext()); } @SmallTest // This test that the VideoSource that the VideoCapturer is connected to can // be stopped and restarted. It tests both the Java and the C++ layer. - public void testStopRestartVideoSource() throws Exception { - PeerConnectionFactory factory = new PeerConnectionFactory(); + public void testStopRestartVideoSource() throws InterruptedException { VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null); - VideoSource source = - factory.createVideoSource(capturer, new MediaConstraints()); - VideoTrack track = factory.createVideoTrack("dummy", source); - RendererCallbacks callbacks = new RendererCallbacks(); - track.addRenderer(new VideoRenderer(callbacks)); - assertTrue(callbacks.WaitForNextFrameToRender() > 0); - assertEquals(MediaSource.State.LIVE, source.state()); - - source.stop(); - assertEquals(MediaSource.State.ENDED, source.state()); - - source.restart(); - assertTrue(callbacks.WaitForNextFrameToRender() > 0); - assertEquals(MediaSource.State.LIVE, source.state()); - track.dispose(); - source.dispose(); - factory.dispose(); - assertTrue(capturer.isReleased()); + VideoCapturerAndroidTestFixtures.stopRestartVideoSource(capturer); } + /* TODO(perkj): Enable once VideoCapture to texture support has landed in C++. + @SmallTest + public void testStopRestartVideoSourceUsingTextures() throws InterruptedException { + VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null, EGL14.EGL_NO_CONTEXT); + VideoCapturerAndroidTestFixtures.stopRestartVideoSource(capturer); + }*/ + @SmallTest // This test that the camera can be started at different resolutions. // It does not test or use the C++ layer. - public void testStartStopWithDifferentResolutions() throws Exception { - FakeCapturerObserver observer = new FakeCapturerObserver(); - + public void testStartStopWithDifferentResolutions() throws InterruptedException { String deviceName = CameraEnumerationAndroid.getDeviceName(0); - List formats = CameraEnumerationAndroid.getSupportedFormats(0); VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName, null); + VideoCapturerAndroidTestFixtures.startStopWithDifferentResolutions(capturer, + getInstrumentation().getContext()); + } - for(int i = 0; i < 3 ; ++i) { - CameraEnumerationAndroid.CaptureFormat format = formats.get(i); - capturer.startCapture(format.width, format.height, format.maxFramerate, - getInstrumentation().getContext(), observer); - assertTrue(observer.WaitForCapturerToStart()); - observer.WaitForNextCapturedFrame(); - // Check the frame size. - assertEquals(format.frameSize(), observer.frameSize()); - capturer.stopCapture(); - for (long timestamp : observer.getCopyAndResetListOftimeStamps()) { - capturer.returnBuffer(timestamp); - } - } - capturer.dispose(); - assertTrue(capturer.isReleased()); + @SmallTest + public void testStartStopWithDifferentResolutionsUsingTextures() throws InterruptedException { + String deviceName = CameraEnumerationAndroid.getDeviceName(0); + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create(deviceName, null, EGL14.EGL_NO_CONTEXT); + VideoCapturerAndroidTestFixtures.startStopWithDifferentResolutions(capturer, + getInstrumentation().getContext()); } @SmallTest // This test what happens if buffers are returned after the capturer have // been stopped and restarted. It does not test or use the C++ layer. - public void testReturnBufferLate() throws Exception { - FakeCapturerObserver observer = new FakeCapturerObserver(); - + public void testReturnBufferLate() throws InterruptedException { String deviceName = CameraEnumerationAndroid.getDeviceName(0); - List formats = CameraEnumerationAndroid.getSupportedFormats(0); VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName, null); + VideoCapturerAndroidTestFixtures.returnBufferLate(capturer, + getInstrumentation().getContext()); + } - CameraEnumerationAndroid.CaptureFormat format = formats.get(0); - capturer.startCapture(format.width, format.height, format.maxFramerate, - getInstrumentation().getContext(), observer); - assertTrue(observer.WaitForCapturerToStart()); - - observer.WaitForNextCapturedFrame(); - capturer.stopCapture(); - List listOftimestamps = observer.getCopyAndResetListOftimeStamps(); - assertTrue(listOftimestamps.size() >= 1); - - format = formats.get(1); - capturer.startCapture(format.width, format.height, format.maxFramerate, - getInstrumentation().getContext(), observer); - observer.WaitForCapturerToStart(); - observer.WaitForNextCapturedFrame(); - - for (Long timeStamp : listOftimestamps) { - capturer.returnBuffer(timeStamp); - } - - observer.WaitForNextCapturedFrame(); - capturer.stopCapture(); - - listOftimestamps = observer.getCopyAndResetListOftimeStamps(); - assertTrue(listOftimestamps.size() >= 2); - for (Long timeStamp : listOftimestamps) { - capturer.returnBuffer(timeStamp); - } - capturer.dispose(); - assertTrue(capturer.isReleased()); + @SmallTest + public void testReturnBufferLateUsingTextures() throws InterruptedException { + String deviceName = CameraEnumerationAndroid.getDeviceName(0); + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create(deviceName, null, EGL14.EGL_NO_CONTEXT); + VideoCapturerAndroidTestFixtures.returnBufferLate(capturer, + getInstrumentation().getContext()); } @MediumTest @@ -421,38 +234,14 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { // also test the JNI and C++ AndroidVideoCapturer parts. public void testReturnBufferLateEndToEnd() throws InterruptedException { final VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null); - final PeerConnectionFactory factory = new PeerConnectionFactory(); - final VideoSource source = factory.createVideoSource(capturer, new MediaConstraints()); - final VideoTrack track = factory.createVideoTrack("dummy", source); - final FakeAsyncRenderer renderer = new FakeAsyncRenderer(); - track.addRenderer(new VideoRenderer(renderer)); - // Wait for at least one frame that has not been returned. - assertFalse(renderer.waitForPendingFrames().isEmpty()); - - capturer.stopCapture(); - - // Dispose everything. - track.dispose(); - source.dispose(); - factory.dispose(); - - // The pending frames should keep the JNI parts and |capturer| alive. - assertFalse(capturer.isReleased()); - - // Return the frame(s), on a different thread out of spite. - final List pendingFrames = renderer.waitForPendingFrames(); - final Thread returnThread = new Thread(new Runnable() { - @Override - public void run() { - for (I420Frame frame : pendingFrames) { - VideoRenderer.renderFrameDone(frame); - } - } - }); - returnThread.start(); - returnThread.join(); - - // Check that frames have successfully returned. This will cause |capturer| to be released. - assertTrue(capturer.isReleased()); + VideoCapturerAndroidTestFixtures.returnBufferLateEndToEnd(capturer); } + + /* TODO(perkj): Enable once VideoCapture to texture support has landed in C++. + @MediumTest + public void testReturnBufferLateEndToEndUsingTextures() throws InterruptedException { + final VideoCapturerAndroid capturer = + VideoCapturerAndroid.create("", null, EGL14.EGL_NO_CONTEXT); + VideoCapturerAndroidTestFixtures.returnBufferLateEndToEnd(capturer); + }*/ } diff --git a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTestFixtures.java b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTestFixtures.java new file mode 100644 index 0000000000..7e24f8c0dc --- /dev/null +++ b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTestFixtures.java @@ -0,0 +1,395 @@ +/* + * libjingle + * Copyright 2015 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.webrtc; + +import android.content.Context; +import android.hardware.Camera; + +import org.webrtc.CameraEnumerationAndroid.CaptureFormat; +import org.webrtc.VideoRenderer.I420Frame; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import static junit.framework.Assert.*; + +public class VideoCapturerAndroidTestFixtures { + static class RendererCallbacks implements VideoRenderer.Callbacks { + private int framesRendered = 0; + private Object frameLock = 0; + + @Override + public void renderFrame(I420Frame frame) { + synchronized (frameLock) { + ++framesRendered; + frameLock.notify(); + } + VideoRenderer.renderFrameDone(frame); + } + + public int WaitForNextFrameToRender() throws InterruptedException { + synchronized (frameLock) { + frameLock.wait(); + return framesRendered; + } + } + } + + static class FakeAsyncRenderer implements VideoRenderer.Callbacks { + private final List pendingFrames = new ArrayList(); + + @Override + public void renderFrame(I420Frame frame) { + synchronized (pendingFrames) { + pendingFrames.add(frame); + pendingFrames.notifyAll(); + } + } + + // Wait until at least one frame have been received, before returning them. + public List waitForPendingFrames() throws InterruptedException { + synchronized (pendingFrames) { + while (pendingFrames.isEmpty()) { + pendingFrames.wait(); + } + return new ArrayList(pendingFrames); + } + } + } + + static class FakeCapturerObserver implements + VideoCapturerAndroid.CapturerObserver { + private int framesCaptured = 0; + private int frameSize = 0; + private int frameWidth = 0; + private int frameHeight = 0; + private Object frameLock = 0; + private Object capturerStartLock = 0; + private boolean captureStartResult = false; + private List timestamps = new ArrayList(); + + @Override + public void onCapturerStarted(boolean success) { + synchronized (capturerStartLock) { + captureStartResult = success; + capturerStartLock.notify(); + } + } + + @Override + public void onByteBufferFrameCaptured(byte[] frame, int length, int width, int height, + int rotation, long timeStamp) { + synchronized (frameLock) { + ++framesCaptured; + frameSize = length; + frameWidth = width; + frameHeight = height; + timestamps.add(timeStamp); + frameLock.notify(); + } + } + @Override + public void onTextureFrameCaptured( + int width, int height, int oesTextureId, float[] transformMatrix, long timeStamp) { + synchronized (frameLock) { + ++framesCaptured; + frameWidth = width; + frameHeight = height; + frameSize = 0; + timestamps.add(timeStamp); + frameLock.notify(); + } + } + + @Override + public void onOutputFormatRequest(int width, int height, int fps) {} + + public boolean WaitForCapturerToStart() throws InterruptedException { + synchronized (capturerStartLock) { + capturerStartLock.wait(); + return captureStartResult; + } + } + + public int WaitForNextCapturedFrame() throws InterruptedException { + synchronized (frameLock) { + frameLock.wait(); + return framesCaptured; + } + } + + int frameSize() { + synchronized (frameLock) { + return frameSize; + } + } + + int frameWidth() { + synchronized (frameLock) { + return frameWidth; + } + } + + int frameHeight() { + synchronized (frameLock) { + return frameHeight; + } + } + + List getCopyAndResetListOftimeStamps() { + synchronized (frameLock) { + ArrayList list = new ArrayList(timestamps); + timestamps.clear(); + return list; + } + } + } + + // Return true if the device under test have at least two cameras. + @SuppressWarnings("deprecation") + static public boolean HaveTwoCameras() { + return (Camera.getNumberOfCameras() >= 2); + } + + static public void release(VideoCapturerAndroid capturer) { + assertNotNull(capturer); + capturer.dispose(); + assertTrue(capturer.isReleased()); + } + + static public void startCapturerAndRender(VideoCapturerAndroid capturer) + throws InterruptedException { + PeerConnectionFactory factory = new PeerConnectionFactory(); + VideoSource source = + factory.createVideoSource(capturer, new MediaConstraints()); + VideoTrack track = factory.createVideoTrack("dummy", source); + RendererCallbacks callbacks = new RendererCallbacks(); + track.addRenderer(new VideoRenderer(callbacks)); + assertTrue(callbacks.WaitForNextFrameToRender() > 0); + track.dispose(); + source.dispose(); + factory.dispose(); + assertTrue(capturer.isReleased()); + } + + static public void switchCamera(VideoCapturerAndroid capturer) throws InterruptedException { + PeerConnectionFactory factory = new PeerConnectionFactory(); + VideoSource source = + factory.createVideoSource(capturer, new MediaConstraints()); + VideoTrack track = factory.createVideoTrack("dummy", source); + + // Array with one element to avoid final problem in nested classes. + final boolean[] cameraSwitchSuccessful = new boolean[1]; + final CountDownLatch barrier = new CountDownLatch(1); + capturer.switchCamera(new VideoCapturerAndroid.CameraSwitchHandler() { + @Override + public void onCameraSwitchDone(boolean isFrontCamera) { + cameraSwitchSuccessful[0] = true; + barrier.countDown(); + } + @Override + public void onCameraSwitchError(String errorDescription) { + cameraSwitchSuccessful[0] = false; + barrier.countDown(); + } + }); + // Wait until the camera has been switched. + barrier.await(); + + // Check result. + if (HaveTwoCameras()) { + assertTrue(cameraSwitchSuccessful[0]); + } else { + assertFalse(cameraSwitchSuccessful[0]); + } + // Ensure that frames are received. + RendererCallbacks callbacks = new RendererCallbacks(); + track.addRenderer(new VideoRenderer(callbacks)); + assertTrue(callbacks.WaitForNextFrameToRender() > 0); + track.dispose(); + source.dispose(); + factory.dispose(); + assertTrue(capturer.isReleased()); + } + + static public void cameraCallsAfterStop( + VideoCapturerAndroid capturer, Context appContext) throws InterruptedException { + final List formats = capturer.getSupportedFormats(); + final CameraEnumerationAndroid.CaptureFormat format = formats.get(0); + + final FakeCapturerObserver observer = new FakeCapturerObserver(); + capturer.startCapture(format.width, format.height, format.maxFramerate, + appContext, observer); + // Make sure camera is started and then stop it. + assertTrue(observer.WaitForCapturerToStart()); + capturer.stopCapture(); + for (long timeStamp : observer.getCopyAndResetListOftimeStamps()) { + capturer.returnBuffer(timeStamp); + } + // We can't change |capturer| at this point, but we should not crash. + capturer.switchCamera(null); + capturer.onOutputFormatRequest(640, 480, 15); + capturer.changeCaptureFormat(640, 480, 15); + + capturer.dispose(); + assertTrue(capturer.isReleased()); + } + + static public void stopRestartVideoSource(VideoCapturerAndroid capturer) + throws InterruptedException { + PeerConnectionFactory factory = new PeerConnectionFactory(); + VideoSource source = + factory.createVideoSource(capturer, new MediaConstraints()); + VideoTrack track = factory.createVideoTrack("dummy", source); + RendererCallbacks callbacks = new RendererCallbacks(); + track.addRenderer(new VideoRenderer(callbacks)); + assertTrue(callbacks.WaitForNextFrameToRender() > 0); + assertEquals(MediaSource.State.LIVE, source.state()); + + source.stop(); + assertEquals(MediaSource.State.ENDED, source.state()); + + source.restart(); + assertTrue(callbacks.WaitForNextFrameToRender() > 0); + assertEquals(MediaSource.State.LIVE, source.state()); + track.dispose(); + source.dispose(); + factory.dispose(); + assertTrue(capturer.isReleased()); + } + + static public void startStopWithDifferentResolutions(VideoCapturerAndroid capturer, + Context appContext) throws InterruptedException { + FakeCapturerObserver observer = new FakeCapturerObserver(); + List formats = capturer.getSupportedFormats(); + + for(int i = 0; i < 3 ; ++i) { + CameraEnumerationAndroid.CaptureFormat format = formats.get(i); + capturer.startCapture(format.width, format.height, format.maxFramerate, + appContext, observer); + assertTrue(observer.WaitForCapturerToStart()); + observer.WaitForNextCapturedFrame(); + + // Check the frame size. The actual width and height depend on how the capturer is mounted. + final boolean identicalResolution = (observer.frameWidth() == format.width + && observer.frameHeight() == format.height); + final boolean flippedResolution = (observer.frameWidth() == format.height + && observer.frameHeight() == format.width); + if (!identicalResolution && !flippedResolution) { + fail("Wrong resolution, got: " + observer.frameWidth() + "x" + observer.frameHeight() + + " expected: " + format.width + "x" + format.height + " or " + format.height + "x" + + format.width); + } + + if (capturer.isCapturingToTexture()) { + assertEquals(0, observer.frameSize()); + } else { + assertEquals(format.frameSize(), observer.frameSize()); + } + capturer.stopCapture(); + for (long timestamp : observer.getCopyAndResetListOftimeStamps()) { + capturer.returnBuffer(timestamp); + } + } + capturer.dispose(); + assertTrue(capturer.isReleased()); + } + + static public void returnBufferLate(VideoCapturerAndroid capturer, + Context appContext) throws InterruptedException { + FakeCapturerObserver observer = new FakeCapturerObserver(); + + List formats = capturer.getSupportedFormats(); + CameraEnumerationAndroid.CaptureFormat format = formats.get(0); + capturer.startCapture(format.width, format.height, format.maxFramerate, + appContext, observer); + assertTrue(observer.WaitForCapturerToStart()); + + observer.WaitForNextCapturedFrame(); + capturer.stopCapture(); + List listOftimestamps = observer.getCopyAndResetListOftimeStamps(); + assertTrue(listOftimestamps.size() >= 1); + + format = formats.get(1); + capturer.startCapture(format.width, format.height, format.maxFramerate, + appContext, observer); + observer.WaitForCapturerToStart(); + + for (Long timeStamp : listOftimestamps) { + capturer.returnBuffer(timeStamp); + } + + observer.WaitForNextCapturedFrame(); + capturer.stopCapture(); + + listOftimestamps = observer.getCopyAndResetListOftimeStamps(); + assertTrue(listOftimestamps.size() >= 1); + for (Long timeStamp : listOftimestamps) { + capturer.returnBuffer(timeStamp); + } + capturer.dispose(); + assertTrue(capturer.isReleased()); + } + + static public void returnBufferLateEndToEnd(VideoCapturerAndroid capturer) + throws InterruptedException { + final PeerConnectionFactory factory = new PeerConnectionFactory(); + final VideoSource source = factory.createVideoSource(capturer, new MediaConstraints()); + final VideoTrack track = factory.createVideoTrack("dummy", source); + final FakeAsyncRenderer renderer = new FakeAsyncRenderer(); + track.addRenderer(new VideoRenderer(renderer)); + // Wait for at least one frame that has not been returned. + assertFalse(renderer.waitForPendingFrames().isEmpty()); + + capturer.stopCapture(); + + // Dispose everything. + track.dispose(); + source.dispose(); + factory.dispose(); + + // The pending frames should keep the JNI parts and |capturer| alive. + assertFalse(capturer.isReleased()); + + // Return the frame(s), on a different thread out of spite. + final List pendingFrames = renderer.waitForPendingFrames(); + final Thread returnThread = new Thread(new Runnable() { + @Override + public void run() { + for (I420Frame frame : pendingFrames) { + VideoRenderer.renderFrameDone(frame); + } + } + }); + returnThread.start(); + returnThread.join(); + + // Check that frames have successfully returned. This will cause |capturer| to be released. + assertTrue(capturer.isReleased()); + } +} diff --git a/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java b/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java index fec41c12e9..94d180da5a 100644 --- a/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java +++ b/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java @@ -76,6 +76,15 @@ public class RendererCommon { 0, 1, 0, 1}; } + // Matrix with transform x' = 1 - x. + public static final float[] horizontalFlipMatrix() { + return new float[] { + -1, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0, + 1, 0, 0, 1}; + } + /** * Returns texture matrix that will have the effect of rotating the frame |rotationDegree| * clockwise when rendered. diff --git a/talk/app/webrtc/java/android/org/webrtc/VideoCapturerAndroid.java b/talk/app/webrtc/java/android/org/webrtc/VideoCapturerAndroid.java index f0fbcccdb2..7ec86b005b 100644 --- a/talk/app/webrtc/java/android/org/webrtc/VideoCapturerAndroid.java +++ b/talk/app/webrtc/java/android/org/webrtc/VideoCapturerAndroid.java @@ -28,14 +28,14 @@ package org.webrtc; import android.content.Context; -import android.graphics.SurfaceTexture; import android.hardware.Camera; import android.hardware.Camera.PreviewCallback; -import android.opengl.GLES11Ext; -import android.opengl.GLES20; +import android.opengl.EGL14; +import android.opengl.EGLContext; import android.os.Handler; import android.os.HandlerThread; import android.os.SystemClock; +import android.text.StaticLayout; import android.view.Surface; import android.view.WindowManager; @@ -47,9 +47,11 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -65,34 +67,36 @@ import java.util.concurrent.TimeUnit; // camera thread. The internal *OnCameraThread() methods must check |camera| for null to check if // the camera has been stopped. @SuppressWarnings("deprecation") -public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallback { +public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallback, + SurfaceTextureHelper.OnTextureFrameAvailableListener { private final static String TAG = "VideoCapturerAndroid"; private final static int CAMERA_OBSERVER_PERIOD_MS = 5000; private Camera camera; // Only non-null while capturing. private HandlerThread cameraThread; private final Handler cameraThreadHandler; - // |cameraSurfaceTexture| is used with setPreviewTexture. Must be a member, see issue webrtc:5021. - private SurfaceTexture cameraSurfaceTexture; private Context applicationContext; // Synchronization lock for |id|. private final Object cameraIdLock = new Object(); private int id; private Camera.CameraInfo info; - private int cameraGlTexture = 0; private final FramePool videoBuffers; + private final CameraStatistics cameraStatistics = new CameraStatistics(); // Remember the requested format in case we want to switch cameras. private int requestedWidth; private int requestedHeight; private int requestedFramerate; // The capture format will be the closest supported format to the requested format. private CaptureFormat captureFormat; - private int cameraFramesCount; - private int captureBuffersCount; private final Object pendingCameraSwitchLock = new Object(); private volatile boolean pendingCameraSwitch; private CapturerObserver frameObserver = null; private final CameraErrorHandler errorHandler; + private final boolean isCapturingToTexture; + private final SurfaceTextureHelper surfaceHelper; + // The camera API can output one old frame after the camera has been switched or the resolution + // has been changed. This flag is used for dropping the first frame after camera restart. + private boolean dropNextFrame = false; // Camera error callback. private final Camera.ErrorCallback cameraErrorCallback = @@ -112,34 +116,74 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } }; - // Camera observer - monitors camera framerate and amount of available - // camera buffers. Observer is excecuted on camera thread. + // Camera observer - monitors camera framerate. Observer is executed on camera thread. private final Runnable cameraObserver = new Runnable() { @Override public void run() { + int cameraFramesCount = cameraStatistics.getAndResetFrameCount(); int cameraFps = (cameraFramesCount * 1000 + CAMERA_OBSERVER_PERIOD_MS / 2) / CAMERA_OBSERVER_PERIOD_MS; - double averageCaptureBuffersCount = 0; - if (cameraFramesCount > 0) { - averageCaptureBuffersCount = - (double)captureBuffersCount / cameraFramesCount; - } - Logging.d(TAG, "Camera fps: " + cameraFps + ". CaptureBuffers: " + - String.format("%.1f", averageCaptureBuffersCount) + - ". Pending buffers: " + videoBuffers.pendingFramesTimeStamps()); + + Logging.d(TAG, "Camera fps: " + cameraFps + + ". Pending buffers: " + cameraStatistics.pendingFramesTimeStamps()); if (cameraFramesCount == 0) { Logging.e(TAG, "Camera freezed."); if (errorHandler != null) { errorHandler.onCameraError("Camera failure."); } } else { - cameraFramesCount = 0; - captureBuffersCount = 0; cameraThreadHandler.postDelayed(this, CAMERA_OBSERVER_PERIOD_MS); } } }; + private static class CameraStatistics { + private int frameCount = 0; + private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); + private final Set timeStampsNs = new HashSet(); + + CameraStatistics() { + threadChecker.detachThread(); + } + + public void addPendingFrame(long timestamp) { + threadChecker.checkIsOnValidThread(); + ++frameCount; + timeStampsNs.add(timestamp); + } + + public void frameReturned(long timestamp) { + threadChecker.checkIsOnValidThread(); + if (!timeStampsNs.contains(timestamp)) { + throw new IllegalStateException( + "CameraStatistics.frameReturned called with unknown timestamp " + timestamp); + } + timeStampsNs.remove(timestamp); + } + + public int getAndResetFrameCount() { + threadChecker.checkIsOnValidThread(); + int count = frameCount; + frameCount = 0; + return count; + } + + // Return number of pending frames that have not been returned. + public int pendingFramesCount() { + threadChecker.checkIsOnValidThread(); + return timeStampsNs.size(); + } + + public String pendingFramesTimeStamps() { + threadChecker.checkIsOnValidThread(); + List timeStampsMs = new ArrayList(); + for (long ts : timeStampsNs) { + timeStampsMs.add(TimeUnit.NANOSECONDS.toMillis(ts)); + } + return timeStampsMs.toString(); + } + } + // Camera error handler - invoked when camera stops receiving frames // or any camera exception happens on camera thread. public static interface CameraErrorHandler { @@ -155,12 +199,20 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba void onCameraSwitchError(String errorDescription); } - public static VideoCapturerAndroid create(String name, CameraErrorHandler errorHandler) { + public static VideoCapturerAndroid create(String name, + CameraErrorHandler errorHandler) { + return VideoCapturerAndroid.create(name, errorHandler, null); + } + + public static VideoCapturerAndroid create(String name, + CameraErrorHandler errorHandler, EGLContext sharedContext) { final int cameraId = lookupDeviceName(name); if (cameraId == -1) { return null; } - final VideoCapturerAndroid capturer = new VideoCapturerAndroid(cameraId, errorHandler); + + final VideoCapturerAndroid capturer = new VideoCapturerAndroid(cameraId, errorHandler, + sharedContext); capturer.setNativeCapturer(nativeCreateVideoCapturer(capturer)); return capturer; } @@ -208,10 +260,10 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Requests a new output format from the video capturer. Captured frames // by the camera will be scaled/or dropped by the video capturer. // TODO(magjed/perkj): Document what this function does. Change name? - public void onOutputFormatRequest(final int width, final int height, final int fps) { + public void onOutputFormatRequest(final int width, final int height, final int framerate) { cameraThreadHandler.post(new Runnable() { @Override public void run() { - onOutputFormatRequestOnCameraThread(width, height, fps); + onOutputFormatRequestOnCameraThread(width, height, framerate); } }); } @@ -238,6 +290,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba return CameraEnumerationAndroid.getSupportedFormats(getCurrentCameraId()); } + // Returns true if this VideoCapturer is setup to capture video frames to a SurfaceTexture. + public boolean isCapturingToTexture() { + return isCapturingToTexture; + } + // Called from native code. private String getSupportedFormatsAsJson() throws JSONException { return CameraEnumerationAndroid.getSupportedFormatsAsJson(getCurrentCameraId()); @@ -245,10 +302,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Called from native VideoCapturer_nativeCreateVideoCapturer. private VideoCapturerAndroid(int cameraId) { - this(cameraId, null); + this(cameraId, null, null); } - private VideoCapturerAndroid(int cameraId, CameraErrorHandler errorHandler) { + private VideoCapturerAndroid(int cameraId, CameraErrorHandler errorHandler, + EGLContext sharedContext) { Logging.d(TAG, "VideoCapturerAndroid"); this.id = cameraId; this.errorHandler = errorHandler; @@ -256,6 +314,13 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba cameraThread.start(); cameraThreadHandler = new Handler(cameraThread.getLooper()); videoBuffers = new FramePool(cameraThread); + surfaceHelper = + new SurfaceTextureHelper(sharedContext == null ? EGL14.EGL_NO_CONTEXT : sharedContext, + cameraThread); + if (sharedContext != null) { + surfaceHelper.setListener(this); + } + isCapturingToTexture = sharedContext != null; } private void checkIsOnCameraThread() { @@ -285,6 +350,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Called by native code to quit the camera thread. This needs to be done manually, otherwise the // thread and handler will not be garbage collected. private void release() { + Logging.d(TAG, "release"); if (isReleased()) { throw new IllegalStateException("Already released"); } @@ -294,11 +360,13 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba if (camera != null) { throw new IllegalStateException("Release called while camera is running"); } - if (videoBuffers.pendingFramesCount() != 0) { + if (cameraStatistics.pendingFramesCount() != 0) { throw new IllegalStateException("Release called with pending frames left"); } } }); + surfaceHelper.disconnect(); + cameraThread.quitSafely(); ThreadUtils.joinUninterruptibly(cameraThread); cameraThread = null; @@ -349,16 +417,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba info = new Camera.CameraInfo(); Camera.getCameraInfo(id, info); } - // No local renderer (we only care about onPreviewFrame() buffers, not a - // directly-displayed UI element). Camera won't capture without - // setPreview{Texture,Display}, so we create a SurfaceTexture and hand - // it over to Camera, but never listen for frame-ready callbacks, - // and never call updateTexImage on it. try { - cameraGlTexture = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); - cameraSurfaceTexture = new SurfaceTexture(cameraGlTexture); - cameraSurfaceTexture.setOnFrameAvailableListener(null); - camera.setPreviewTexture(cameraSurfaceTexture); + camera.setPreviewTexture(surfaceHelper.getSurfaceTexture()); } catch (IOException e) { Logging.e(TAG, "setPreviewTexture failed", error); throw new RuntimeException(e); @@ -368,11 +428,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba " .Device orientation: " + getDeviceOrientation()); camera.setErrorCallback(cameraErrorCallback); startPreviewOnCameraThread(width, height, framerate); - frameObserver.OnCapturerStarted(true); + frameObserver.onCapturerStarted(true); // Start camera observer. - cameraFramesCount = 0; - captureBuffersCount = 0; cameraThreadHandler.postDelayed(cameraObserver, CAMERA_OBSERVER_PERIOD_MS); return; } catch (RuntimeException e) { @@ -380,7 +438,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } Logging.e(TAG, "startCapture failed", error); stopCaptureOnCameraThread(); - frameObserver.OnCapturerStarted(false); + frameObserver.onCapturerStarted(false); if (errorHandler != null) { errorHandler.onCameraError("Camera can not be started."); } @@ -438,6 +496,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Temporarily stop preview if it's already running. if (this.captureFormat != null) { camera.stopPreview(); + dropNextFrame = true; // Calling |setPreviewCallbackWithBuffer| with null should clear the internal camera buffer // queue, but sometimes we receive a frame with the old resolution after this call anyway. camera.setPreviewCallbackWithBuffer(null); @@ -453,8 +512,10 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } camera.setParameters(parameters); - videoBuffers.queueCameraBuffers(captureFormat.frameSize(), camera); - camera.setPreviewCallbackWithBuffer(this); + if (!isCapturingToTexture) { + videoBuffers.queueCameraBuffers(captureFormat.frameSize(), camera); + camera.setPreviewCallbackWithBuffer(this); + } camera.startPreview(); } @@ -481,21 +542,22 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } cameraThreadHandler.removeCallbacks(cameraObserver); + cameraStatistics.getAndResetFrameCount(); Logging.d(TAG, "Stop preview."); camera.stopPreview(); camera.setPreviewCallbackWithBuffer(null); - videoBuffers.stopReturnBuffersToCamera(); + if (!isCapturingToTexture()) { + videoBuffers.stopReturnBuffersToCamera(); + Logging.d(TAG, "stopReturnBuffersToCamera called." + + (cameraStatistics.pendingFramesCount() == 0? + " All buffers have been returned." + : " Pending buffers: " + cameraStatistics.pendingFramesTimeStamps() + ".")); + } captureFormat = null; - if (cameraGlTexture != 0) { - GLES20.glDeleteTextures(1, new int[] {cameraGlTexture}, 0); - cameraGlTexture = 0; - } Logging.d(TAG, "Release camera."); camera.release(); camera = null; - cameraSurfaceTexture.release(); - cameraSurfaceTexture = null; } private void switchCameraOnCameraThread() { @@ -505,26 +567,32 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba synchronized (cameraIdLock) { id = (id + 1) % Camera.getNumberOfCameras(); } + dropNextFrame = true; startCaptureOnCameraThread(requestedWidth, requestedHeight, requestedFramerate, frameObserver, applicationContext); Logging.d(TAG, "switchCameraOnCameraThread done"); } - private void onOutputFormatRequestOnCameraThread(int width, int height, int fps) { + private void onOutputFormatRequestOnCameraThread(int width, int height, int framerate) { checkIsOnCameraThread(); if (camera == null) { Logging.e(TAG, "Calling onOutputFormatRequest() on stopped camera."); return; } Logging.d(TAG, "onOutputFormatRequestOnCameraThread: " + width + "x" + height + - "@" + fps); - frameObserver.OnOutputFormatRequest(width, height, fps); + "@" + framerate); + frameObserver.onOutputFormatRequest(width, height, framerate); } public void returnBuffer(final long timeStamp) { cameraThreadHandler.post(new Runnable() { @Override public void run() { - videoBuffers.returnBuffer(timeStamp); + cameraStatistics.frameReturned(timeStamp); + if (isCapturingToTexture) { + surfaceHelper.returnTextureFrame(); + } else { + videoBuffers.returnBuffer(timeStamp); + } } }); } @@ -552,6 +620,14 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba return orientation; } + private int getFrameOrientation() { + int rotation = getDeviceOrientation(); + if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { + rotation = 360 - rotation; + } + return (info.orientation + rotation) % 360; + } + // Called on cameraThread so must not "synchronized". @Override public void onPreviewFrame(byte[] data, Camera callbackCamera) { @@ -566,24 +642,50 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba final long captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); - captureBuffersCount += videoBuffers.numCaptureBuffersAvailable(); - int rotation = getDeviceOrientation(); - if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { - rotation = 360 - rotation; - } - rotation = (info.orientation + rotation) % 360; + // Mark the frame owning |data| as used. // Note that since data is directBuffer, // data.length >= videoBuffers.frameSize. if (videoBuffers.reserveByteBuffer(data, captureTimeNs)) { - cameraFramesCount++; - frameObserver.OnFrameCaptured(data, videoBuffers.frameSize, captureFormat.width, - captureFormat.height, rotation, captureTimeNs); + cameraStatistics.addPendingFrame(captureTimeNs); + frameObserver.onByteBufferFrameCaptured(data, videoBuffers.frameSize, captureFormat.width, + captureFormat.height, getFrameOrientation(), captureTimeNs); } else { Logging.w(TAG, "reserveByteBuffer failed - dropping frame."); } } + @Override + public void onTextureFrameAvailable( + int oesTextureId, float[] transformMatrix, long timestampNs) { + checkIsOnCameraThread(); + if (camera == null) { + // Camera is stopped, we need to return the buffer immediately. + surfaceHelper.returnTextureFrame(); + return; + } + if (!dropNextFrame) { + surfaceHelper.returnTextureFrame(); + dropNextFrame = true; + return; + } + + int rotation = getFrameOrientation(); + if (info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) { + // Undo the mirror that the OS "helps" us with. + // http://developer.android.com/reference/android/hardware/Camera.html#setDisplayOrientation(int) + transformMatrix = + RendererCommon.multiplyMatrices(transformMatrix, RendererCommon.horizontalFlipMatrix()); + } + transformMatrix = RendererCommon.rotateTextureMatrix(transformMatrix, rotation); + + final int rotatedWidth = (rotation % 180 == 0) ? captureFormat.width : captureFormat.height; + final int rotatedHeight = (rotation % 180 == 0) ? captureFormat.height : captureFormat.width; + cameraStatistics.addPendingFrame(timestampNs); + frameObserver.onTextureFrameCaptured(rotatedWidth, rotatedHeight, oesTextureId, + transformMatrix, timestampNs); + } + // Class used for allocating and bookkeeping video frames. All buffers are // direct allocated so that they can be directly used from native code. This class is // not thread-safe, and enforces single thread use. @@ -613,11 +715,6 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } } - public int numCaptureBuffersAvailable() { - checkIsOnValidThread(); - return queuedBuffers.size(); - } - // Discards previous queued buffers and adds new callback buffers to camera. public void queueCameraBuffers(int frameSize, Camera camera) { checkIsOnValidThread(); @@ -634,30 +731,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba + " buffers of size " + frameSize + "."); } - // Return number of pending frames that have not been returned. - public int pendingFramesCount() { - checkIsOnValidThread(); - return pendingBuffers.size(); - } - - public String pendingFramesTimeStamps() { - checkIsOnValidThread(); - List timeStampsMs = new ArrayList(); - for (Long timeStampNs : pendingBuffers.keySet()) { - timeStampsMs.add(TimeUnit.NANOSECONDS.toMillis(timeStampNs)); - } - return timeStampsMs.toString(); - } - public void stopReturnBuffersToCamera() { checkIsOnValidThread(); this.camera = null; queuedBuffers.clear(); // Frames in |pendingBuffers| need to be kept alive until they are returned. - Logging.d(TAG, "stopReturnBuffersToCamera called." - + (pendingBuffers.isEmpty() ? - " All buffers have been returned." - : " Pending buffers: " + pendingFramesTimeStamps() + ".")); } public boolean reserveByteBuffer(byte[] data, long timeStamp) { @@ -679,8 +757,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } pendingBuffers.put(timeStamp, buffer); if (queuedBuffers.isEmpty()) { - Logging.v(TAG, "Camera is running out of capture buffers." - + " Pending buffers: " + pendingFramesTimeStamps()); + Logging.v(TAG, "Camera is running out of capture buffers."); } return true; } @@ -722,17 +799,22 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba interface CapturerObserver { // Notify if the camera have been started successfully or not. // Called on a Java thread owned by VideoCapturerAndroid. - abstract void OnCapturerStarted(boolean success); + abstract void onCapturerStarted(boolean success); // Delivers a captured frame. Called on a Java thread owned by // VideoCapturerAndroid. - abstract void OnFrameCaptured(byte[] data, int length, int width, int height, + abstract void onByteBufferFrameCaptured(byte[] data, int length, int width, int height, int rotation, long timeStamp); + // Delivers a captured frame in a texture with id |oesTextureId|. Called on a Java thread + // owned by VideoCapturerAndroid. + abstract void onTextureFrameCaptured( + int width, int height, int oesTextureId, float[] transformMatrix, long timestamp); + // Requests an output format from the video capturer. Captured frames // by the camera will be scaled/or dropped by the video capturer. // Called on a Java thread owned by VideoCapturerAndroid. - abstract void OnOutputFormatRequest(int width, int height, int fps); + abstract void onOutputFormatRequest(int width, int height, int framerate); } // An implementation of CapturerObserver that forwards all calls from @@ -745,27 +827,37 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } @Override - public void OnCapturerStarted(boolean success) { + public void onCapturerStarted(boolean success) { nativeCapturerStarted(nativeCapturer, success); } @Override - public void OnFrameCaptured(byte[] data, int length, int width, int height, + public void onByteBufferFrameCaptured(byte[] data, int length, int width, int height, int rotation, long timeStamp) { - nativeOnFrameCaptured(nativeCapturer, data, length, width, height, rotation, timeStamp); + nativeOnByteBufferFrameCaptured(nativeCapturer, data, length, width, height, rotation, + timeStamp); } @Override - public void OnOutputFormatRequest(int width, int height, int fps) { - nativeOnOutputFormatRequest(nativeCapturer, width, height, fps); + public void onTextureFrameCaptured( + int width, int height, int oesTextureId, float[] transformMatrix, long timestamp) { + nativeOnTextureFrameCaptured(nativeCapturer, width, height, oesTextureId, transformMatrix, + timestamp); + } + + @Override + public void onOutputFormatRequest(int width, int height, int framerate) { + nativeOnOutputFormatRequest(nativeCapturer, width, height, framerate); } private native void nativeCapturerStarted(long nativeCapturer, boolean success); - private native void nativeOnFrameCaptured(long nativeCapturer, + private native void nativeOnByteBufferFrameCaptured(long nativeCapturer, byte[] data, int length, int width, int height, int rotation, long timeStamp); + private native void nativeOnTextureFrameCaptured(long nativeCapturer, int width, int height, + int oesTextureId, float[] transformMatrix, long timestamp); private native void nativeOnOutputFormatRequest(long nativeCapturer, - int width, int height, int fps); + int width, int height, int framerate); } private static native long nativeCreateVideoCapturer(VideoCapturerAndroid videoCapturer); diff --git a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc index 9ac64063a6..1455885f1c 100644 --- a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc +++ b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc @@ -188,7 +188,8 @@ void AndroidVideoCapturerJni::OnOutputFormatRequest(int width, JNIEnv* AndroidVideoCapturerJni::jni() { return AttachCurrentThreadIfNeeded(); } -JOW(void, VideoCapturerAndroid_00024NativeObserver_nativeOnFrameCaptured) +JOW(void, + VideoCapturerAndroid_00024NativeObserver_nativeOnByteBufferFrameCaptured) (JNIEnv* jni, jclass, jlong j_capturer, jbyteArray j_frame, jint length, jint width, jint height, jint rotation, jlong ts) { jboolean is_copy = true;