diff --git a/webrtc/sdk/android/BUILD.gn b/webrtc/sdk/android/BUILD.gn index 401b11f695..289f8dddc9 100644 --- a/webrtc/sdk/android/BUILD.gn +++ b/webrtc/sdk/android/BUILD.gn @@ -108,6 +108,7 @@ rtc_static_library("video_jni") { "src/jni/videodecoderwrapper.cc", "src/jni/videodecoderwrapper.h", "src/jni/videofilerenderer_jni.cc", + "src/jni/videoframe_jni.cc", "src/jni/videotrack_jni.cc", "src/jni/wrapped_native_i420_buffer.cc", "src/jni/wrapped_native_i420_buffer.h", @@ -393,8 +394,8 @@ android_library("libjingle_peerconnection_java") { "api/org/webrtc/VideoSource.java", "api/org/webrtc/VideoTrack.java", "src/java/org/webrtc/AndroidVideoTrackSourceObserver.java", - "src/java/org/webrtc/BitrateAdjuster.java", "src/java/org/webrtc/BaseBitrateAdjuster.java", + "src/java/org/webrtc/BitrateAdjuster.java", "src/java/org/webrtc/Camera1Session.java", "src/java/org/webrtc/Camera2Session.java", "src/java/org/webrtc/CameraCapturer.java", @@ -407,9 +408,10 @@ android_library("libjingle_peerconnection_java") { "src/java/org/webrtc/HardwareVideoEncoder.java", "src/java/org/webrtc/Histogram.java", "src/java/org/webrtc/I420BufferImpl.java", - "src/java/org/webrtc/VideoDecoderWrapperCallback.java", "src/java/org/webrtc/MediaCodecUtils.java", + "src/java/org/webrtc/TextureBufferImpl.java", "src/java/org/webrtc/VideoCodecType.java", + "src/java/org/webrtc/VideoDecoderWrapperCallback.java", "src/java/org/webrtc/WrappedNativeI420Buffer.java", "src/java/org/webrtc/YuvConverter.java", ] diff --git a/webrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java b/webrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java index bd3d545a18..a3db84de8a 100644 --- a/webrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java +++ b/webrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java @@ -10,6 +10,7 @@ package org.webrtc; +import android.graphics.Matrix; import android.graphics.SurfaceTexture; import android.opengl.GLES11Ext; import android.opengl.GLES20; @@ -288,86 +289,13 @@ public class SurfaceTextureHelper { * The returned TextureBuffer holds a reference to the SurfaceTextureHelper that created it. The * buffer calls returnTextureFrame() when it is released. */ - public TextureBuffer createTextureBuffer(int width, int height, float[] transformMatrix) { - return new OesTextureBuffer(oesTextureId, width, height, transformMatrix, this); - } - - /** - * Android OES texture buffer backed by a SurfaceTextureHelper's texture. The buffer calls - * returnTextureFrame() when it is released. - */ - private static class OesTextureBuffer implements TextureBuffer { - private final int id; - private final int width; - private final int height; - private final float[] transformMatrix; - private final SurfaceTextureHelper helper; - private int refCount; - - OesTextureBuffer( - int id, int width, int height, float[] transformMatrix, SurfaceTextureHelper helper) { - this.id = id; - this.width = width; - this.height = height; - this.transformMatrix = transformMatrix; - this.helper = helper; - this.refCount = 1; // Creator implicitly holds a reference. - } - - @Override - public TextureBuffer.Type getType() { - return TextureBuffer.Type.OES; - } - - @Override - public int getTextureId() { - return id; - } - - @Override - public int getWidth() { - return width; - } - - @Override - public int getHeight() { - return height; - } - - @Override - public I420Buffer toI420() { - // SurfaceTextureHelper requires a stride that is divisible by 8. Round width up. - // See SurfaceTextureHelper for details on the size and format. - int stride = ((width + 7) / 8) * 8; - int uvHeight = (height + 1) / 2; - // Due to the layout used by SurfaceTextureHelper, vPos + stride * uvHeight would overrun the - // buffer. Add one row at the bottom to compensate for this. There will never be data in the - // extra row, but now other code does not have to deal with v stride * v height exceeding the - // buffer's capacity. - int size = stride * (height + uvHeight + 1); - ByteBuffer buffer = ByteBuffer.allocateDirect(size); - helper.textureToYUV(buffer, width, height, stride, id, transformMatrix); - - int yPos = 0; - int uPos = yPos + stride * height; - // Rows of U and V alternate in the buffer, so V data starts after the first row of U. - int vPos = yPos + stride / 2; - - // SurfaceTextureHelper uses the same stride for Y, U, and V data. - return new I420BufferImpl( - buffer, width, height, yPos, stride, uPos, stride, vPos, stride, null); - } - - @Override - public void retain() { - ++refCount; - } - - @Override - public void release() { - if (--refCount == 0) { - helper.returnTextureFrame(); - } - } + public TextureBuffer createTextureBuffer(int width, int height, Matrix transformMatrix) { + return new TextureBufferImpl( + width, height, TextureBuffer.Type.OES, oesTextureId, transformMatrix, this, new Runnable() { + @Override + public void run() { + returnTextureFrame(); + } + }); } } diff --git a/webrtc/sdk/android/api/org/webrtc/VideoFrame.java b/webrtc/sdk/android/api/org/webrtc/VideoFrame.java index 3da772dd9b..69b6aa6251 100644 --- a/webrtc/sdk/android/api/org/webrtc/VideoFrame.java +++ b/webrtc/sdk/android/api/org/webrtc/VideoFrame.java @@ -44,6 +44,13 @@ public class VideoFrame { */ void retain(); void release(); + + /** + * Crops a region defined by |cropx|, |cropY|, |cropWidth| and |cropHeight|. Scales it to size + * |scaleWidth| x |scaleHeight|. + */ + Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight); } /** @@ -67,24 +74,26 @@ public class VideoFrame { Type getType(); int getTextureId(); + + /** + * Retrieve the transform matrix associated with the frame. This transform matrix maps 2D + * homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to + * the coordinate that should be used to sample that location from the buffer. + */ + public Matrix getTransformMatrix(); } private final Buffer buffer; private final int rotation; private final long timestampNs; - private final Matrix transformMatrix; - public VideoFrame(Buffer buffer, int rotation, long timestampNs, Matrix transformMatrix) { + public VideoFrame(Buffer buffer, int rotation, long timestampNs) { if (buffer == null) { throw new IllegalArgumentException("buffer not allowed to be null"); } - if (transformMatrix == null) { - throw new IllegalArgumentException("transformMatrix not allowed to be null"); - } this.buffer = buffer; this.rotation = rotation; this.timestampNs = timestampNs; - this.transformMatrix = transformMatrix; } public Buffer getBuffer() { @@ -105,26 +114,6 @@ public class VideoFrame { return timestampNs; } - /** - * Retrieve the transform matrix associated with the frame. This transform matrix maps 2D - * homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to the - * coordinate that should be used to sample that location from the buffer. - */ - public Matrix getTransformMatrix() { - return transformMatrix; - } - - /** - * Resolution of the frame in pixels. - */ - public int getWidth() { - return buffer.getWidth(); - } - - public int getHeight() { - return buffer.getHeight(); - } - /** * Reference counting of the underlying buffer. */ @@ -135,4 +124,41 @@ public class VideoFrame { public void release() { buffer.release(); } + + public static VideoFrame.Buffer cropAndScaleI420(final I420Buffer buffer, int cropX, int cropY, + int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + if (cropWidth == scaleWidth && cropHeight == scaleHeight) { + // No scaling. + ByteBuffer dataY = buffer.getDataY(); + ByteBuffer dataU = buffer.getDataU(); + ByteBuffer dataV = buffer.getDataV(); + + dataY.position(cropX + cropY * buffer.getStrideY()); + dataU.position(cropX / 2 + cropY / 2 * buffer.getStrideU()); + dataV.position(cropX / 2 + cropY / 2 * buffer.getStrideV()); + + buffer.retain(); + return new I420BufferImpl(buffer.getWidth(), buffer.getHeight(), dataY.slice(), + buffer.getStrideY(), dataU.slice(), buffer.getStrideU(), dataV.slice(), + buffer.getStrideV(), new Runnable() { + @Override + public void run() { + buffer.release(); + } + }); + } + + I420BufferImpl newBuffer = I420BufferImpl.allocate(scaleWidth, scaleHeight); + nativeCropAndScaleI420(buffer.getDataY(), buffer.getStrideY(), buffer.getDataU(), + buffer.getStrideU(), buffer.getDataV(), buffer.getStrideV(), cropX, cropY, cropWidth, + cropHeight, newBuffer.getDataY(), newBuffer.getStrideY(), newBuffer.getDataU(), + newBuffer.getStrideU(), newBuffer.getDataV(), newBuffer.getStrideV(), scaleWidth, + scaleHeight); + return newBuffer; + } + + private static native void nativeCropAndScaleI420(ByteBuffer srcY, int srcStrideY, + ByteBuffer srcU, int srcStrideU, ByteBuffer srcV, int srcStrideV, int cropX, int cropY, + int cropWidth, int cropHeight, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU, + int dstStrideU, ByteBuffer dstV, int dstStrideV, int scaleWidth, int scaleHeight); } diff --git a/webrtc/sdk/android/api/org/webrtc/VideoRenderer.java b/webrtc/sdk/android/api/org/webrtc/VideoRenderer.java index 0825b7408b..b2812ea8d4 100644 --- a/webrtc/sdk/android/api/org/webrtc/VideoRenderer.java +++ b/webrtc/sdk/android/api/org/webrtc/VideoRenderer.java @@ -84,12 +84,11 @@ public class VideoRenderer { } /** - * Construct a frame of the given dimensions from VideoFrame.Buffer. + * Construct a frame from VideoFrame.Buffer. */ - public I420Frame(int width, int height, int rotationDegree, float[] samplingMatrix, - VideoFrame.Buffer buffer, long nativeFramePointer) { - this.width = width; - this.height = height; + public I420Frame(int rotationDegree, VideoFrame.Buffer buffer, long nativeFramePointer) { + this.width = buffer.getWidth(); + this.height = buffer.getHeight(); this.rotationDegree = rotationDegree; if (rotationDegree % 90 != 0) { throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree); @@ -98,7 +97,8 @@ public class VideoRenderer { VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) buffer; this.yuvFrame = false; this.textureId = textureBuffer.getTextureId(); - this.samplingMatrix = samplingMatrix; + this.samplingMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix( + textureBuffer.getTransformMatrix()); this.yuvStrides = null; this.yuvPlanes = null; @@ -113,8 +113,7 @@ public class VideoRenderer { // top-left corner of the image, but in glTexImage2D() the first element corresponds to the // bottom-left corner. This discrepancy is corrected by multiplying the sampling matrix with // a vertical flip matrix. - this.samplingMatrix = - RendererCommon.multiplyMatrices(samplingMatrix, RendererCommon.verticalFlipMatrix()); + this.samplingMatrix = RendererCommon.verticalFlipMatrix(); this.textureId = 0; } diff --git a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java index 6c9d0b7b5c..59dbc87113 100644 --- a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java +++ b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java @@ -18,8 +18,8 @@ import android.annotation.TargetApi; import android.graphics.Matrix; import android.support.test.filters.MediumTest; import android.util.Log; -import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import org.chromium.base.test.BaseJUnit4ClassRunner; import org.junit.Test; import org.junit.runner.RunWith; @@ -123,8 +123,7 @@ public final class HardwareVideoDecoderTest { // First, encode a frame. VideoFrame.I420Buffer buffer = I420BufferImpl.allocate(SETTINGS.width, SETTINGS.height); - VideoFrame frame = - new VideoFrame(buffer, rotation, presentationTimestampUs * 1000, new Matrix()); + VideoFrame frame = new VideoFrame(buffer, rotation, presentationTimestampUs * 1000); VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameKey}); @@ -141,9 +140,8 @@ public final class HardwareVideoDecoderTest { frame = decoded.get(); assertEquals(frame.getRotation(), rotation); assertEquals(frame.getTimestampNs(), presentationTimestampUs * 1000); - assertEquals(frame.getTransformMatrix(), new Matrix()); - assertEquals(frame.getWidth(), SETTINGS.width); - assertEquals(frame.getHeight(), SETTINGS.height); + assertEquals(frame.getBuffer().getWidth(), SETTINGS.width); + assertEquals(frame.getBuffer().getHeight(), SETTINGS.height); frame.release(); assertEquals(decoder.release(), VideoCodecStatus.OK); @@ -200,8 +198,7 @@ public final class HardwareVideoDecoderTest { // First, encode a frame. VideoFrame.I420Buffer buffer = I420BufferImpl.allocate(SETTINGS.width, SETTINGS.height); - VideoFrame frame = - new VideoFrame(buffer, rotation, presentationTimestampUs * 1000, new Matrix()); + VideoFrame frame = new VideoFrame(buffer, rotation, presentationTimestampUs * 1000); VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameKey}); @@ -218,13 +215,13 @@ public final class HardwareVideoDecoderTest { frame = decoded.get(); assertEquals(frame.getRotation(), rotation); assertEquals(frame.getTimestampNs(), presentationTimestampUs * 1000); - // TODO(mellem): Compare the matrix to whatever we expect to get back? - assertNotNull(frame.getTransformMatrix()); - assertEquals(frame.getWidth(), SETTINGS.width); - assertEquals(frame.getHeight(), SETTINGS.height); assertTrue(frame.getBuffer() instanceof VideoFrame.TextureBuffer); VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) frame.getBuffer(); + // TODO(mellem): Compare the matrix to whatever we expect to get back? + assertNotNull(textureBuffer.getTransformMatrix()); + assertEquals(textureBuffer.getWidth(), SETTINGS.width); + assertEquals(textureBuffer.getHeight(), SETTINGS.height); assertEquals(textureBuffer.getType(), VideoFrame.TextureBuffer.Type.OES); assertEquals(decoder.release(), VideoCodecStatus.OK); diff --git a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java index 7b0a38510f..39a9672dd3 100644 --- a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java +++ b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java @@ -101,8 +101,7 @@ public class HardwareVideoEncoderTest { assertEquals(encoder.initEncode(SETTINGS, callback), VideoCodecStatus.OK); VideoFrame.I420Buffer buffer = I420BufferImpl.allocate(SETTINGS.width, SETTINGS.height); - VideoFrame frame = - new VideoFrame(buffer, 0 /* rotation */, presentationTimestampUs * 1000, new Matrix()); + VideoFrame frame = new VideoFrame(buffer, 0 /* rotation */, presentationTimestampUs * 1000); VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameKey}); @@ -162,6 +161,11 @@ public class HardwareVideoEncoderTest { return oesTextureId; } + @Override + public Matrix getTransformMatrix() { + return new Matrix(); + } + @Override public int getWidth() { return SETTINGS.width; @@ -182,9 +186,14 @@ public class HardwareVideoEncoderTest { @Override public void release() {} + + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + return null; + } }; - VideoFrame frame = - new VideoFrame(buffer, 0 /* rotation */, presentationTimestampUs * 1000, new Matrix()); + VideoFrame frame = new VideoFrame(buffer, 0 /* rotation */, presentationTimestampUs * 1000); VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameKey}); diff --git a/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java b/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java index 13ccedc26d..b7aae2a338 100644 --- a/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java +++ b/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java @@ -417,12 +417,11 @@ class HardwareVideoDecoder @Override public void onTextureFrameAvailable(int oesTextureId, float[] transformMatrix, long timestampNs) { VideoFrame.TextureBuffer oesBuffer = surfaceTextureHelper.createTextureBuffer( - renderedTextureMetadata.width, renderedTextureMetadata.height, transformMatrix); - - Matrix matrix = RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix); + renderedTextureMetadata.width, renderedTextureMetadata.height, + RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix)); VideoFrame frame = new VideoFrame(oesBuffer, renderedTextureMetadata.rotation, - renderedTextureMetadata.presentationTimestampUs * 1000, matrix); + renderedTextureMetadata.presentationTimestampUs * 1000); callback.onDecodedFrame(frame, renderedTextureMetadata.decodeTimeMs, null /* qp */); frame.release(); } @@ -477,7 +476,7 @@ class HardwareVideoDecoder } long presentationTimeNs = info.presentationTimeUs * 1000; - VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs, new Matrix()); + VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs); // Note that qp is parsed on the C++ side. callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */); @@ -605,9 +604,9 @@ class HardwareVideoDecoder activeOutputBuffers++; } - I420BufferImpl.ReleaseCallback callback = new I420BufferImpl.ReleaseCallback() { + Runnable callback = new Runnable() { @Override - public void onRelease() { + public void run() { codec.releaseOutputBuffer(outputBufferIndex, false); synchronized (activeOutputBuffersLock) { activeOutputBuffers--; @@ -616,8 +615,20 @@ class HardwareVideoDecoder } }; + buffer.position(yPos); + buffer.limit(uPos); + ByteBuffer dataY = buffer.slice(); + + buffer.position(uPos); + buffer.limit(vPos); + ByteBuffer dataU = buffer.slice(); + + buffer.position(vPos); + buffer.limit(vPos + uvStride * sliceHeight / 2); + ByteBuffer dataV = buffer.slice(); + return new I420BufferImpl( - buffer, width, height, yPos, stride, uPos, uvStride, vPos, uvStride, callback); + width, height, dataY, stride, dataU, uvStride, dataV, uvStride, callback); } private static void copyI420(ByteBuffer src, int offset, VideoFrame.I420Buffer frameBuffer, diff --git a/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java index 8c3eef5ec4..fc868c4b8c 100644 --- a/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java +++ b/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java @@ -235,8 +235,8 @@ class HardwareVideoEncoder implements VideoEncoder { } // If input resolution changed, restart the codec with the new resolution. - int frameWidth = videoFrame.getWidth(); - int frameHeight = videoFrame.getHeight(); + int frameWidth = videoFrame.getBuffer().getWidth(); + int frameHeight = videoFrame.getBuffer().getHeight(); if (frameWidth != width || frameHeight != height) { VideoCodecStatus status = resetCodec(frameWidth, frameHeight); if (status != VideoCodecStatus.OK) { @@ -271,8 +271,8 @@ class HardwareVideoEncoder implements VideoEncoder { EncodedImage.Builder builder = EncodedImage.builder() .setCaptureTimeMs(presentationTimestampMs) .setCompleteFrame(true) - .setEncodedWidth(videoFrame.getWidth()) - .setEncodedHeight(videoFrame.getHeight()) + .setEncodedWidth(videoFrame.getBuffer().getWidth()) + .setEncodedHeight(videoFrame.getBuffer().getHeight()) .setRotation(videoFrame.getRotation()); outputBuilders.offer(builder); @@ -293,7 +293,7 @@ class HardwareVideoEncoder implements VideoEncoder { private VideoCodecStatus encodeTextureBuffer( VideoFrame videoFrame, VideoFrame.TextureBuffer textureBuffer) { - Matrix matrix = videoFrame.getTransformMatrix(); + Matrix matrix = textureBuffer.getTransformMatrix(); float[] transformationMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(matrix); try { diff --git a/webrtc/sdk/android/src/java/org/webrtc/I420BufferImpl.java b/webrtc/sdk/android/src/java/org/webrtc/I420BufferImpl.java index 09c0782271..62d8f6ea92 100644 --- a/webrtc/sdk/android/src/java/org/webrtc/I420BufferImpl.java +++ b/webrtc/sdk/android/src/java/org/webrtc/I420BufferImpl.java @@ -15,32 +15,28 @@ import org.webrtc.VideoFrame.I420Buffer; /** Implementation of an I420 VideoFrame buffer. */ class I420BufferImpl implements VideoFrame.I420Buffer { - private final ByteBuffer buffer; private final int width; private final int height; - private final int chromaHeight; - private final int yPos; + private final ByteBuffer dataY; + private final ByteBuffer dataU; + private final ByteBuffer dataV; private final int strideY; - private final int uPos; private final int strideU; - private final int vPos; private final int strideV; - private final ReleaseCallback releaseCallback; + private final Runnable releaseCallback; private int refCount; - /** Allocates an I420Buffer backed by existing data. */ - I420BufferImpl(ByteBuffer buffer, int width, int height, int yPos, int strideY, int uPos, - int strideU, int vPos, int strideV, ReleaseCallback releaseCallback) { - this.buffer = buffer; + /** Constructs an I420Buffer backed by existing data. */ + I420BufferImpl(int width, int height, ByteBuffer dataY, int strideY, ByteBuffer dataU, + int strideU, ByteBuffer dataV, int strideV, Runnable releaseCallback) { this.width = width; this.height = height; - this.chromaHeight = (height + 1) / 2; - this.yPos = yPos; + this.dataY = dataY; + this.dataU = dataU; + this.dataV = dataV; this.strideY = strideY; - this.uPos = uPos; this.strideU = strideU; - this.vPos = vPos; this.strideV = strideV; this.releaseCallback = releaseCallback; @@ -54,9 +50,22 @@ class I420BufferImpl implements VideoFrame.I420Buffer { int yPos = 0; int uPos = yPos + width * height; int vPos = uPos + strideUV * chromaHeight; + ByteBuffer buffer = ByteBuffer.allocateDirect(width * height + 2 * strideUV * chromaHeight); - return new I420BufferImpl( - buffer, width, height, yPos, width, uPos, strideUV, vPos, strideUV, null); + + buffer.position(yPos); + buffer.limit(uPos); + ByteBuffer dataY = buffer.slice(); + + buffer.position(uPos); + buffer.limit(vPos); + ByteBuffer dataU = buffer.slice(); + + buffer.position(vPos); + buffer.limit(vPos + strideUV * chromaHeight); + ByteBuffer dataV = buffer.slice(); + + return new I420BufferImpl(width, height, dataY, width, dataU, strideUV, dataV, strideUV, null); } @Override @@ -71,26 +80,17 @@ class I420BufferImpl implements VideoFrame.I420Buffer { @Override public ByteBuffer getDataY() { - ByteBuffer data = buffer.slice(); - data.position(yPos); - data.limit(yPos + getStrideY() * height); - return data; + return dataY; } @Override public ByteBuffer getDataU() { - ByteBuffer data = buffer.slice(); - data.position(uPos); - data.limit(uPos + strideU * chromaHeight); - return data; + return dataU; } @Override public ByteBuffer getDataV() { - ByteBuffer data = buffer.slice(); - data.position(vPos); - data.limit(vPos + strideV * chromaHeight); - return data; + return dataV; } @Override @@ -121,13 +121,14 @@ class I420BufferImpl implements VideoFrame.I420Buffer { @Override public void release() { if (--refCount == 0 && releaseCallback != null) { - releaseCallback.onRelease(); + releaseCallback.run(); } } - // Callback called when the frame is no longer referenced. - interface ReleaseCallback { - // Called when the frame is no longer referenced. - void onRelease(); + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + return VideoFrame.cropAndScaleI420( + this, cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); } } diff --git a/webrtc/sdk/android/src/java/org/webrtc/TextureBufferImpl.java b/webrtc/sdk/android/src/java/org/webrtc/TextureBufferImpl.java new file mode 100644 index 0000000000..ebcb22ff9c --- /dev/null +++ b/webrtc/sdk/android/src/java/org/webrtc/TextureBufferImpl.java @@ -0,0 +1,131 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Matrix; +import java.nio.ByteBuffer; + +/** + * Android texture buffer backed by a SurfaceTextureHelper's texture. The buffer calls + * |releaseCallback| when it is released. + */ +class TextureBufferImpl implements VideoFrame.TextureBuffer { + private final int width; + private final int height; + private final Type type; + private final int id; + private final Matrix transformMatrix; + private final SurfaceTextureHelper surfaceTextureHelper; + private final Runnable releaseCallback; + private int refCount; + + public TextureBufferImpl(int width, int height, Type type, int id, Matrix transformMatrix, + SurfaceTextureHelper surfaceTextureHelper, Runnable releaseCallback) { + this.width = width; + this.height = height; + this.type = type; + this.id = id; + this.transformMatrix = transformMatrix; + this.surfaceTextureHelper = surfaceTextureHelper; + this.releaseCallback = releaseCallback; + this.refCount = 1; // Creator implicitly holds a reference. + } + + @Override + public VideoFrame.TextureBuffer.Type getType() { + return type; + } + + @Override + public int getTextureId() { + return id; + } + + @Override + public Matrix getTransformMatrix() { + return transformMatrix; + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getHeight() { + return height; + } + + @Override + public VideoFrame.I420Buffer toI420() { + // SurfaceTextureHelper requires a stride that is divisible by 8. Round width up. + // See SurfaceTextureHelper for details on the size and format. + int stride = ((width + 7) / 8) * 8; + int uvHeight = (height + 1) / 2; + // Due to the layout used by SurfaceTextureHelper, vPos + stride * uvHeight would overrun the + // buffer. Add one row at the bottom to compensate for this. There will never be data in the + // extra row, but now other code does not have to deal with v stride * v height exceeding the + // buffer's capacity. + int size = stride * (height + uvHeight + 1); + ByteBuffer buffer = ByteBuffer.allocateDirect(size); + surfaceTextureHelper.textureToYUV(buffer, width, height, stride, id, + RendererCommon.convertMatrixFromAndroidGraphicsMatrix(transformMatrix)); + + int yPos = 0; + int uPos = yPos + stride * height; + // Rows of U and V alternate in the buffer, so V data starts after the first row of U. + int vPos = uPos + stride / 2; + + buffer.position(yPos); + buffer.limit(yPos + stride * height); + ByteBuffer dataY = buffer.slice(); + + buffer.position(uPos); + buffer.limit(uPos + stride * uvHeight); + ByteBuffer dataU = buffer.slice(); + + buffer.position(vPos); + buffer.limit(vPos + stride * uvHeight); + ByteBuffer dataV = buffer.slice(); + + // SurfaceTextureHelper uses the same stride for Y, U, and V data. + return new I420BufferImpl(width, height, dataY, stride, dataU, stride, dataV, stride, null); + } + + @Override + public void retain() { + ++refCount; + } + + @Override + public void release() { + if (--refCount == 0) { + releaseCallback.run(); + } + } + + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + retain(); + Matrix newMatrix = new Matrix(transformMatrix); + newMatrix.postScale(cropWidth / (float) width, cropHeight / (float) height); + newMatrix.postTranslate(cropX / (float) width, cropY / (float) height); + + return new TextureBufferImpl( + scaleWidth, scaleHeight, type, id, newMatrix, surfaceTextureHelper, new Runnable() { + @Override + public void run() { + release(); + } + }); + } +} diff --git a/webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java b/webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java index 3beef16e60..0282afbbef 100644 --- a/webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java +++ b/webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java @@ -94,6 +94,13 @@ class WrappedNativeI420Buffer implements VideoFrame.I420Buffer { nativeRelease(nativeBuffer); } + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + return VideoFrame.cropAndScaleI420( + this, cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); + } + private static native long nativeAddRef(long nativeBuffer); private static native long nativeRelease(long nativeBuffer); } diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.cc b/webrtc/sdk/android/src/jni/native_handle_impl.cc index 562e7ec6f8..f04a0b82b3 100644 --- a/webrtc/sdk/android/src/jni/native_handle_impl.cc +++ b/webrtc/sdk/android/src/jni/native_handle_impl.cc @@ -34,42 +34,6 @@ Matrix::Matrix(JNIEnv* jni, jfloatArray a) { jni->ReleaseFloatArrayElements(a, ptr, 0); } -Matrix Matrix::fromAndroidGraphicsMatrix(JNIEnv* jni, jobject j_matrix) { - jfloatArray array_3x3 = jni->NewFloatArray(9); - jclass j_matrix_class = jni->FindClass("android/graphics/Matrix"); - jni->CallVoidMethod(j_matrix, - GetMethodID(jni, j_matrix_class, "getValues", "([F)V"), - array_3x3); - jfloat* array_3x3_ptr = jni->GetFloatArrayElements(array_3x3, nullptr); - Matrix matrix; - memset(matrix.elem_, 0, sizeof(matrix.elem_)); - // The android.graphics.Matrix looks like this: - // [x1 y1 w1] - // [x2 y2 w2] - // [x3 y3 w3] - // We want to contruct a matrix that looks like this: - // [x1 y1 0 w1] - // [x2 y2 0 w2] - // [ 0 0 1 0] - // [x3 y3 0 w3] - // Since it is stored in column-major order, it looks like this: - // [x1 x2 0 x3 - // y1 y2 0 y3 - // 0 0 1 0 - // w1 w2 0 w3] - matrix.elem_[0 * 4 + 0] = array_3x3_ptr[0 * 3 + 0]; - matrix.elem_[0 * 4 + 1] = array_3x3_ptr[1 * 3 + 0]; - matrix.elem_[0 * 4 + 3] = array_3x3_ptr[2 * 3 + 0]; - matrix.elem_[1 * 4 + 0] = array_3x3_ptr[0 * 3 + 1]; - matrix.elem_[1 * 4 + 1] = array_3x3_ptr[1 * 3 + 1]; - matrix.elem_[1 * 4 + 3] = array_3x3_ptr[2 * 3 + 1]; - matrix.elem_[2 * 4 + 2] = 1; // Z-scale should be 1. - matrix.elem_[3 * 4 + 0] = array_3x3_ptr[0 * 3 + 2]; - matrix.elem_[3 * 4 + 1] = array_3x3_ptr[1 * 3 + 2]; - matrix.elem_[3 * 4 + 3] = array_3x3_ptr[2 * 3 + 2]; - return matrix; -} - jfloatArray Matrix::ToJava(JNIEnv* jni) const { jfloatArray matrix = jni->NewFloatArray(16); jni->SetFloatArrayRegion(matrix, 0, 16, elem_); @@ -237,12 +201,10 @@ AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni, jmethodID j_release_id, int width, int height, - const Matrix& matrix, jobject j_video_frame_buffer) : j_release_id_(j_release_id), width_(width), height_(height), - matrix_(matrix), j_video_frame_buffer_(jni, j_video_frame_buffer) { jni->CallVoidMethod(j_video_frame_buffer, j_retain_id); } @@ -274,23 +236,19 @@ rtc::scoped_refptr AndroidVideoBuffer::ToI420() { } jobject AndroidVideoBuffer::ToJavaI420Frame(JNIEnv* jni, - int width, - int height, int rotation) { jclass j_byte_buffer_class = jni->FindClass("java/nio/ByteBuffer"); jclass j_i420_frame_class = FindClass(jni, "org/webrtc/VideoRenderer$I420Frame"); - jmethodID j_i420_frame_ctor_id = - GetMethodID(jni, j_i420_frame_class, "", - "(III[FLorg/webrtc/VideoFrame$Buffer;J)V"); + jmethodID j_i420_frame_ctor_id = GetMethodID( + jni, j_i420_frame_class, "", "(ILorg/webrtc/VideoFrame$Buffer;J)V"); // Java code just uses the native frame to hold a reference to the buffer so // this is okay. webrtc::VideoFrame* native_frame = new webrtc::VideoFrame( this, 0 /* timestamp */, 0 /* render_time_ms */, webrtc::VideoRotation::kVideoRotation_0 /* rotation */); - return jni->NewObject(j_i420_frame_class, j_i420_frame_ctor_id, width, height, - rotation, matrix_.ToJava(jni), *j_video_frame_buffer_, - jlongFromPointer(native_frame)); + return jni->NewObject(j_i420_frame_class, j_i420_frame_ctor_id, rotation, + *j_video_frame_buffer_, jlongFromPointer(native_frame)); } AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni) @@ -299,16 +257,8 @@ AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni) *j_video_frame_class_, "getBuffer", "()Lorg/webrtc/VideoFrame$Buffer;")), - j_get_width_id_( - GetMethodID(jni, *j_video_frame_class_, "getWidth", "()I")), - j_get_height_id_( - GetMethodID(jni, *j_video_frame_class_, "getHeight", "()I")), j_get_rotation_id_( GetMethodID(jni, *j_video_frame_class_, "getRotation", "()I")), - j_get_transform_matrix_id_(GetMethodID(jni, - *j_video_frame_class_, - "getTransformMatrix", - "()Landroid/graphics/Matrix;")), j_get_timestamp_ns_id_( GetMethodID(jni, *j_video_frame_class_, "getTimestampNs", "()J")), j_video_frame_buffer_class_( @@ -317,7 +267,11 @@ AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni) j_retain_id_( GetMethodID(jni, *j_video_frame_buffer_class_, "retain", "()V")), j_release_id_( - GetMethodID(jni, *j_video_frame_buffer_class_, "release", "()V")) {} + GetMethodID(jni, *j_video_frame_buffer_class_, "release", "()V")), + j_get_width_id_( + GetMethodID(jni, *j_video_frame_buffer_class_, "getWidth", "()I")), + j_get_height_id_( + GetMethodID(jni, *j_video_frame_buffer_class_, "getHeight", "()I")) {} webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame( JNIEnv* jni, @@ -325,30 +279,23 @@ webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame( uint32_t timestamp_rtp) const { jobject j_video_frame_buffer = jni->CallObjectMethod(j_video_frame, j_get_buffer_id_); - int width = jni->CallIntMethod(j_video_frame, j_get_width_id_); - int height = jni->CallIntMethod(j_video_frame, j_get_height_id_); int rotation = jni->CallIntMethod(j_video_frame, j_get_rotation_id_); - jobject j_matrix = - jni->CallObjectMethod(j_video_frame, j_get_transform_matrix_id_); - Matrix matrix = Matrix::fromAndroidGraphicsMatrix(jni, j_matrix); uint32_t timestamp_ns = jni->CallLongMethod(j_video_frame, j_get_timestamp_ns_id_); rtc::scoped_refptr buffer = - CreateBuffer(width, height, matrix, j_video_frame_buffer); + CreateBuffer(j_video_frame_buffer); return webrtc::VideoFrame(buffer, timestamp_rtp, timestamp_ns / rtc::kNumNanosecsPerMillisec, static_cast(rotation)); } rtc::scoped_refptr AndroidVideoBufferFactory::CreateBuffer( - int width, - int height, - const Matrix& matrix, jobject j_video_frame_buffer) const { JNIEnv* jni = AttachCurrentThreadIfNeeded(); + int width = jni->CallIntMethod(j_video_frame_buffer, j_get_width_id_); + int height = jni->CallIntMethod(j_video_frame_buffer, j_get_height_id_); return new rtc::RefCountedObject( - jni, j_retain_id_, j_release_id_, width, height, matrix, - j_video_frame_buffer); + jni, j_retain_id_, j_release_id_, width, height, j_video_frame_buffer); } } // namespace webrtc_jni diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.h b/webrtc/sdk/android/src/jni/native_handle_impl.h index f70ae3b97c..cd9ad9cf01 100644 --- a/webrtc/sdk/android/src/jni/native_handle_impl.h +++ b/webrtc/sdk/android/src/jni/native_handle_impl.h @@ -108,14 +108,13 @@ class AndroidVideoBuffer : public AndroidVideoFrameBuffer { jmethodID j_release_id, int width, int height, - const Matrix& matrix, jobject j_video_frame_buffer); ~AndroidVideoBuffer() override; jobject video_frame_buffer() const; // Returns an instance of VideoRenderer.I420Frame (deprecated) - jobject ToJavaI420Frame(JNIEnv* jni, int width, int height, int rotation); + jobject ToJavaI420Frame(JNIEnv* jni, int rotation); private: Type type() const override; @@ -129,7 +128,6 @@ class AndroidVideoBuffer : public AndroidVideoFrameBuffer { const jmethodID j_release_id_; const int width_; const int height_; - const Matrix matrix_; // Holds a VideoFrame.Buffer. ScopedGlobalRef j_video_frame_buffer_; }; @@ -143,23 +141,19 @@ class AndroidVideoBufferFactory { uint32_t timestamp_rtp) const; rtc::scoped_refptr CreateBuffer( - int width, - int height, - const Matrix& matrix, jobject j_video_frame_buffer) const; private: ScopedGlobalRef j_video_frame_class_; jmethodID j_get_buffer_id_; - jmethodID j_get_width_id_; - jmethodID j_get_height_id_; jmethodID j_get_rotation_id_; - jmethodID j_get_transform_matrix_id_; jmethodID j_get_timestamp_ns_id_; ScopedGlobalRef j_video_frame_buffer_class_; jmethodID j_retain_id_; jmethodID j_release_id_; + jmethodID j_get_width_id_; + jmethodID j_get_height_id_; }; } // namespace webrtc_jni diff --git a/webrtc/sdk/android/src/jni/video_renderer_jni.cc b/webrtc/sdk/android/src/jni/video_renderer_jni.cc index 1bea6395d7..4f6807ad1d 100644 --- a/webrtc/sdk/android/src/jni/video_renderer_jni.cc +++ b/webrtc/sdk/android/src/jni/video_renderer_jni.cc @@ -59,9 +59,7 @@ class JavaVideoRendererWrapper break; case AndroidVideoFrameBuffer::AndroidType::kJavaBuffer: j_frame = static_cast(android_buffer) - ->ToJavaI420Frame(jni(), video_frame.width(), - video_frame.height(), - video_frame.rotation()); + ->ToJavaI420Frame(jni(), video_frame.rotation()); break; default: RTC_NOTREACHED(); diff --git a/webrtc/sdk/android/src/jni/videoframe_jni.cc b/webrtc/sdk/android/src/jni/videoframe_jni.cc new file mode 100644 index 0000000000..d761fb954b --- /dev/null +++ b/webrtc/sdk/android/src/jni/videoframe_jni.cc @@ -0,0 +1,62 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "libyuv/scale.h" + +#include "webrtc/rtc_base/checks.h" + +namespace webrtc_jni { + +extern "C" JNIEXPORT void JNICALL +Java_org_webrtc_VideoFrame_nativeCropAndScaleI420(JNIEnv* jni, + jclass, + jobject j_src_y, + jint src_stride_y, + jobject j_src_u, + jint src_stride_u, + jobject j_src_v, + jint src_stride_v, + jint crop_x, + jint crop_y, + jint crop_width, + jint crop_height, + jobject j_dst_y, + jint dst_stride_y, + jobject j_dst_u, + jint dst_stride_u, + jobject j_dst_v, + jint dst_stride_v, + jint scale_width, + jint scale_height) { + uint8_t const* src_y = + static_cast(jni->GetDirectBufferAddress(j_src_y)); + uint8_t const* src_u = + static_cast(jni->GetDirectBufferAddress(j_src_u)); + uint8_t const* src_v = + static_cast(jni->GetDirectBufferAddress(j_src_v)); + uint8_t* dst_y = static_cast(jni->GetDirectBufferAddress(j_dst_y)); + uint8_t* dst_u = static_cast(jni->GetDirectBufferAddress(j_dst_u)); + uint8_t* dst_v = static_cast(jni->GetDirectBufferAddress(j_dst_v)); + + // Perform cropping using pointer arithmetic. + src_y += crop_x + crop_y * src_stride_y; + src_u += crop_x / 2 + crop_y / 2 * src_stride_u; + src_v += crop_x / 2 + crop_y / 2 * src_stride_v; + + bool ret = libyuv::I420Scale( + src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, crop_width, + crop_height, dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v, + dst_stride_v, scale_width, scale_height, libyuv::kFilterBox); + RTC_DCHECK_EQ(ret, 0) << "I420Scale failed"; +} + +} // namespace webrtc_jni