From 6507054db101d8bd8643408ede8b95266c55fd96 Mon Sep 17 00:00:00 2001 From: Magnus Jedvert Date: Thu, 14 Jun 2018 12:23:01 +0200 Subject: [PATCH] Android: Add tests for VideoFrame.Buffer.toI420() and cropAndScale() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL adds tests that are primarily targeting VideoFrame.Buffer.toI420() and cropAndScale(), but includes the whole chain for YuvConverter, GlRectDrawer, and VideoFrameDrawer. It also includes a couple of fixes to bugs that were exposed by the new tests. Bug: webrtc:9186, webrtc:9391 Change-Id: I5eb62979a8fd8def28c3cb2e82dcede57c42216f Reviewed-on: https://webrtc-review.googlesource.com/83163 Commit-Queue: Magnus Jedvert Reviewed-by: Sami Kalliomäki Cr-Commit-Position: refs/heads/master@{#23611} --- sdk/android/BUILD.gn | 1 + .../api/org/webrtc/JavaI420Buffer.java | 25 +- .../api/org/webrtc/TextureBufferImpl.java | 5 +- sdk/android/api/org/webrtc/VideoFrame.java | 5 +- sdk/android/api/org/webrtc/YuvConverter.java | 6 +- .../src/org/webrtc/VideoFrameBufferTest.java | 562 ++++++++++++++++++ .../org/webrtc/JavaVideoSourceTestHelper.java | 11 +- .../video/videosource_unittest.cc | 12 +- 8 files changed, 597 insertions(+), 30 deletions(-) create mode 100644 sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 34a4f279ad..1ec1d685f5 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -1048,6 +1048,7 @@ if (rtc_include_tests) { "instrumentationtests/src/org/webrtc/EglRendererTest.java", "instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java", "instrumentationtests/src/org/webrtc/GlRectDrawerTest.java", + "instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java", "instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java", "instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java", "instrumentationtests/src/org/webrtc/MediaCodecVideoEncoderTest.java", diff --git a/sdk/android/api/org/webrtc/JavaI420Buffer.java b/sdk/android/api/org/webrtc/JavaI420Buffer.java index 19dd3e91e2..7231fdab91 100644 --- a/sdk/android/api/org/webrtc/JavaI420Buffer.java +++ b/sdk/android/api/org/webrtc/JavaI420Buffer.java @@ -39,6 +39,15 @@ public class JavaI420Buffer implements VideoFrame.I420Buffer { this.refCountDelegate = new RefCountDelegate(releaseCallback); } + private static void checkCapacity(ByteBuffer data, int width, int height, int stride) { + // The last row does not necessarily need padding. + final int minCapacity = stride * (height - 1) + width; + if (data.capacity() < minCapacity) { + throw new IllegalArgumentException( + "Buffer must be at least " + minCapacity + " bytes, but was " + data.capacity()); + } + } + /** Wraps existing ByteBuffers into JavaI420Buffer object without copying the contents. */ public static JavaI420Buffer wrap(int width, int height, ByteBuffer dataY, int strideY, ByteBuffer dataU, int strideU, ByteBuffer dataV, int strideV, Runnable releaseCallback) { @@ -55,19 +64,11 @@ public class JavaI420Buffer implements VideoFrame.I420Buffer { dataU = dataU.slice(); dataV = dataV.slice(); + final int chromaWidth = (width + 1) / 2; final int chromaHeight = (height + 1) / 2; - final int minCapacityY = strideY * height; - final int minCapacityU = strideU * chromaHeight; - final int minCapacityV = strideV * chromaHeight; - if (dataY.capacity() < minCapacityY) { - throw new IllegalArgumentException("Y-buffer must be at least " + minCapacityY + " bytes."); - } - if (dataU.capacity() < minCapacityU) { - throw new IllegalArgumentException("U-buffer must be at least " + minCapacityU + " bytes."); - } - if (dataV.capacity() < minCapacityV) { - throw new IllegalArgumentException("V-buffer must be at least " + minCapacityV + " bytes."); - } + checkCapacity(dataY, width, height, strideY); + checkCapacity(dataU, chromaWidth, chromaHeight, strideU); + checkCapacity(dataV, chromaWidth, chromaHeight, strideV); return new JavaI420Buffer( width, height, dataY, strideY, dataU, strideU, dataV, strideV, releaseCallback); diff --git a/sdk/android/api/org/webrtc/TextureBufferImpl.java b/sdk/android/api/org/webrtc/TextureBufferImpl.java index 19036589ec..c34728a3b4 100644 --- a/sdk/android/api/org/webrtc/TextureBufferImpl.java +++ b/sdk/android/api/org/webrtc/TextureBufferImpl.java @@ -85,7 +85,10 @@ public class TextureBufferImpl implements VideoFrame.TextureBuffer { public VideoFrame.Buffer cropAndScale( int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { final Matrix newMatrix = new Matrix(transformMatrix); - newMatrix.preTranslate(cropX / (float) width, cropY / (float) height); + // In WebRTC, Y=0 is the top row, while in OpenGL Y=0 is the bottom row. This means that the Y + // direction is effectively reversed. + final int cropYFromBottom = height - (cropY + cropHeight); + newMatrix.preTranslate(cropX / (float) width, cropYFromBottom / (float) height); newMatrix.preScale(cropWidth / (float) width, cropHeight / (float) height); retain(); diff --git a/sdk/android/api/org/webrtc/VideoFrame.java b/sdk/android/api/org/webrtc/VideoFrame.java index b496f00910..6d19260471 100644 --- a/sdk/android/api/org/webrtc/VideoFrame.java +++ b/sdk/android/api/org/webrtc/VideoFrame.java @@ -204,9 +204,8 @@ public class VideoFrame implements RefCounted { dataV.position(cropX / 2 + cropY / 2 * buffer.getStrideV()); buffer.retain(); - return JavaI420Buffer.wrap(buffer.getWidth(), buffer.getHeight(), dataY.slice(), - buffer.getStrideY(), dataU.slice(), buffer.getStrideU(), dataV.slice(), - buffer.getStrideV(), buffer::release); + return JavaI420Buffer.wrap(scaleWidth, scaleHeight, dataY.slice(), buffer.getStrideY(), + dataU.slice(), buffer.getStrideU(), dataV.slice(), buffer.getStrideV(), buffer::release); } JavaI420Buffer newBuffer = JavaI420Buffer.allocate(scaleWidth, scaleHeight); diff --git a/sdk/android/api/org/webrtc/YuvConverter.java b/sdk/android/api/org/webrtc/YuvConverter.java index 2d8e54882c..f7922d6adc 100644 --- a/sdk/android/api/org/webrtc/YuvConverter.java +++ b/sdk/android/api/org/webrtc/YuvConverter.java @@ -286,7 +286,7 @@ public class YuvConverter { // Y'UV444 to RGB888, see // https://en.wikipedia.org/wiki/YUV#Y.27UV444_to_RGB888_conversion. // We use the ITU-R coefficients for U and V */ - GLES20.glUniform4f(coeffsLoc, 0.299f, 0.587f, 0.114f, 0.0f); + GLES20.glUniform4f(coeffsLoc, 0.2987856f, 0.5871095f, 0.1141049f, 0.0f); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); // Draw U @@ -294,12 +294,12 @@ public class YuvConverter { // Matrix * (1;0;0;0) / (width / 2). Note that opengl uses column major order. GLES20.glUniform2f( xUnitLoc, 2.0f * transformMatrix[0] / width, 2.0f * transformMatrix[1] / width); - GLES20.glUniform4f(coeffsLoc, -0.169f, -0.331f, 0.499f, 0.5f); + GLES20.glUniform4f(coeffsLoc, -0.168805420f, -0.3317003f, 0.5005057f, 0.5f); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); // Draw V GLES20.glViewport(stride / 8, height, uv_width, uv_height); - GLES20.glUniform4f(coeffsLoc, 0.499f, -0.418f, -0.0813f, 0.5f); + GLES20.glUniform4f(coeffsLoc, 0.4997964f, -0.4184672f, -0.0813292f, 0.5f); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glReadPixels( diff --git a/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java b/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java new file mode 100644 index 0000000000..62b81f0e7a --- /dev/null +++ b/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java @@ -0,0 +1,562 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import android.graphics.Matrix; +import android.graphics.SurfaceTexture; +import android.opengl.GLES20; +import android.os.Handler; +import android.os.HandlerThread; +import android.support.test.InstrumentationRegistry; +import android.support.test.filters.SmallTest; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.chromium.base.test.params.BaseJUnit4RunnerDelegate; +import org.chromium.base.test.params.ParameterAnnotations.ClassParameter; +import org.chromium.base.test.params.ParameterAnnotations.UseRunnerDelegate; +import org.chromium.base.test.params.ParameterSet; +import org.chromium.base.test.params.ParameterizedRunner; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Test VideoFrame buffers of different kind of formats: I420, RGB, OES, NV12, NV21, and verify + * toI420() and cropAndScale() behavior. Creating RGB/OES textures involves VideoFrameDrawer and + * GlRectDrawer and we are testing the full chain I420 -> OES/RGB texture -> I420, with and without + * cropping in the middle. Reading textures back to I420 also exercises the YuvConverter code. + */ +@RunWith(ParameterizedRunner.class) +@UseRunnerDelegate(BaseJUnit4RunnerDelegate.class) +public class VideoFrameBufferTest { + /** + * These tests are parameterized on this enum which represents the different VideoFrame.Buffers. + */ + private static enum BufferType { I420, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 } + + @ClassParameter private static List CLASS_PARAMS = new ArrayList<>(); + + static { + for (BufferType bufferType : BufferType.values()) { + // The parameterization framework can not send Enums, so we pass it as a string instead. + CLASS_PARAMS.add(new ParameterSet().value(bufferType.name()).name(bufferType.name())); + } + } + + @BeforeClass + public static void setUp() { + // Needed for JniCommon.nativeAllocateByteBuffer() to work, which is used from JavaI420Buffer. + System.loadLibrary("jingle_peerconnection_so"); + } + + private final BufferType bufferType; + + public VideoFrameBufferTest(String bufferTypeName) { + // Parse the string back to an enum. + bufferType = BufferType.valueOf(bufferTypeName); + } + + /** + * Create a VideoFrame.Buffer of the given type with the same pixel content as the given I420 + * buffer. + */ + private static VideoFrame.Buffer createBufferWithType( + BufferType bufferType, VideoFrame.I420Buffer i420Buffer) { + switch (bufferType) { + case I420: + return i420Buffer.toI420(); + case RGB_TEXTURE: + return createRgbTextureBuffer(/* eglContext= */ null, i420Buffer); + case OES_TEXTURE: + return createOesTextureBuffer(/* eglContext= */ null, i420Buffer); + case NV21: + return createNV21Buffer(i420Buffer); + case NV12: + return createNV12Buffer(i420Buffer); + default: + throw new IllegalArgumentException("Unknown buffer type: " + bufferType); + } + } + + private VideoFrame.Buffer createBufferToTest(VideoFrame.I420Buffer i420Buffer) { + return createBufferWithType(this.bufferType, i420Buffer); + } + + /** + * Creates a 16x16 I420 buffer that varies smoothly and spans all RGB values. + */ + public static VideoFrame.I420Buffer createTestI420Buffer() { + final int width = 16; + final int height = 16; + final int[] yData = new int[] {164, 170, 176, 182, 188, 194, 200, 206, 213, 218, 225, 231, 237, + 243, 249, 255, 153, 159, 165, 171, 177, 183, 189, 195, 201, 207, 213, 220, 226, 232, 238, + 244, 142, 148, 154, 160, 166, 172, 178, 184, 191, 196, 203, 209, 215, 221, 227, 233, 131, + 137, 143, 149, 155, 161, 167, 174, 180, 186, 192, 198, 204, 210, 216, 222, 120, 126, 132, + 138, 145, 151, 157, 163, 169, 175, 181, 187, 193, 199, 205, 211, 109, 115, 121, 128, 133, + 140, 146, 152, 158, 164, 170, 176, 182, 188, 194, 200, 99, 104, 111, 117, 123, 129, 135, + 141, 147, 153, 159, 165, 171, 177, 183, 189, 87, 94, 100, 106, 112, 118, 124, 130, 136, 142, + 148, 154, 160, 166, 172, 178, 77, 83, 89, 95, 101, 107, 113, 119, 125, 131, 137, 143, 149, + 155, 161, 167, 66, 72, 78, 84, 90, 96, 102, 108, 114, 120, 126, 132, 138, 144, 150, 156, 55, + 61, 67, 73, 79, 85, 91, 97, 103, 109, 115, 121, 127, 133, 139, 145, 44, 50, 56, 62, 68, 74, + 80, 86, 92, 98, 104, 110, 116, 122, 128, 134, 33, 39, 45, 51, 57, 63, 69, 75, 81, 87, 93, + 99, 105, 111, 117, 123, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94, 100, 106, 113, + 11, 17, 23, 29, 35, 41, 47, 53, 59, 65, 71, 77, 83, 89, 96, 102, 0, 6, 12, 18, 24, 30, 36, + 42, 48, 54, 60, 66, 72, 79, 85, 91}; + final int[] uData = new int[] {108, 111, 114, 117, 119, 122, 125, 128, 111, 114, 117, 119, 122, + 125, 128, 130, 114, 117, 119, 122, 125, 128, 130, 133, 117, 119, 122, 125, 128, 130, 133, + 136, 119, 122, 125, 128, 130, 133, 136, 139, 122, 125, 128, 130, 133, 136, 139, 141, 125, + 128, 130, 133, 136, 139, 141, 144, 128, 130, 133, 136, 139, 141, 144, 147}; + final int[] vData = new int[] {18, 34, 49, 65, 81, 96, 112, 127, 34, 49, 65, 81, 96, 112, 128, + 143, 49, 65, 81, 96, 112, 127, 143, 159, 65, 81, 96, 112, 127, 143, 159, 174, 81, 96, 112, + 128, 143, 159, 174, 190, 96, 112, 128, 143, 159, 174, 190, 206, 112, 127, 143, 159, 174, + 190, 205, 221, 127, 143, 159, 174, 190, 205, 221, 237}; + return JavaI420Buffer.wrap(width, height, toByteBuffer(yData), + /* strideY= */ width, toByteBuffer(uData), /* strideU= */ width / 2, toByteBuffer(vData), + /* strideV= */ width / 2, + /* releaseCallback= */ null); + } + + /** + * Create an RGB texture buffer available in |eglContext| with the same pixel content as the given + * I420 buffer. + */ + public static VideoFrame.TextureBuffer createRgbTextureBuffer( + EglBase.Context eglContext, VideoFrame.I420Buffer i420Buffer) { + final int width = i420Buffer.getWidth(); + final int height = i420Buffer.getHeight(); + + final HandlerThread renderThread = new HandlerThread("RGB texture thread"); + renderThread.start(); + final Handler renderThreadHandler = new Handler(renderThread.getLooper()); + return ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> { + // Create EGL base with a pixel buffer as display output. + final EglBase eglBase = EglBase.create(eglContext, EglBase.CONFIG_PIXEL_BUFFER); + eglBase.createDummyPbufferSurface(); + eglBase.makeCurrent(); + + final GlTextureFrameBuffer textureFrameBuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA); + textureFrameBuffer.setSize(width, height); + + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, textureFrameBuffer.getFrameBufferId()); + drawI420Buffer(i420Buffer); + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + + final YuvConverter yuvConverter = new YuvConverter(); + return new TextureBufferImpl(width, height, VideoFrame.TextureBuffer.Type.RGB, + textureFrameBuffer.getTextureId(), + /* transformMatrix= */ new Matrix(), renderThreadHandler, yuvConverter, + /* releaseCallback= */ () -> renderThreadHandler.post(() -> { + textureFrameBuffer.release(); + yuvConverter.release(); + eglBase.release(); + renderThread.quit(); + })); + }); + } + + /** + * Create an OES texture buffer available in |eglContext| with the same pixel content as the given + * I420 buffer. + */ + public static VideoFrame.TextureBuffer createOesTextureBuffer( + EglBase.Context eglContext, VideoFrame.I420Buffer i420Buffer) { + final int width = i420Buffer.getWidth(); + final int height = i420Buffer.getHeight(); + + // Create resources for generating OES textures. + final SurfaceTextureHelper surfaceTextureHelper = + SurfaceTextureHelper.create("SurfaceTextureHelper test", eglContext); + final SurfaceTexture surfaceTexture = surfaceTextureHelper.getSurfaceTexture(); + surfaceTexture.setDefaultBufferSize(width, height); + + final HandlerThread renderThread = new HandlerThread("OES texture thread"); + renderThread.start(); + final Handler renderThreadHandler = new Handler(renderThread.getLooper()); + final VideoFrame.TextureBuffer oesBuffer = + ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> { + // Create EGL base with the SurfaceTexture as display output. + final EglBase eglBase = EglBase.create(eglContext, EglBase.CONFIG_PLAIN); + eglBase.createSurface(surfaceTexture); + eglBase.makeCurrent(); + assertEquals(width, eglBase.surfaceWidth()); + assertEquals(height, eglBase.surfaceHeight()); + + final SurfaceTextureHelperTest.MockTextureListener listener = + new SurfaceTextureHelperTest.MockTextureListener(); + surfaceTextureHelper.startListening(listener); + + // Draw the frame and block until an OES texture is delivered. + drawI420Buffer(i420Buffer); + eglBase.swapBuffers(); + listener.waitForNewFrame(); + surfaceTextureHelper.stopListening(); + surfaceTextureHelper.dispose(); + + return surfaceTextureHelper.createTextureBuffer(width, height, + RendererCommon.convertMatrixToAndroidGraphicsMatrix(listener.transformMatrix)); + }); + + // Wrap |oesBuffer| to quit |renderThread| once |oesBuffer| is released. + return new VideoFrame.TextureBuffer() { + private final RefCountDelegate refCountDelegate = new RefCountDelegate(() -> { + oesBuffer.release(); + renderThread.quit(); + }); + + @Override + public void retain() { + refCountDelegate.retain(); + } + + @Override + public void release() { + refCountDelegate.release(); + } + + @Override + public VideoFrame.TextureBuffer.Type getType() { + return oesBuffer.getType(); + } + + @Override + public int getTextureId() { + return oesBuffer.getTextureId(); + } + + @Override + public Matrix getTransformMatrix() { + return oesBuffer.getTransformMatrix(); + } + + @Override + public int getWidth() { + return oesBuffer.getWidth(); + } + + @Override + public int getHeight() { + return oesBuffer.getHeight(); + } + + @Override + public VideoFrame.I420Buffer toI420() { + return oesBuffer.toI420(); + } + + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + return oesBuffer.cropAndScale(cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); + } + }; + } + + /** Create an NV21Buffer with the same pixel content as the given I420 buffer. */ + public static NV21Buffer createNV21Buffer(VideoFrame.I420Buffer i420Buffer) { + final int width = i420Buffer.getWidth(); + final int height = i420Buffer.getHeight(); + final int chromaStride = width; + final int chromaWidth = (width + 1) / 2; + final int chromaHeight = (height + 1) / 2; + final int ySize = width * height; + + final ByteBuffer nv21Buffer = ByteBuffer.allocateDirect(ySize + chromaStride * chromaHeight); + // We don't care what the array offset is since we only want an array that is direct. + @SuppressWarnings("ByteBufferBackingArray") final byte[] nv21Data = nv21Buffer.array(); + + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + final byte yValue = i420Buffer.getDataY().get(y * i420Buffer.getStrideY() + x); + nv21Data[y * width + x] = yValue; + } + } + for (int y = 0; y < chromaHeight; ++y) { + for (int x = 0; x < chromaWidth; ++x) { + final byte uValue = i420Buffer.getDataU().get(y * i420Buffer.getStrideU() + x); + final byte vValue = i420Buffer.getDataV().get(y * i420Buffer.getStrideV() + x); + nv21Data[ySize + y * chromaStride + 2 * x + 0] = vValue; + nv21Data[ySize + y * chromaStride + 2 * x + 1] = uValue; + } + } + return new NV21Buffer(nv21Data, width, height, /* releaseCallback= */ null); + } + + /** Create an NV12Buffer with the same pixel content as the given I420 buffer. */ + public static NV12Buffer createNV12Buffer(VideoFrame.I420Buffer i420Buffer) { + final int width = i420Buffer.getWidth(); + final int height = i420Buffer.getHeight(); + final int chromaStride = width; + final int chromaWidth = (width + 1) / 2; + final int chromaHeight = (height + 1) / 2; + final int ySize = width * height; + + final ByteBuffer nv12Buffer = ByteBuffer.allocateDirect(ySize + chromaStride * chromaHeight); + + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + final byte yValue = i420Buffer.getDataY().get(y * i420Buffer.getStrideY() + x); + nv12Buffer.put(y * width + x, yValue); + } + } + for (int y = 0; y < chromaHeight; ++y) { + for (int x = 0; x < chromaWidth; ++x) { + final byte uValue = i420Buffer.getDataU().get(y * i420Buffer.getStrideU() + x); + final byte vValue = i420Buffer.getDataV().get(y * i420Buffer.getStrideV() + x); + nv12Buffer.put(ySize + y * chromaStride + 2 * x + 0, uValue); + nv12Buffer.put(ySize + y * chromaStride + 2 * x + 1, vValue); + } + } + return new NV12Buffer(width, height, /* stride= */ width, /* sliceHeight= */ height, nv12Buffer, + /* releaseCallback */ null); + } + + /** Print the ByteBuffer plane to the StringBuilder. */ + private static void printPlane( + StringBuilder stringBuilder, int width, int height, ByteBuffer plane, int stride) { + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + final int value = plane.get(y * stride + x) & 0xFF; + if (x != 0) { + stringBuilder.append(", "); + } + stringBuilder.append(value); + } + stringBuilder.append("\n"); + } + } + + /** Convert the pixel content of an I420 buffer to a string representation. */ + private static String i420BufferToString(VideoFrame.I420Buffer buffer) { + final StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append( + "I420 buffer with size: " + buffer.getWidth() + "x" + buffer.getHeight() + ".\n"); + stringBuilder.append("Y-plane:\n"); + printPlane(stringBuilder, buffer.getWidth(), buffer.getHeight(), buffer.getDataY(), + buffer.getStrideY()); + final int chromaWidth = (buffer.getWidth() + 1) / 2; + final int chromaHeight = (buffer.getHeight() + 1) / 2; + stringBuilder.append("U-plane:\n"); + printPlane(stringBuilder, chromaWidth, chromaHeight, buffer.getDataU(), buffer.getStrideU()); + stringBuilder.append("V-plane:\n"); + printPlane(stringBuilder, chromaWidth, chromaHeight, buffer.getDataV(), buffer.getStrideV()); + return stringBuilder.toString(); + } + + /** + * Assert that the given I420 buffers are almost identical, allowing for some difference due to + * numerical errors. It has limits for both overall PSNR and maximum individual pixel difference. + */ + public static void assertAlmostEqualI420Buffers( + VideoFrame.I420Buffer bufferA, VideoFrame.I420Buffer bufferB) { + final int diff = maxDiff(bufferA, bufferB); + assertThat("Pixel difference too high: " + diff + "." + + "\nBuffer A: " + i420BufferToString(bufferA) + + "Buffer B: " + i420BufferToString(bufferB), + diff, lessThanOrEqualTo(4)); + final double psnr = calculatePsnr(bufferA, bufferB); + assertThat("PSNR too low: " + psnr + "." + + "\nBuffer A: " + i420BufferToString(bufferA) + + "Buffer B: " + i420BufferToString(bufferB), + psnr, greaterThanOrEqualTo(50.0)); + } + + /** Returns a flattened list of pixel differences for two ByteBuffer planes. */ + private static List getPixelDiffs( + int width, int height, ByteBuffer planeA, int strideA, ByteBuffer planeB, int strideB) { + List res = new ArrayList<>(); + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + final int valueA = planeA.get(y * strideA + x) & 0xFF; + final int valueB = planeB.get(y * strideB + x) & 0xFF; + res.add(Math.abs(valueA - valueB)); + } + } + return res; + } + + /** Returns a flattened list of pixel differences for two I420 buffers. */ + private static List getPixelDiffs( + VideoFrame.I420Buffer bufferA, VideoFrame.I420Buffer bufferB) { + assertEquals(bufferA.getWidth(), bufferB.getWidth()); + assertEquals(bufferA.getHeight(), bufferB.getHeight()); + final int width = bufferA.getWidth(); + final int height = bufferA.getHeight(); + final int chromaWidth = (width + 1) / 2; + final int chromaHeight = (height + 1) / 2; + final List diffs = getPixelDiffs(width, height, bufferA.getDataY(), + bufferA.getStrideY(), bufferB.getDataY(), bufferB.getStrideY()); + diffs.addAll(getPixelDiffs(chromaWidth, chromaHeight, bufferA.getDataU(), bufferA.getStrideU(), + bufferB.getDataU(), bufferB.getStrideU())); + diffs.addAll(getPixelDiffs(chromaWidth, chromaHeight, bufferA.getDataV(), bufferA.getStrideV(), + bufferB.getDataV(), bufferB.getStrideV())); + return diffs; + } + + /** Returns the maximum pixel difference from any of the Y/U/V planes in the given buffers. */ + private static int maxDiff(VideoFrame.I420Buffer bufferA, VideoFrame.I420Buffer bufferB) { + return Collections.max(getPixelDiffs(bufferA, bufferB)); + } + + /** + * Returns the PSNR given a sum of squared error and the number of measurements that were added. + */ + private static double sseToPsnr(long sse, int count) { + if (sse == 0) { + return Double.POSITIVE_INFINITY; + } + final double meanSquaredError = (double) sse / (double) count; + final double maxPixelValue = 255.0; + return 10.0 * Math.log10(maxPixelValue * maxPixelValue / meanSquaredError); + } + + /** Returns the PSNR of the given I420 buffers. */ + private static double calculatePsnr( + VideoFrame.I420Buffer bufferA, VideoFrame.I420Buffer bufferB) { + final List pixelDiffs = getPixelDiffs(bufferA, bufferB); + long sse = 0; + for (int pixelDiff : pixelDiffs) { + sse += pixelDiff * pixelDiff; + } + return sseToPsnr(sse, pixelDiffs.size()); + } + + /** + * Convert an int array to a byte array and make sure the values are within the range [0, 255]. + */ + private static byte[] toByteArray(int[] array) { + final byte[] res = new byte[array.length]; + for (int i = 0; i < array.length; ++i) { + final int value = array[i]; + assertThat(value, greaterThanOrEqualTo(0)); + assertThat(value, lessThanOrEqualTo(255)); + res[i] = (byte) value; + } + return res; + } + + /** Convert a byte array to a direct ByteBuffer. */ + private static ByteBuffer toByteBuffer(int[] array) { + final ByteBuffer buffer = ByteBuffer.allocateDirect(array.length); + buffer.put(toByteArray(array)); + buffer.rewind(); + return buffer; + } + + /** + * Draw an I420 buffer on the currently bound frame buffer, allocating and releasing any + * resources necessary. + */ + private static void drawI420Buffer(VideoFrame.I420Buffer i420Buffer) { + final GlRectDrawer drawer = new GlRectDrawer(); + final VideoFrameDrawer videoFrameDrawer = new VideoFrameDrawer(); + videoFrameDrawer.drawFrame( + new VideoFrame(i420Buffer, /* rotation= */ 0, /* timestampNs= */ 0), drawer); + videoFrameDrawer.release(); + drawer.release(); + } + + /** + * Helper function that tests cropAndScale() with the given cropping and scaling parameters, and + * compares the pixel content against a reference I420 buffer. + */ + private void testCropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + final VideoFrame.I420Buffer referenceI420Buffer = createTestI420Buffer(); + final VideoFrame.Buffer bufferToTest = createBufferToTest(referenceI420Buffer); + + final VideoFrame.Buffer croppedReferenceBuffer = referenceI420Buffer.cropAndScale( + cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); + referenceI420Buffer.release(); + final VideoFrame.I420Buffer croppedReferenceI420Buffer = croppedReferenceBuffer.toI420(); + croppedReferenceBuffer.release(); + + final VideoFrame.Buffer croppedBufferToTest = + bufferToTest.cropAndScale(cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); + bufferToTest.release(); + + final VideoFrame.I420Buffer croppedOutputI420Buffer = croppedBufferToTest.toI420(); + croppedBufferToTest.release(); + + assertAlmostEqualI420Buffers(croppedReferenceI420Buffer, croppedOutputI420Buffer); + croppedReferenceI420Buffer.release(); + croppedOutputI420Buffer.release(); + } + + @Test + @SmallTest + /** Test calling toI420() and comparing pixel content against I420 reference. */ + public void testToI420() { + final VideoFrame.I420Buffer referenceI420Buffer = createTestI420Buffer(); + final VideoFrame.Buffer bufferToTest = createBufferToTest(referenceI420Buffer); + + final VideoFrame.I420Buffer outputI420Buffer = bufferToTest.toI420(); + bufferToTest.release(); + + assertAlmostEqualI420Buffers(referenceI420Buffer, outputI420Buffer); + referenceI420Buffer.release(); + outputI420Buffer.release(); + } + + @Test + @SmallTest + /** Pure 2x scaling with no cropping. */ + public void testScale2x() { + testCropAndScale(0 /* cropX= */, 0 /* cropY= */, /* cropWidth= */ 16, /* cropHeight= */ 16, + /* scaleWidth= */ 8, /* scaleHeight= */ 8); + } + + @Test + @SmallTest + /** Test cropping only X direction, with no scaling. */ + public void testCropX() { + testCropAndScale(8 /* cropX= */, 0 /* cropY= */, /* cropWidth= */ 8, /* cropHeight= */ 16, + /* scaleWidth= */ 8, /* scaleHeight= */ 16); + } + + @Test + @SmallTest + /** Test cropping only Y direction, with no scaling. */ + public void testCropY() { + testCropAndScale(0 /* cropX= */, 8 /* cropY= */, /* cropWidth= */ 16, /* cropHeight= */ 8, + /* scaleWidth= */ 16, /* scaleHeight= */ 8); + } + + @Test + @SmallTest + /** Test center crop, with no scaling. */ + public void testCenterCrop() { + testCropAndScale(4 /* cropX= */, 4 /* cropY= */, /* cropWidth= */ 8, /* cropHeight= */ 8, + /* scaleWidth= */ 8, /* scaleHeight= */ 8); + } + + @Test + @SmallTest + /** Test non-center crop for right bottom corner, with no scaling. */ + public void testRightBottomCornerCrop() { + testCropAndScale(8 /* cropX= */, 8 /* cropY= */, /* cropWidth= */ 8, /* cropHeight= */ 8, + /* scaleWidth= */ 8, /* scaleHeight= */ 8); + } + + @Test + @SmallTest + /** Test combined cropping and scaling. */ + public void testCropAndScale() { + testCropAndScale(4 /* cropX= */, 4 /* cropY= */, /* cropWidth= */ 12, /* cropHeight= */ 12, + /* scaleWidth= */ 8, /* scaleHeight= */ 8); + } +} diff --git a/sdk/android/native_unittests/org/webrtc/JavaVideoSourceTestHelper.java b/sdk/android/native_unittests/org/webrtc/JavaVideoSourceTestHelper.java index 56b9204fbe..d1aa9c6105 100644 --- a/sdk/android/native_unittests/org/webrtc/JavaVideoSourceTestHelper.java +++ b/sdk/android/native_unittests/org/webrtc/JavaVideoSourceTestHelper.java @@ -24,12 +24,9 @@ public class JavaVideoSourceTestHelper { } @CalledByNative - public static void deliverFrame(VideoCapturer.CapturerObserver observer) { - final int FRAME_WIDTH = 2; - final int FRAME_HEIGHT = 3; - final int FRAME_ROTATION = 180; - - observer.onFrameCaptured(new VideoFrame( - JavaI420Buffer.allocate(FRAME_WIDTH, FRAME_HEIGHT), FRAME_ROTATION, 0 /* timestampNs */)); + public static void deliverFrame( + int width, int height, int rotation, VideoCapturer.CapturerObserver observer) { + observer.onFrameCaptured( + new VideoFrame(JavaI420Buffer.allocate(width, height), rotation, 0 /* timestampNs= */)); } } diff --git a/sdk/android/native_unittests/video/videosource_unittest.cc b/sdk/android/native_unittests/video/videosource_unittest.cc index 5365ababf6..b58ed1a45a 100644 --- a/sdk/android/native_unittests/video/videosource_unittest.cc +++ b/sdk/android/native_unittests/video/videosource_unittest.cc @@ -65,15 +65,19 @@ TEST(JavaVideoSourceTest, OnFrameCapturedFrameIsDeliveredToSink) { jni::Java_JavaVideoSourceTestHelper_startCapture( env, video_track_source->GetJavaVideoCapturerObserver(env), true /* success */); + const int width = 20; + const int height = 32; + const int rotation = 180; jni::Java_JavaVideoSourceTestHelper_deliverFrame( - env, video_track_source->GetJavaVideoCapturerObserver(env)); + env, width, height, rotation, + video_track_source->GetJavaVideoCapturerObserver(env)); std::vector frames = test_video_sink.GetFrames(); ASSERT_EQ(1u, frames.size()); webrtc::VideoFrame frame = frames[0]; - EXPECT_EQ(2, frame.width()); - EXPECT_EQ(3, frame.height()); - EXPECT_EQ(180, frame.rotation()); + EXPECT_EQ(width, frame.width()); + EXPECT_EQ(height, frame.height()); + EXPECT_EQ(rotation, frame.rotation()); } TEST(JavaVideoSourceTest, CapturerStartedSuccessStateBecomesLive) {