From 529528cc36d2c34f886e34165a1635285db11b8a Mon Sep 17 00:00:00 2001 From: Magnus Jedvert Date: Wed, 9 Sep 2015 10:59:46 +0200 Subject: [PATCH] Android video rendering: Apply SurfaceTexture.getTransformationMatrix() This CL applies the transformation matrix instead of assuming it is always a vertical flip. BUG=webrtc:4968,webrtc:4742 R=hbos@webrtc.org, pbos@webrtc.org Review URL: https://codereview.webrtc.org/1318153007 . Cr-Commit-Position: refs/heads/master@{#9905} --- .../src/org/webrtc/RendererCommonTest.java | 111 ++++++++++-------- .../android/org/webrtc/RendererCommon.java | 86 ++++++++++---- .../android/org/webrtc/VideoRendererGui.java | 51 ++++---- 3 files changed, 153 insertions(+), 95 deletions(-) diff --git a/talk/app/webrtc/androidtests/src/org/webrtc/RendererCommonTest.java b/talk/app/webrtc/androidtests/src/org/webrtc/RendererCommonTest.java index 28c0f283c3..cd8bfcb48c 100644 --- a/talk/app/webrtc/androidtests/src/org/webrtc/RendererCommonTest.java +++ b/talk/app/webrtc/androidtests/src/org/webrtc/RendererCommonTest.java @@ -35,7 +35,8 @@ import android.graphics.Point; import static org.webrtc.RendererCommon.ScalingType.*; import static org.webrtc.RendererCommon.getDisplaySize; -import static org.webrtc.RendererCommon.getTextureMatrix; +import static org.webrtc.RendererCommon.getLayoutMatrix; +import static org.webrtc.RendererCommon.getSamplingMatrix; public class RendererCommonTest extends ActivityTestCase { @SmallTest @@ -100,64 +101,76 @@ public class RendererCommonTest extends ActivityTestCase { return doubleArray; } + // Brief summary about matrix transformations: + // A coordinate p = [u, v, 0, 1] is transformed by matrix m like this p' = [u', v', 0, 1] = m * p. + // OpenGL uses column-major order, so: + // u' = u * m[0] + v * m[4] + m[12]. + // v' = u * m[1] + v * m[5] + m[13]. + @SmallTest - static public void testTexMatrixDefault() { - final float texMatrix[] = new float[16]; - getTextureMatrix(texMatrix, 0, false, 1.0f, 1.0f); - // TODO(magjed): Every tex matrix contains a vertical flip, because we ignore the texture - // transform matrix from the SurfaceTexture (which contains a vertical flip). Update tests when - // this is fixed. + static public void testLayoutMatrixDefault() { + final float layoutMatrix[] = getLayoutMatrix(false, 1.0f, 1.0f); + // Assert: + // u' = u. + // v' = v. + MoreAsserts.assertEquals(round(layoutMatrix), new double[] + {1, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0, + 0, 0, 0, 1}); + } + + @SmallTest + static public void testLayoutMatrixMirror() { + final float layoutMatrix[] = getLayoutMatrix(true, 1.0f, 1.0f); + // Assert: + // u' = 1 - u. + // v' = v. + MoreAsserts.assertEquals(round(layoutMatrix), new double[] + {-1, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0, + 1, 0, 0, 1}); + } + + @SmallTest + static public void testLayoutMatrixScale() { + // Video has aspect ratio 2, but layout is square. This will cause only the center part of the + // video to be visible, i.e. the u coordinate will go from 0.25 to 0.75 instead of from 0 to 1. + final float layoutMatrix[] = getLayoutMatrix(false, 2.0f, 1.0f); + // Assert: + // u' = 0.25 + 0.5 u. + // v' = v. + MoreAsserts.assertEquals(round(layoutMatrix), new double[] + { 0.5, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0, + 0.25, 0, 0, 1}); + } + + @SmallTest + static public void testSamplingMatrixDefault() { + final float samplingMatrix[] = getSamplingMatrix(null, 0); // Assert: // u' = u. // v' = 1 - v. - MoreAsserts.assertEquals(round(texMatrix), new double[] - {1, 0, 0, 0, + MoreAsserts.assertEquals(round(samplingMatrix), new double[] + {1, 0, 0, 0, 0, -1, 0, 0, - 0, 0, 1, 0, - 0, 1, 0, 1}); + 0, 0, 1, 0, + 0, 1, 0, 1}); } @SmallTest - static public void testTexMatrixMirror() { - final float texMatrix[] = new float[16]; - getTextureMatrix(texMatrix, 0, true, 1.0f, 1.0f); + static public void testSamplingMatrixRotation90Deg() { + final float samplingMatrix[] = getSamplingMatrix(null, 90); // Assert: // u' = 1 - u. // v' = 1 - v. - MoreAsserts.assertEquals(round(texMatrix), new double[] - {-1, 0, 0, 0, - 0, -1, 0, 0, - 0, 0, 1, 0, - 1, 1, 0, 1}); - } - - @SmallTest - static public void testTexMatrixRotation90Deg() { - final float texMatrix[] = new float[16]; - getTextureMatrix(texMatrix, 90, false, 1.0f, 1.0f); - // Assert: - // u' = 1 - v. - // v' = 1 - u. - MoreAsserts.assertEquals(round(texMatrix), new double[] - {0, -1, 0, 0, - -1, 0, 0, 0, - 0, 0, 1, 0, - 1, 1, 0, 1}); - } - - @SmallTest - static public void testTexMatrixScale() { - final float texMatrix[] = new float[16]; - // Video has aspect ratio 2, but layout is square. This will cause only the center part of the - // video to be visible, i.e. the u coordinate will go from 0.25 to 0.75 instead of from 0 to 1. - getTextureMatrix(texMatrix, 0, false, 2.0f, 1.0f); - // Assert: - // u' = 0.25 + 0.5 u. - // v' = 1 - v. - MoreAsserts.assertEquals(round(texMatrix), new double[] - {0.5, 0, 0, 0, - 0, -1, 0, 0, - 0, 0, 1, 0, - 0.25, 1, 0, 1}); + MoreAsserts.assertEquals(round(samplingMatrix), new double[] + { 0, -1, 0, 0, + -1, 0, 0, 0, + 0, 0, 1, 0, + 1, 1, 0, 1}); } } diff --git a/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java b/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java index 9f419da203..44b64151f5 100644 --- a/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java +++ b/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java @@ -28,6 +28,7 @@ package org.webrtc; import android.graphics.Point; +import android.graphics.SurfaceTexture; import android.opengl.Matrix; /** @@ -60,37 +61,62 @@ public class RendererCommon { // The minimum fraction of the frame content that will be shown for |SCALE_ASPECT_BALANCED|. // This limits excessive cropping when adjusting display size. private static float BALANCED_VISIBLE_FRACTION = 0.5625f; + // Matrix with transform y' = 1 - y. + private static final float[] VERTICAL_FLIP = new float[] { + 1, 0, 0, 0, + 0, -1, 0, 0, + 0, 0, 1, 0, + 0, 1, 0, 1}; /** - * Calculates a texture transformation matrix based on rotation, mirror, and video vs display - * aspect ratio. + * Returns matrix that transforms standard coordinates to their proper sampling locations in + * the texture. This transform compensates for any properties of the video source that + * cause it to appear different from a normalized texture. If the video source is based on + * ByteBuffers, pass null in |surfaceTexture|. */ - public static void getTextureMatrix(float[] outputTextureMatrix, float rotationDegree, - boolean mirror, float videoAspectRatio, float displayAspectRatio) { - // The matrix stack is using post-multiplication, which means that matrix operations: - // A; B; C; will end up as A * B * C. When you apply this to a vertex, it will result in: - // v' = A * B * C * v, i.e. the last matrix operation is the first thing that affects the - // vertex. This is the opposite of what you might expect. - Matrix.setIdentityM(outputTextureMatrix, 0); - // Move coordinates back to [0,1]x[0,1]. - Matrix.translateM(outputTextureMatrix, 0, 0.5f, 0.5f, 0.0f); - // Rotate frame clockwise in the XY-plane (around the Z-axis). - Matrix.rotateM(outputTextureMatrix, 0, -rotationDegree, 0, 0, 1); - // Scale one dimension until video and display size have same aspect ratio. - if (displayAspectRatio > videoAspectRatio) { - Matrix.scaleM(outputTextureMatrix, 0, 1, videoAspectRatio / displayAspectRatio, 1); + public static float[] getSamplingMatrix(SurfaceTexture surfaceTexture, float rotationDegree) { + final float[] samplingMatrix; + if (surfaceTexture == null) { + // For ByteBuffers, row 0 specifies the top row, but for a texture, row 0 specifies the + // bottom row. Flip the image vertically to compensate for this. + samplingMatrix = VERTICAL_FLIP; } else { - Matrix.scaleM(outputTextureMatrix, 0, displayAspectRatio / videoAspectRatio, 1, 1); + samplingMatrix = new float[16]; + surfaceTexture.getTransformMatrix(samplingMatrix); + } + // Clockwise rotation matrix in the XY-plane (around the Z-axis). + final float[] rotationMatrix = new float[16]; + Matrix.setRotateM(rotationMatrix, 0, -rotationDegree, 0, 0, 1); + adjustOrigin(rotationMatrix); + // Multiply matrices together. + final float[] tmpMatrix = new float[16]; + Matrix.multiplyMM(tmpMatrix, 0, rotationMatrix, 0, samplingMatrix, 0); + return tmpMatrix; + } + + /** + * Returns layout transformation matrix that applies an optional mirror effect and compensates + * for video vs display aspect ratio. + */ + public static float[] getLayoutMatrix( + boolean mirror, float videoAspectRatio, float displayAspectRatio) { + float scaleX = 1; + float scaleY = 1; + // Scale X or Y dimension so that video and display size have same aspect ratio. + if (displayAspectRatio > videoAspectRatio) { + scaleY = videoAspectRatio / displayAspectRatio; + } else { + scaleX = displayAspectRatio / videoAspectRatio; } - // TODO(magjed): We currently ignore the texture transform matrix from the SurfaceTexture. - // It contains a vertical flip that is hardcoded here instead. - Matrix.scaleM(outputTextureMatrix, 0, 1, -1, 1); // Apply optional horizontal flip. if (mirror) { - Matrix.scaleM(outputTextureMatrix, 0, -1, 1, 1); + scaleX *= -1; } - // Center coordinates around origin. - Matrix.translateM(outputTextureMatrix, 0, -0.5f, -0.5f, 0.0f); + final float matrix[] = new float[16]; + Matrix.setIdentityM(matrix, 0); + Matrix.scaleM(matrix, 0, scaleX, scaleY, 1); + adjustOrigin(matrix); + return matrix; } /** @@ -102,6 +128,20 @@ public class RendererCommon { maxDisplayWidth, maxDisplayHeight); } + /** + * Move |matrix| transformation origin to (0.5, 0.5). This is the origin for texture coordinates + * that are in the range 0 to 1. + */ + private static void adjustOrigin(float[] matrix) { + // Note that OpenGL is using column-major order. + // Pre translate with -0.5 to move coordinates to range [-0.5, 0.5]. + matrix[12] -= 0.5f * (matrix[0] + matrix[4]); + matrix[13] -= 0.5f * (matrix[1] + matrix[5]); + // Post translate with 0.5 to move coordinates to range [0, 1]. + matrix[12] += 0.5f; + matrix[13] += 0.5f; + } + /** * Each scaling type has a one-to-one correspondence to a numeric minimum fraction of the video * that must remain visible. diff --git a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java index 85680073fd..3072613fa4 100644 --- a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java +++ b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java @@ -41,6 +41,7 @@ import android.opengl.EGL14; import android.opengl.EGLContext; import android.opengl.GLES20; import android.opengl.GLSurfaceView; +import android.opengl.Matrix; import android.util.Log; import org.webrtc.VideoRenderer.I420Frame; @@ -133,12 +134,15 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { // The actual view area in pixels. It is a centered subrectangle of the rectangle defined by // |layoutInPercentage|. private final Rect displayLayout = new Rect(); - // Cached texture transformation matrix, calculated from current layout parameters. - private final float[] texMatrix = new float[16]; - // Flag if texture vertices or coordinates update is needed. - private boolean updateTextureProperties; - // Texture properties update lock. - private final Object updateTextureLock = new Object(); + // Cached layout transformation matrix, calculated from current layout parameters. + private float[] layoutMatrix; + // Flag if layout transformation matrix update is needed. + private boolean updateLayoutProperties; + // Layout properties update lock. Guards |updateLayoutProperties|, |screenWidth|, + // |screenHeight|, |videoWidth|, |videoHeight|, |rotationDegree|, |scalingType|, and |mirror|. + private final Object updateLayoutLock = new Object(); + // Texture sampling matrix. + private float[] samplingMatrix; // Viewport dimensions. private int screenWidth; private int screenHeight; @@ -160,7 +164,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { this.scalingType = scalingType; this.mirror = mirror; layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height)); - updateTextureProperties = false; + updateLayoutProperties = false; rotationDegree = 0; } @@ -184,9 +188,9 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { } } - private void checkAdjustTextureCoords() { - synchronized(updateTextureLock) { - if (!updateTextureProperties) { + private void updateLayoutMatrix() { + synchronized(updateLayoutLock) { + if (!updateLayoutProperties) { return; } // Initialize to maximum allowed area. Round to integer coordinates inwards the layout @@ -209,9 +213,9 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { (displayLayout.height() - displaySize.y) / 2); Log.d(TAG, " Adjusted display size: " + displayLayout.width() + " x " + displayLayout.height()); - RendererCommon.getTextureMatrix(texMatrix, rotationDegree, mirror, videoAspectRatio, - (float) displayLayout.width() / displayLayout.height()); - updateTextureProperties = false; + layoutMatrix = RendererCommon.getLayoutMatrix( + mirror, videoAspectRatio, (float) displayLayout.width() / displayLayout.height()); + updateLayoutProperties = false; Log.d(TAG, " AdjustTextureCoords done"); } } @@ -229,10 +233,6 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { final boolean isNewFrame; synchronized (pendingFrameLock) { - // Check if texture vertices/coordinates adjustment is required when - // screen orientation changes or video frame size changes. - checkAdjustTextureCoords(); - isNewFrame = (pendingFrame != null); if (isNewFrame && startTimeNs == -1) { startTimeNs = now; @@ -255,12 +255,17 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { surfaceTexture.updateTexImage(); } } + samplingMatrix = RendererCommon.getSamplingMatrix( + (SurfaceTexture) pendingFrame.textureObject, pendingFrame.rotationDegree); copyTimeNs += (System.nanoTime() - now); VideoRenderer.renderFrameDone(pendingFrame); pendingFrame = null; } } + updateLayoutMatrix(); + final float[] texMatrix = new float[16]; + Matrix.multiplyMM(texMatrix, 0, samplingMatrix, 0, layoutMatrix, 0); if (rendererType == RendererType.RENDERER_YUV) { drawer.drawYuv(yuvTextures, texMatrix); } else { @@ -291,7 +296,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { } public void setScreenSize(final int screenWidth, final int screenHeight) { - synchronized(updateTextureLock) { + synchronized(updateLayoutLock) { if (screenWidth == this.screenWidth && screenHeight == this.screenHeight) { return; } @@ -299,7 +304,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { screenWidth + " x " + screenHeight); this.screenWidth = screenWidth; this.screenHeight = screenHeight; - updateTextureProperties = true; + updateLayoutProperties = true; } } @@ -307,7 +312,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { RendererCommon.ScalingType scalingType, boolean mirror) { final Rect layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height)); - synchronized(updateTextureLock) { + synchronized(updateLayoutLock) { if (layoutInPercentage.equals(this.layoutInPercentage) && scalingType == this.scalingType && mirror == this.mirror) { return; @@ -318,7 +323,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { this.layoutInPercentage.set(layoutInPercentage); this.scalingType = scalingType; this.mirror = mirror; - updateTextureProperties = true; + updateLayoutProperties = true; } } @@ -333,14 +338,14 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { rendererEvents.onFrameResolutionChanged(videoWidth, videoHeight, rotation); } - synchronized (updateTextureLock) { + synchronized (updateLayoutLock) { Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setSize: " + videoWidth + " x " + videoHeight + " rotation " + rotation); this.videoWidth = videoWidth; this.videoHeight = videoHeight; rotationDegree = rotation; - updateTextureProperties = true; + updateLayoutProperties = true; Log.d(TAG, " YuvImageRenderer.setSize done."); } }