diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 41367132f6..eaa7715f75 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -69,6 +69,12 @@ rtc_library("video_frame") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } +if (is_android) { + java_cpp_enum("video_frame_enums") { + sources = [ "video_frame_buffer.h" ] + } +} + rtc_library("video_frame_i010") { visibility = [ "*" ] sources = [ diff --git a/api/video/video_frame_buffer.h b/api/video/video_frame_buffer.h index 62adc204f6..3e41a9b2ca 100644 --- a/api/video/video_frame_buffer.h +++ b/api/video/video_frame_buffer.h @@ -46,6 +46,8 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface { // New frame buffer types will be added conservatively when there is an // opportunity to optimize the path between some pair of video source and // video sink. + // GENERATED_JAVA_ENUM_PACKAGE: org.webrtc + // GENERATED_JAVA_CLASS_NAME_OVERRIDE: VideoFrameBufferType enum class Type { kNative, kI420, diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 57b9d629be..d4e8d182fe 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -202,7 +202,9 @@ if (is_android) { ":base_java", "//rtc_base:base_java", "//third_party/android_deps:com_android_support_support_annotations_java", + "//third_party/androidx:androidx_annotation_annotation_java", ] + srcjar_deps = [ "//api/video:video_frame_enums" ] } rtc_android_library("video_java") { @@ -1486,11 +1488,16 @@ if (is_android) { rtc_library("instrumentationtests_jni") { testonly = true - sources = [ "instrumentationtests/loggable_test.cc" ] + sources = [ + "instrumentationtests/loggable_test.cc", + "instrumentationtests/video_frame_buffer_test.cc", + ] deps = [ ":base_jni", ":native_api_jni", + ":videoframe_jni", + "../../api/video:video_frame", "../../rtc_base:rtc_base_approved", ] } diff --git a/sdk/android/api/org/webrtc/VideoFrame.java b/sdk/android/api/org/webrtc/VideoFrame.java index a0a0d4eecb..bb30069b92 100644 --- a/sdk/android/api/org/webrtc/VideoFrame.java +++ b/sdk/android/api/org/webrtc/VideoFrame.java @@ -34,6 +34,15 @@ public class VideoFrame implements RefCounted { * and the buffer needs to be returned to the VideoSource as soon as all references are gone. */ public interface Buffer extends RefCounted { + /** + * Representation of the underlying buffer. Currently, only NATIVE and I420 are supported. + */ + @CalledByNative("Buffer") + @VideoFrameBufferType + default int getBufferType() { + return VideoFrameBufferType.NATIVE; + } + /** * Resolution of the buffer in pixels. */ @@ -63,6 +72,11 @@ public class VideoFrame implements RefCounted { * Interface for I420 buffers. */ public interface I420Buffer extends Buffer { + @Override + default int getBufferType() { + return VideoFrameBufferType.I420; + } + /** * Returns a direct ByteBuffer containing Y-plane data. The buffer capacity is at least * getStrideY() * getHeight() bytes. The position of the returned buffer is ignored and must diff --git a/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java b/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java index 53b2c58e18..4388e4a60e 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java @@ -32,6 +32,7 @@ import org.chromium.base.test.params.ParameterizedRunner; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; +import org.webrtc.VideoFrame; /** * Test VideoFrame buffers of different kind of formats: I420, RGB, OES, NV12, NV21, and verify @@ -45,7 +46,7 @@ public class VideoFrameBufferTest { /** * These tests are parameterized on this enum which represents the different VideoFrame.Buffers. */ - private static enum BufferType { I420, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 } + private static enum BufferType { I420_JAVA, I420_NATIVE, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 } @ClassParameter private static List CLASS_PARAMS = new ArrayList<>(); @@ -75,17 +76,39 @@ public class VideoFrameBufferTest { */ private static VideoFrame.Buffer createBufferWithType( BufferType bufferType, VideoFrame.I420Buffer i420Buffer) { + VideoFrame.Buffer buffer; switch (bufferType) { - case I420: - return i420Buffer.toI420(); + case I420_JAVA: + buffer = i420Buffer; + buffer.retain(); + assertEquals(VideoFrameBufferType.I420, buffer.getBufferType()); + assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(buffer)); + return buffer; + case I420_NATIVE: + buffer = nativeGetNativeI420Buffer(i420Buffer); + assertEquals(VideoFrameBufferType.I420, buffer.getBufferType()); + assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(buffer)); + return buffer; case RGB_TEXTURE: - return createRgbTextureBuffer(/* eglContext= */ null, i420Buffer); + buffer = createRgbTextureBuffer(/* eglContext= */ null, i420Buffer); + assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType()); + assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer)); + return buffer; case OES_TEXTURE: - return createOesTextureBuffer(/* eglContext= */ null, i420Buffer); + buffer = createOesTextureBuffer(/* eglContext= */ null, i420Buffer); + assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType()); + assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer)); + return buffer; case NV21: - return createNV21Buffer(i420Buffer); + buffer = createNV21Buffer(i420Buffer); + assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType()); + assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer)); + return buffer; case NV12: - return createNV12Buffer(i420Buffer); + buffer = createNV12Buffer(i420Buffer); + assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType()); + assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer)); + return buffer; default: throw new IllegalArgumentException("Unknown buffer type: " + bufferType); } @@ -452,6 +475,7 @@ public class VideoFrameBufferTest { final VideoFrame.I420Buffer outputI420Buffer = bufferToTest.toI420(); bufferToTest.release(); + assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(outputI420Buffer)); assertAlmostEqualI420Buffers(referenceI420Buffer, outputI420Buffer); referenceI420Buffer.release(); outputI420Buffer.release(); @@ -504,4 +528,10 @@ public class VideoFrameBufferTest { testCropAndScale(4 /* cropX= */, 4 /* cropY= */, /* cropWidth= */ 12, /* cropHeight= */ 12, /* scaleWidth= */ 8, /* scaleHeight= */ 8); } + + @VideoFrameBufferType private static native int nativeGetBufferType(VideoFrame.Buffer buffer); + + /** Returns the copy of I420Buffer using WrappedNativeI420Buffer. */ + private static native VideoFrame.Buffer nativeGetNativeI420Buffer( + VideoFrame.I420Buffer i420Buffer); } diff --git a/sdk/android/instrumentationtests/video_frame_buffer_test.cc b/sdk/android/instrumentationtests/video_frame_buffer_test.cc new file mode 100644 index 0000000000..686b232f6d --- /dev/null +++ b/sdk/android/instrumentationtests/video_frame_buffer_test.cc @@ -0,0 +1,45 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/i420_buffer.h" +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/video_frame.h" +#include "sdk/android/src/jni/wrapped_native_i420_buffer.h" + +namespace webrtc { +namespace jni { + +JNI_FUNCTION_DECLARATION(jint, + VideoFrameBufferTest_nativeGetBufferType, + JNIEnv* jni, + jclass, + jobject video_frame_buffer) { + const JavaParamRef j_video_frame_buffer(video_frame_buffer); + rtc::scoped_refptr buffer = + JavaToNativeFrameBuffer(jni, j_video_frame_buffer); + return static_cast(buffer->type()); +} + +JNI_FUNCTION_DECLARATION(jobject, + VideoFrameBufferTest_nativeGetNativeI420Buffer, + JNIEnv* jni, + jclass, + jobject i420_buffer) { + const JavaParamRef j_i420_buffer(i420_buffer); + rtc::scoped_refptr buffer = + JavaToNativeFrameBuffer(jni, j_i420_buffer); + const I420BufferInterface* inputBuffer = buffer->GetI420(); + RTC_DCHECK(inputBuffer != nullptr); + rtc::scoped_refptr outputBuffer = I420Buffer::Copy(*inputBuffer); + return WrapI420Buffer(jni, outputBuffer).Release(); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/video_frame.cc b/sdk/android/src/jni/video_frame.cc index e27a5c841b..35d04d8b9f 100644 --- a/sdk/android/src/jni/video_frame.cc +++ b/sdk/android/src/jni/video_frame.cc @@ -70,6 +70,13 @@ class AndroidVideoBuffer : public VideoFrameBuffer { class AndroidVideoI420Buffer : public I420BufferInterface { public: + // Creates a native VideoFrameBuffer from a Java VideoFrame.I420Buffer. + static rtc::scoped_refptr Create( + JNIEnv* jni, + int width, + int height, + const JavaRef& j_video_frame_buffer); + // Adopts and takes ownership of the Java VideoFrame.Buffer. I.e. retain() // will not be called, but release() will be called when the returned // AndroidVideoBuffer is destroyed. @@ -113,11 +120,24 @@ class AndroidVideoI420Buffer : public I420BufferInterface { int stride_v_; }; +rtc::scoped_refptr AndroidVideoI420Buffer::Create( + JNIEnv* jni, + int width, + int height, + const JavaRef& j_video_frame_buffer) { + Java_Buffer_retain(jni, j_video_frame_buffer); + return AndroidVideoI420Buffer::Adopt(jni, width, height, + j_video_frame_buffer); +} + rtc::scoped_refptr AndroidVideoI420Buffer::Adopt( JNIEnv* jni, int width, int height, const JavaRef& j_video_frame_buffer) { + RTC_DCHECK_EQ( + static_cast(Java_Buffer_getBufferType(jni, j_video_frame_buffer)), + Type::kI420); return rtc::make_ref_counted(jni, width, height, j_video_frame_buffer); } @@ -164,6 +184,9 @@ int64_t GetJavaVideoFrameTimestampNs(JNIEnv* jni, rtc::scoped_refptr AndroidVideoBuffer::Adopt( JNIEnv* jni, const JavaRef& j_video_frame_buffer) { + RTC_DCHECK_EQ( + static_cast(Java_Buffer_getBufferType(jni, j_video_frame_buffer)), + Type::kNative); return rtc::make_ref_counted(jni, j_video_frame_buffer); } @@ -233,7 +256,20 @@ rtc::scoped_refptr AndroidVideoBuffer::ToI420() { rtc::scoped_refptr JavaToNativeFrameBuffer( JNIEnv* jni, const JavaRef& j_video_frame_buffer) { - return AndroidVideoBuffer::Create(jni, j_video_frame_buffer); + VideoFrameBuffer::Type type = static_cast( + Java_Buffer_getBufferType(jni, j_video_frame_buffer)); + switch (type) { + case VideoFrameBuffer::Type::kI420: { + const int width = Java_Buffer_getWidth(jni, j_video_frame_buffer); + const int height = Java_Buffer_getHeight(jni, j_video_frame_buffer); + return AndroidVideoI420Buffer::Create(jni, width, height, + j_video_frame_buffer); + } + case VideoFrameBuffer::Type::kNative: + return AndroidVideoBuffer::Create(jni, j_video_frame_buffer); + default: + RTC_CHECK_NOTREACHED(); + } } VideoFrame JavaToNativeFrame(JNIEnv* jni, @@ -243,8 +279,8 @@ VideoFrame JavaToNativeFrame(JNIEnv* jni, Java_VideoFrame_getBuffer(jni, j_video_frame); int rotation = Java_VideoFrame_getRotation(jni, j_video_frame); int64_t timestamp_ns = Java_VideoFrame_getTimestampNs(jni, j_video_frame); - rtc::scoped_refptr buffer = - AndroidVideoBuffer::Create(jni, j_video_frame_buffer); + rtc::scoped_refptr buffer = + JavaToNativeFrameBuffer(jni, j_video_frame_buffer); return VideoFrame::Builder() .set_video_frame_buffer(buffer) .set_timestamp_rtp(timestamp_rtp)