From 42a2fc9cbafb17b054c6e4a0891eb6abbc1de662 Mon Sep 17 00:00:00 2001 From: Rasmus Brandt Date: Mon, 9 Jul 2018 13:38:01 +0200 Subject: [PATCH] Support injecting custom native codecs from Java. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL extends our support for injecting native codecs such that downstream users can create Java codecs that are backed by custom native codecs. After this CL, the Java codec interfaces expose createNativeVideo{En,Decoder}() methods that may return a value representing a pointer to the backing native codec. Previously, a similar mechanism was used for the special case of non-public Java codecs extending from the internal WrappedNativeVideo{En,De}coder classes. Tested: AppRTCMobile on Pixel XL and Pixel 2. Bug: webrtc:9495 Change-Id: I079ff744afc7bf9873ff983e775c136a6667266d Reviewed-on: https://webrtc-review.googlesource.com/87264 Commit-Queue: Rasmus Brandt Reviewed-by: Sami Kalliomäki Cr-Commit-Position: refs/heads/master@{#23883} --- sdk/android/BUILD.gn | 4 -- sdk/android/api/org/webrtc/VideoDecoder.java | 22 +++++++ .../api/org/webrtc/VideoDecoderFallback.java | 2 +- sdk/android/api/org/webrtc/VideoEncoder.java | 30 ++++++++++ .../api/org/webrtc/VideoEncoderFallback.java | 6 +- .../src/java/org/webrtc/VP8Decoder.java | 2 +- .../src/java/org/webrtc/VP8Encoder.java | 6 +- .../src/java/org/webrtc/VP9Decoder.java | 2 +- .../src/java/org/webrtc/VP9Encoder.java | 6 +- .../org/webrtc/WrappedNativeVideoDecoder.java | 7 +-- .../org/webrtc/WrappedNativeVideoEncoder.java | 16 +---- .../src/jni/videodecoderfactorywrapper.cc | 2 +- sdk/android/src/jni/videodecoderfallback.cc | 2 +- sdk/android/src/jni/videodecoderwrapper.cc | 14 +++++ sdk/android/src/jni/videodecoderwrapper.h | 7 +++ .../src/jni/videoencoderfactorywrapper.cc | 4 +- sdk/android/src/jni/videoencoderfallback.cc | 2 +- sdk/android/src/jni/videoencoderwrapper.cc | 18 ++++++ sdk/android/src/jni/videoencoderwrapper.h | 9 +++ sdk/android/src/jni/wrappednativecodec.cc | 59 ------------------- sdk/android/src/jni/wrappednativecodec.h | 43 -------------- 21 files changed, 120 insertions(+), 143 deletions(-) delete mode 100644 sdk/android/src/jni/wrappednativecodec.cc delete mode 100644 sdk/android/src/jni/wrappednativecodec.h diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 83d5f64064..8cb8741095 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -282,8 +282,6 @@ generate_jni("generated_video_jni") { "src/java/org/webrtc/VideoDecoderWrapper.java", "src/java/org/webrtc/VideoEncoderWrapper.java", "src/java/org/webrtc/WrappedNativeI420Buffer.java", - "src/java/org/webrtc/WrappedNativeVideoDecoder.java", - "src/java/org/webrtc/WrappedNativeVideoEncoder.java", ] jni_package = "" namespace = "webrtc::jni" @@ -327,8 +325,6 @@ rtc_static_library("video_jni") { "src/jni/videotrack.cc", "src/jni/wrapped_native_i420_buffer.cc", "src/jni/wrapped_native_i420_buffer.h", - "src/jni/wrappednativecodec.cc", - "src/jni/wrappednativecodec.h", "src/jni/yuvhelper.cc", ] diff --git a/sdk/android/api/org/webrtc/VideoDecoder.java b/sdk/android/api/org/webrtc/VideoDecoder.java index 7bf9ac54d7..879b694b28 100644 --- a/sdk/android/api/org/webrtc/VideoDecoder.java +++ b/sdk/android/api/org/webrtc/VideoDecoder.java @@ -51,6 +51,28 @@ public interface VideoDecoder { void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp); } + /** + * The decoder implementation backing this interface is either 1) a Java + * decoder (e.g., an Android platform decoder), or alternatively 2) a native + * decoder (e.g., a software decoder or a C++ decoder adapter). + * + * For case 1), createNativeVideoDecoder() should return zero. + * In this case, we expect the native library to call the decoder through + * JNI using the Java interface declared below. + * + * For case 2), createNativeVideoDecoder() should return a non-zero value. + * In this case, we expect the native library to treat the returned value as + * a raw pointer of type webrtc::VideoDecoder* (ownership is transferred to + * the caller). The native library should then directly call the + * webrtc::VideoDecoder interface without going through JNI. All calls to + * the Java interface methods declared below should thus throw an + * UnsupportedOperationException. + */ + @CalledByNative + default long createNativeVideoDecoder() { + return 0; + } + /** * Initializes the decoding process with specified settings. Will be called on the decoding thread * before any decode calls. diff --git a/sdk/android/api/org/webrtc/VideoDecoderFallback.java b/sdk/android/api/org/webrtc/VideoDecoderFallback.java index 743a614621..ddfa3ecd40 100644 --- a/sdk/android/api/org/webrtc/VideoDecoderFallback.java +++ b/sdk/android/api/org/webrtc/VideoDecoderFallback.java @@ -23,7 +23,7 @@ public class VideoDecoderFallback extends WrappedNativeVideoDecoder { } @Override - long createNativeDecoder() { + public long createNativeVideoDecoder() { return nativeCreateDecoder(fallback, primary); } diff --git a/sdk/android/api/org/webrtc/VideoEncoder.java b/sdk/android/api/org/webrtc/VideoEncoder.java index 3001120801..b739e99aa6 100644 --- a/sdk/android/api/org/webrtc/VideoEncoder.java +++ b/sdk/android/api/org/webrtc/VideoEncoder.java @@ -163,6 +163,36 @@ public interface VideoEncoder { void onEncodedFrame(EncodedImage frame, CodecSpecificInfo info); } + /** + * The encoder implementation backing this interface is either 1) a Java + * encoder (e.g., an Android platform encoder), or alternatively 2) a native + * encoder (e.g., a software encoder or a C++ encoder adapter). + * + * For case 1), createNativeVideoEncoder() should return zero. + * In this case, we expect the native library to call the encoder through + * JNI using the Java interface declared below. + * + * For case 2), createNativeVideoEncoder() should return a non-zero value. + * In this case, we expect the native library to treat the returned value as + * a raw pointer of type webrtc::VideoEncoder* (ownership is transferred to + * the caller). The native library should then directly call the + * webrtc::VideoEncoder interface without going through JNI. All calls to + * the Java interface methods declared below should thus throw an + * UnsupportedOperationException. + */ + @CalledByNative + default long createNativeVideoEncoder() { + return 0; + } + + /** + * Returns true if the encoder is backed by hardware. + */ + @CalledByNative + default boolean isHardwareEncoder() { + return true; + } + /** * Initializes the encoding process. Call before any calls to encode. */ diff --git a/sdk/android/api/org/webrtc/VideoEncoderFallback.java b/sdk/android/api/org/webrtc/VideoEncoderFallback.java index cd45eb7520..fa36b7c989 100644 --- a/sdk/android/api/org/webrtc/VideoEncoderFallback.java +++ b/sdk/android/api/org/webrtc/VideoEncoderFallback.java @@ -23,13 +23,13 @@ public class VideoEncoderFallback extends WrappedNativeVideoEncoder { } @Override - long createNativeEncoder() { + public long createNativeVideoEncoder() { return nativeCreateEncoder(fallback, primary); } @Override - boolean isSoftwareEncoder() { - return isWrappedSoftwareEncoder(primary); + public boolean isHardwareEncoder() { + return primary.isHardwareEncoder(); } private static native long nativeCreateEncoder(VideoEncoder fallback, VideoEncoder primary); diff --git a/sdk/android/src/java/org/webrtc/VP8Decoder.java b/sdk/android/src/java/org/webrtc/VP8Decoder.java index c307edcf26..ab29c349c7 100644 --- a/sdk/android/src/java/org/webrtc/VP8Decoder.java +++ b/sdk/android/src/java/org/webrtc/VP8Decoder.java @@ -12,7 +12,7 @@ package org.webrtc; class VP8Decoder extends WrappedNativeVideoDecoder { @Override - long createNativeDecoder() { + public long createNativeVideoDecoder() { return nativeCreateDecoder(); } diff --git a/sdk/android/src/java/org/webrtc/VP8Encoder.java b/sdk/android/src/java/org/webrtc/VP8Encoder.java index f04c079566..c2f8f50417 100644 --- a/sdk/android/src/java/org/webrtc/VP8Encoder.java +++ b/sdk/android/src/java/org/webrtc/VP8Encoder.java @@ -12,14 +12,14 @@ package org.webrtc; class VP8Encoder extends WrappedNativeVideoEncoder { @Override - long createNativeEncoder() { + public long createNativeVideoEncoder() { return nativeCreateEncoder(); } static native long nativeCreateEncoder(); @Override - boolean isSoftwareEncoder() { - return true; + public boolean isHardwareEncoder() { + return false; } } diff --git a/sdk/android/src/java/org/webrtc/VP9Decoder.java b/sdk/android/src/java/org/webrtc/VP9Decoder.java index 5c7dc969eb..b6fee2d362 100644 --- a/sdk/android/src/java/org/webrtc/VP9Decoder.java +++ b/sdk/android/src/java/org/webrtc/VP9Decoder.java @@ -12,7 +12,7 @@ package org.webrtc; class VP9Decoder extends WrappedNativeVideoDecoder { @Override - long createNativeDecoder() { + public long createNativeVideoDecoder() { return nativeCreateDecoder(); } diff --git a/sdk/android/src/java/org/webrtc/VP9Encoder.java b/sdk/android/src/java/org/webrtc/VP9Encoder.java index 449eb7eb53..bc705bd9c6 100644 --- a/sdk/android/src/java/org/webrtc/VP9Encoder.java +++ b/sdk/android/src/java/org/webrtc/VP9Encoder.java @@ -12,15 +12,15 @@ package org.webrtc; class VP9Encoder extends WrappedNativeVideoEncoder { @Override - long createNativeEncoder() { + public long createNativeVideoEncoder() { return nativeCreateEncoder(); } static native long nativeCreateEncoder(); @Override - boolean isSoftwareEncoder() { - return true; + public boolean isHardwareEncoder() { + return false; } static native boolean nativeIsSupported(); diff --git a/sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java b/sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java index 9655eb90a7..80070d3ada 100644 --- a/sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java +++ b/sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java @@ -14,7 +14,7 @@ package org.webrtc; * Wraps a native webrtc::VideoDecoder. */ abstract class WrappedNativeVideoDecoder implements VideoDecoder { - @CalledByNative abstract long createNativeDecoder(); + @Override public abstract long createNativeVideoDecoder(); @Override public VideoCodecStatus initDecode(Settings settings, Callback decodeCallback) { @@ -40,9 +40,4 @@ abstract class WrappedNativeVideoDecoder implements VideoDecoder { public String getImplementationName() { throw new UnsupportedOperationException("Not implemented."); } - - @CalledByNative - static boolean isInstanceOf(VideoDecoder decoder) { - return decoder instanceof WrappedNativeVideoDecoder; - } } diff --git a/sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java b/sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java index c41b3b0212..07e8428621 100644 --- a/sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java +++ b/sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java @@ -14,9 +14,8 @@ package org.webrtc; * Wraps a native webrtc::VideoEncoder. */ abstract class WrappedNativeVideoEncoder implements VideoEncoder { - @CalledByNative abstract long createNativeEncoder(); - - abstract boolean isSoftwareEncoder(); + @Override public abstract long createNativeVideoEncoder(); + @Override public abstract boolean isHardwareEncoder(); @Override public VideoCodecStatus initEncode(Settings settings, Callback encodeCallback) { @@ -52,15 +51,4 @@ abstract class WrappedNativeVideoEncoder implements VideoEncoder { public String getImplementationName() { throw new UnsupportedOperationException("Not implemented."); } - - @CalledByNative - static boolean isWrappedSoftwareEncoder(VideoEncoder encoder) { - return (encoder instanceof WrappedNativeVideoEncoder) - && ((WrappedNativeVideoEncoder) encoder).isSoftwareEncoder(); - } - - @CalledByNative - static boolean isInstanceOf(VideoEncoder encoder) { - return encoder instanceof WrappedNativeVideoEncoder; - } } diff --git a/sdk/android/src/jni/videodecoderfactorywrapper.cc b/sdk/android/src/jni/videodecoderfactorywrapper.cc index 22bd17fae4..30640ea802 100644 --- a/sdk/android/src/jni/videodecoderfactorywrapper.cc +++ b/sdk/android/src/jni/videodecoderfactorywrapper.cc @@ -17,7 +17,7 @@ #include "sdk/android/generated_video_jni/jni/VideoDecoderFactory_jni.h" #include "sdk/android/native_api/jni/java_types.h" #include "sdk/android/src/jni/videocodecinfo.h" -#include "sdk/android/src/jni/wrappednativecodec.h" +#include "sdk/android/src/jni/videodecoderwrapper.h" namespace webrtc { namespace jni { diff --git a/sdk/android/src/jni/videodecoderfallback.cc b/sdk/android/src/jni/videodecoderfallback.cc index 66d7fdf224..abb042ff66 100644 --- a/sdk/android/src/jni/videodecoderfallback.cc +++ b/sdk/android/src/jni/videodecoderfallback.cc @@ -13,7 +13,7 @@ #include "api/video_codecs/video_decoder_software_fallback_wrapper.h" #include "sdk/android/generated_video_jni/jni/VideoDecoderFallback_jni.h" #include "sdk/android/src/jni/jni_helpers.h" -#include "sdk/android/src/jni/wrappednativecodec.h" +#include "sdk/android/src/jni/videodecoderwrapper.h" namespace webrtc { namespace jni { diff --git a/sdk/android/src/jni/videodecoderwrapper.cc b/sdk/android/src/jni/videodecoderwrapper.cc index 5fbd72fec4..7bd2b5ae46 100644 --- a/sdk/android/src/jni/videodecoderwrapper.cc +++ b/sdk/android/src/jni/videodecoderwrapper.cc @@ -263,5 +263,19 @@ absl::optional VideoDecoderWrapper::ParseQP( return qp; } +std::unique_ptr JavaToNativeVideoDecoder( + JNIEnv* jni, + const JavaRef& j_decoder) { + const jlong native_decoder = + Java_VideoDecoder_createNativeVideoDecoder(jni, j_decoder); + VideoDecoder* decoder; + if (native_decoder == 0) { + decoder = new VideoDecoderWrapper(jni, j_decoder); + } else { + decoder = reinterpret_cast(native_decoder); + } + return std::unique_ptr(decoder); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/videodecoderwrapper.h b/sdk/android/src/jni/videodecoderwrapper.h index c719aa4525..eb2958a908 100644 --- a/sdk/android/src/jni/videodecoderwrapper.h +++ b/sdk/android/src/jni/videodecoderwrapper.h @@ -110,6 +110,13 @@ class VideoDecoderWrapper : public VideoDecoder { RTC_GUARDED_BY(frame_extra_infos_lock_); }; +/* If the j_decoder is a wrapped native decoder, unwrap it. If it is not, + * wrap it in a VideoDecoderWrapper. + */ +std::unique_ptr JavaToNativeVideoDecoder( + JNIEnv* jni, + const JavaRef& j_decoder); + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/videoencoderfactorywrapper.cc b/sdk/android/src/jni/videoencoderfactorywrapper.cc index 2c3d111479..44026662db 100644 --- a/sdk/android/src/jni/videoencoderfactorywrapper.cc +++ b/sdk/android/src/jni/videoencoderfactorywrapper.cc @@ -17,7 +17,7 @@ #include "sdk/android/native_api/jni/class_loader.h" #include "sdk/android/native_api/jni/java_types.h" #include "sdk/android/src/jni/videocodecinfo.h" -#include "sdk/android/src/jni/wrappednativecodec.h" +#include "sdk/android/src/jni/videoencoderwrapper.h" namespace webrtc { namespace jni { @@ -60,7 +60,7 @@ VideoEncoderFactory::CodecInfo VideoEncoderFactoryWrapper::QueryVideoEncoder( CodecInfo codec_info; // Check if this is a wrapped native software encoder implementation. - codec_info.is_hardware_accelerated = !IsWrappedSoftwareEncoder(jni, encoder); + codec_info.is_hardware_accelerated = IsHardwareVideoEncoder(jni, encoder); codec_info.has_internal_source = false; return codec_info; } diff --git a/sdk/android/src/jni/videoencoderfallback.cc b/sdk/android/src/jni/videoencoderfallback.cc index 55c5dd2afc..88ea3df18c 100644 --- a/sdk/android/src/jni/videoencoderfallback.cc +++ b/sdk/android/src/jni/videoencoderfallback.cc @@ -13,7 +13,7 @@ #include "api/video_codecs/video_encoder_software_fallback_wrapper.h" #include "sdk/android/generated_video_jni/jni/VideoEncoderFallback_jni.h" #include "sdk/android/src/jni/jni_helpers.h" -#include "sdk/android/src/jni/wrappednativecodec.h" +#include "sdk/android/src/jni/videoencoderwrapper.h" namespace webrtc { namespace jni { diff --git a/sdk/android/src/jni/videoencoderwrapper.cc b/sdk/android/src/jni/videoencoderwrapper.cc index 94719ead16..9500a1f7a9 100644 --- a/sdk/android/src/jni/videoencoderwrapper.cc +++ b/sdk/android/src/jni/videoencoderwrapper.cc @@ -453,5 +453,23 @@ std::string VideoEncoderWrapper::GetImplementationName(JNIEnv* jni) const { jni, Java_VideoEncoder_getImplementationName(jni, encoder_)); } +std::unique_ptr JavaToNativeVideoEncoder( + JNIEnv* jni, + const JavaRef& j_encoder) { + const jlong native_encoder = + Java_VideoEncoder_createNativeVideoEncoder(jni, j_encoder); + VideoEncoder* encoder; + if (native_encoder == 0) { + encoder = new VideoEncoderWrapper(jni, j_encoder); + } else { + encoder = reinterpret_cast(native_encoder); + } + return std::unique_ptr(encoder); +} + +bool IsHardwareVideoEncoder(JNIEnv* jni, const JavaRef& j_encoder) { + return Java_VideoEncoder_isHardwareEncoder(jni, j_encoder); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/videoencoderwrapper.h b/sdk/android/src/jni/videoencoderwrapper.h index e350cadb7c..c30560503c 100644 --- a/sdk/android/src/jni/videoencoderwrapper.h +++ b/sdk/android/src/jni/videoencoderwrapper.h @@ -112,6 +112,15 @@ class VideoEncoderWrapper : public VideoEncoder { size_t gof_idx_; }; +/* If the j_encoder is a wrapped native encoder, unwrap it. If it is not, + * wrap it in a VideoEncoderWrapper. + */ +std::unique_ptr JavaToNativeVideoEncoder( + JNIEnv* jni, + const JavaRef& j_encoder); + +bool IsHardwareVideoEncoder(JNIEnv* jni, const JavaRef& j_encoder); + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/wrappednativecodec.cc b/sdk/android/src/jni/wrappednativecodec.cc deleted file mode 100644 index 20a2a601c1..0000000000 --- a/sdk/android/src/jni/wrappednativecodec.cc +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "sdk/android/src/jni/wrappednativecodec.h" - -#include "sdk/android/generated_video_jni/jni/WrappedNativeVideoDecoder_jni.h" -#include "sdk/android/generated_video_jni/jni/WrappedNativeVideoEncoder_jni.h" -#include "sdk/android/native_api/jni/class_loader.h" -#include "sdk/android/src/jni/jni_helpers.h" -#include "sdk/android/src/jni/videodecoderwrapper.h" -#include "sdk/android/src/jni/videoencoderwrapper.h" - -namespace webrtc { -namespace jni { - -std::unique_ptr JavaToNativeVideoDecoder( - JNIEnv* jni, - const JavaRef& j_decoder) { - VideoDecoder* decoder; - if (Java_WrappedNativeVideoDecoder_isInstanceOf(jni, j_decoder)) { - jlong native_decoder = - Java_WrappedNativeVideoDecoder_createNativeDecoder(jni, j_decoder); - decoder = reinterpret_cast(native_decoder); - } else { - decoder = new VideoDecoderWrapper(jni, j_decoder); - } - - return std::unique_ptr(decoder); -} - -std::unique_ptr JavaToNativeVideoEncoder( - JNIEnv* jni, - const JavaRef& j_encoder) { - VideoEncoder* encoder; - if (Java_WrappedNativeVideoEncoder_isInstanceOf(jni, j_encoder)) { - jlong native_encoder = - Java_WrappedNativeVideoEncoder_createNativeEncoder(jni, j_encoder); - encoder = reinterpret_cast(native_encoder); - } else { - encoder = new VideoEncoderWrapper(jni, j_encoder); - } - - return std::unique_ptr(encoder); -} - -bool IsWrappedSoftwareEncoder(JNIEnv* jni, const JavaRef& j_encoder) { - return Java_WrappedNativeVideoEncoder_isWrappedSoftwareEncoder(jni, - j_encoder); -} - -} // namespace jni -} // namespace webrtc diff --git a/sdk/android/src/jni/wrappednativecodec.h b/sdk/android/src/jni/wrappednativecodec.h deleted file mode 100644 index 1d24271865..0000000000 --- a/sdk/android/src/jni/wrappednativecodec.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_ -#define SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_ - -#include -#include - -#include "api/video_codecs/video_decoder.h" -#include "api/video_codecs/video_encoder.h" -#include "sdk/android/native_api/jni/scoped_java_ref.h" - -namespace webrtc { -namespace jni { - -/* If the j_decoder is a wrapped native decoder, unwrap it. If it is not, - * wrap it in a VideoDecoderWrapper. - */ -std::unique_ptr JavaToNativeVideoDecoder( - JNIEnv* jni, - const JavaRef& j_decoder); - -/* If the j_encoder is a wrapped native encoder, unwrap it. If it is not, - * wrap it in a VideoEncoderWrapper. - */ -std::unique_ptr JavaToNativeVideoEncoder( - JNIEnv* jni, - const JavaRef& j_encoder); - -bool IsWrappedSoftwareEncoder(JNIEnv* jni, const JavaRef& j_encoder); - -} // namespace jni -} // namespace webrtc - -#endif // SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_