From 22b6564d77af5aa637c0da59219bfb681cc55156 Mon Sep 17 00:00:00 2001 From: Danil Chapovalov Date: Tue, 6 Feb 2024 21:31:44 +0100 Subject: [PATCH] Propagate webrtc::Environment to create VideoDecoder through java wrappers Bug: webrtc:15791 Change-Id: I25976158ef780211355deb934e83d275d4f881e2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/336980 Reviewed-by: Sergey Silkin Commit-Queue: Danil Chapovalov Cr-Commit-Position: refs/heads/main@{#41686} --- sdk/android/BUILD.gn | 7 +++++- .../webrtc/SoftwareVideoDecoderFactory.java | 17 +++++++++---- sdk/android/api/org/webrtc/VideoDecoder.java | 10 ++++++-- .../api/org/webrtc/VideoDecoderFallback.java | 8 ++++++ .../org/webrtc/WrappedNativeVideoDecoder.java | 4 ++- .../src/jni/software_video_decoder_factory.cc | 25 +++++++++++++++++++ .../src/jni/video_decoder_factory_wrapper.cc | 6 +++-- .../src/jni/video_decoder_factory_wrapper.h | 5 ++-- sdk/android/src/jni/video_decoder_fallback.cc | 18 +++++++++++++ sdk/android/src/jni/video_decoder_wrapper.cc | 15 +++++++++++ sdk/android/src/jni/video_decoder_wrapper.h | 7 ++++++ 11 files changed, 109 insertions(+), 13 deletions(-) diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 5296e54076..13c27c0a44 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -663,6 +663,7 @@ if (current_os == "linux" || is_android) { "../../api:libjingle_peerconnection_api", "../../api:media_stream_interface", "../../api:sequence_checker", + "../../api/environment", "../../api/task_queue", "../../api/video:encoded_image", "../../api/video:render_resolution", @@ -689,7 +690,10 @@ if (current_os == "linux" || is_android) { "../../rtc_base/synchronization:mutex", "//third_party/libyuv", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/types:optional", + ] } # Sources here require -lEGL linker flag. It is separated from video_jni @@ -908,6 +912,7 @@ if (current_os == "linux" || is_android) { ":libvpx_vp9_jni", ":native_api_jni", ":video_jni", + "../../api/environment", "../../api/video_codecs:builtin_video_decoder_factory", "../../api/video_codecs:builtin_video_encoder_factory", "../../api/video_codecs:video_codecs_api", diff --git a/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java b/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java index 2ac42e834e..ac815d395b 100644 --- a/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java +++ b/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java @@ -11,7 +11,6 @@ package org.webrtc; import androidx.annotation.Nullable; -import java.util.Arrays; import java.util.List; public class SoftwareVideoDecoderFactory implements VideoDecoderFactory { @@ -26,16 +25,19 @@ public class SoftwareVideoDecoderFactory implements VideoDecoderFactory { @Nullable @Override public VideoDecoder createDecoder(VideoCodecInfo info) { - long nativeDecoder = nativeCreateDecoder(nativeFactory, info); - if (nativeDecoder == 0) { + if (!nativeIsSupported(nativeFactory, info)) { Logging.w(TAG, "Trying to create decoder for unsupported format. " + info); return null; } - return new WrappedNativeVideoDecoder() { @Override public long createNativeVideoDecoder() { - return nativeDecoder; + return nativeCreateDecoder(nativeFactory, info); + } + + @Override + public long createNative(long webrtcEnvRef) { + return nativeCreate(nativeFactory, webrtcEnvRef, info); } }; } @@ -49,5 +51,10 @@ public class SoftwareVideoDecoderFactory implements VideoDecoderFactory { private static native long nativeCreateDecoder(long factory, VideoCodecInfo videoCodecInfo); + private static native boolean nativeIsSupported(long factory, VideoCodecInfo info); + + private static native long nativeCreate( + long factory, long webrtcEnvRef, VideoCodecInfo info); + private static native List nativeGetSupportedCodecs(long factory); } diff --git a/sdk/android/api/org/webrtc/VideoDecoder.java b/sdk/android/api/org/webrtc/VideoDecoder.java index a80fa4fef2..f44672da08 100644 --- a/sdk/android/api/org/webrtc/VideoDecoder.java +++ b/sdk/android/api/org/webrtc/VideoDecoder.java @@ -56,11 +56,11 @@ public interface VideoDecoder { * decoder (e.g., an Android platform decoder), or alternatively 2) a native * decoder (e.g., a software decoder or a C++ decoder adapter). * - * For case 1), createNativeVideoDecoder() should return zero. + * For case 1), createNative() should return zero. * In this case, we expect the native library to call the decoder through * JNI using the Java interface declared below. * - * For case 2), createNativeVideoDecoder() should return a non-zero value. + * For case 2), createNative() should return a non-zero value. * In this case, we expect the native library to treat the returned value as * a raw pointer of type webrtc::VideoDecoder* (ownership is transferred to * the caller). The native library should then directly call the @@ -69,6 +69,12 @@ public interface VideoDecoder { * UnsupportedOperationException. */ @CalledByNative + default long createNative(long webrtcEnvRef) { + return createNativeVideoDecoder(); + } + + @CalledByNative + @Deprecated default long createNativeVideoDecoder() { return 0; } diff --git a/sdk/android/api/org/webrtc/VideoDecoderFallback.java b/sdk/android/api/org/webrtc/VideoDecoderFallback.java index ddfa3ecd40..dc2e2eeabe 100644 --- a/sdk/android/api/org/webrtc/VideoDecoderFallback.java +++ b/sdk/android/api/org/webrtc/VideoDecoderFallback.java @@ -27,5 +27,13 @@ public class VideoDecoderFallback extends WrappedNativeVideoDecoder { return nativeCreateDecoder(fallback, primary); } + @Override + public long createNative(long webrtcEnvRef) { + return nativeCreate(webrtcEnvRef, fallback, primary); + } + private static native long nativeCreateDecoder(VideoDecoder fallback, VideoDecoder primary); + + private static native long nativeCreate( + long webrtcEnvRef, VideoDecoder fallback, VideoDecoder primary); } diff --git a/sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java b/sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java index 027120e48e..ac9886e6a3 100644 --- a/sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java +++ b/sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java @@ -14,7 +14,9 @@ package org.webrtc; * Wraps a native webrtc::VideoDecoder. */ public abstract class WrappedNativeVideoDecoder implements VideoDecoder { - @Override public abstract long createNativeVideoDecoder(); + // TODO: bugs.webrtc.org/15791 - uncomment when implemented by all + // derived classes instead of the createNativeVideoDecoder + // @Override public abstract long createNative(long webrtcEnvRef); @Override public final VideoCodecStatus initDecode(Settings settings, Callback decodeCallback) { diff --git a/sdk/android/src/jni/software_video_decoder_factory.cc b/sdk/android/src/jni/software_video_decoder_factory.cc index 151bf5f205..46b67b88eb 100644 --- a/sdk/android/src/jni/software_video_decoder_factory.cc +++ b/sdk/android/src/jni/software_video_decoder_factory.cc @@ -8,8 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + +#include "api/environment/environment.h" #include "api/video_codecs/builtin_video_decoder_factory.h" #include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" #include "sdk/android/generated_swcodecs_jni/SoftwareVideoDecoderFactory_jni.h" #include "sdk/android/native_api/jni/java_types.h" #include "sdk/android/src/jni/jni_helpers.h" @@ -39,6 +43,27 @@ static jlong JNI_SoftwareVideoDecoderFactory_CreateDecoder( return webrtc::NativeToJavaPointer(decoder.release()); } +jboolean JNI_SoftwareVideoDecoderFactory_IsSupported( + JNIEnv* env, + jlong j_factory, + const JavaParamRef& j_info) { + return VideoCodecInfoToSdpVideoFormat(env, j_info) + .IsCodecInList(reinterpret_cast(j_factory) + ->GetSupportedFormats()); +} + +jlong JNI_SoftwareVideoDecoderFactory_Create( + JNIEnv* env, + jlong j_factory, + jlong j_webrtc_env_ref, + const JavaParamRef& j_info) { + return NativeToJavaPointer( + reinterpret_cast(j_factory) + ->Create(*reinterpret_cast(j_webrtc_env_ref), + VideoCodecInfoToSdpVideoFormat(env, j_info)) + .release()); +} + static webrtc::ScopedJavaLocalRef JNI_SoftwareVideoDecoderFactory_GetSupportedCodecs(JNIEnv* env, jlong j_factory) { diff --git a/sdk/android/src/jni/video_decoder_factory_wrapper.cc b/sdk/android/src/jni/video_decoder_factory_wrapper.cc index 2d9240493a..1f6f4a0b8d 100644 --- a/sdk/android/src/jni/video_decoder_factory_wrapper.cc +++ b/sdk/android/src/jni/video_decoder_factory_wrapper.cc @@ -10,6 +10,7 @@ #include "sdk/android/src/jni/video_decoder_factory_wrapper.h" +#include "api/environment/environment.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder.h" #include "rtc_base/logging.h" @@ -27,7 +28,8 @@ VideoDecoderFactoryWrapper::VideoDecoderFactoryWrapper( : decoder_factory_(jni, decoder_factory) {} VideoDecoderFactoryWrapper::~VideoDecoderFactoryWrapper() = default; -std::unique_ptr VideoDecoderFactoryWrapper::CreateVideoDecoder( +std::unique_ptr VideoDecoderFactoryWrapper::Create( + const Environment& env, const SdpVideoFormat& format) { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedJavaLocalRef j_codec_info = @@ -36,7 +38,7 @@ std::unique_ptr VideoDecoderFactoryWrapper::CreateVideoDecoder( jni, decoder_factory_, j_codec_info); if (!decoder.obj()) return nullptr; - return JavaToNativeVideoDecoder(jni, decoder); + return JavaToNativeVideoDecoder(jni, decoder, NativeToJavaPointer(&env)); } std::vector VideoDecoderFactoryWrapper::GetSupportedFormats() diff --git a/sdk/android/src/jni/video_decoder_factory_wrapper.h b/sdk/android/src/jni/video_decoder_factory_wrapper.h index 2122fdc008..689e829128 100644 --- a/sdk/android/src/jni/video_decoder_factory_wrapper.h +++ b/sdk/android/src/jni/video_decoder_factory_wrapper.h @@ -13,6 +13,7 @@ #include +#include "api/environment/environment.h" #include "api/video_codecs/video_decoder_factory.h" #include "sdk/android/src/jni/jni_helpers.h" @@ -28,8 +29,8 @@ class VideoDecoderFactoryWrapper : public VideoDecoderFactory { ~VideoDecoderFactoryWrapper() override; std::vector GetSupportedFormats() const override; - std::unique_ptr CreateVideoDecoder( - const SdpVideoFormat& format) override; + std::unique_ptr Create(const Environment& env, + const SdpVideoFormat& format) override; private: const ScopedJavaGlobalRef decoder_factory_; diff --git a/sdk/android/src/jni/video_decoder_fallback.cc b/sdk/android/src/jni/video_decoder_fallback.cc index a678280f69..8b521c8f2f 100644 --- a/sdk/android/src/jni/video_decoder_fallback.cc +++ b/sdk/android/src/jni/video_decoder_fallback.cc @@ -35,5 +35,23 @@ static jlong JNI_VideoDecoderFallback_CreateDecoder( return jlongFromPointer(nativeWrapper); } +static jlong JNI_VideoDecoderFallback_Create( + JNIEnv* jni, + jlong j_webrtc_env_ref, + const JavaParamRef& j_fallback_decoder, + const JavaParamRef& j_primary_decoder) { + std::unique_ptr fallback_decoder = + JavaToNativeVideoDecoder(jni, j_fallback_decoder, j_webrtc_env_ref); + std::unique_ptr primary_decoder = + JavaToNativeVideoDecoder(jni, j_primary_decoder, j_webrtc_env_ref); + + VideoDecoder* native_wrapper = + CreateVideoDecoderSoftwareFallbackWrapper(std::move(fallback_decoder), + std::move(primary_decoder)) + .release(); + + return NativeToJavaPointer(native_wrapper); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/video_decoder_wrapper.cc b/sdk/android/src/jni/video_decoder_wrapper.cc index e083df5ae8..c96961879c 100644 --- a/sdk/android/src/jni/video_decoder_wrapper.cc +++ b/sdk/android/src/jni/video_decoder_wrapper.cc @@ -10,6 +10,8 @@ #include "sdk/android/src/jni/video_decoder_wrapper.h" +#include "absl/memory/memory.h" +#include "api/environment/environment.h" #include "api/video/render_resolution.h" #include "api/video/video_frame.h" #include "api/video_codecs/video_decoder.h" @@ -281,5 +283,18 @@ std::unique_ptr JavaToNativeVideoDecoder( return std::unique_ptr(decoder); } +std::unique_ptr JavaToNativeVideoDecoder( + JNIEnv* jni, + const JavaRef& j_decoder, + jlong webrtcEnvRef) { + if (jlong native_decoder = + Java_VideoDecoder_createNative(jni, j_decoder, webrtcEnvRef); + native_decoder != 0) { + return absl::WrapUnique(reinterpret_cast(native_decoder)); + } else { + return std::make_unique(jni, j_decoder); + } +} + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/video_decoder_wrapper.h b/sdk/android/src/jni/video_decoder_wrapper.h index 53246f380e..80638a7b61 100644 --- a/sdk/android/src/jni/video_decoder_wrapper.h +++ b/sdk/android/src/jni/video_decoder_wrapper.h @@ -116,10 +116,17 @@ class VideoDecoderWrapper : public VideoDecoder { /* If the j_decoder is a wrapped native decoder, unwrap it. If it is not, * wrap it in a VideoDecoderWrapper. */ +// TODO: bugs.webrtc.org/15791 - delete variant without the webrtcEnvRef +// parameter when unused. std::unique_ptr JavaToNativeVideoDecoder( JNIEnv* jni, const JavaRef& j_decoder); +std::unique_ptr JavaToNativeVideoDecoder( + JNIEnv* jni, + const JavaRef& j_decoder, + jlong webrtcEnvRef); + } // namespace jni } // namespace webrtc