From f3096afd48d1639d9d263db0f4e6a631e4567dc9 Mon Sep 17 00:00:00 2001 From: Danil Chapovalov Date: Tue, 12 Mar 2024 13:22:27 +0100 Subject: [PATCH] Propagate Environment to create VideoEncoder through java wrappers Bug: webrtc:15860 Change-Id: If1a2873a899e1b839822a4b56aa87d4bae70c581 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/342740 Reviewed-by: Sergey Silkin Commit-Queue: Danil Chapovalov Cr-Commit-Position: refs/heads/main@{#41887} --- sdk/android/BUILD.gn | 4 +++ .../api/org/webrtc/LibaomAv1Encoder.java | 6 +++++ .../api/org/webrtc/LibvpxVp8Encoder.java | 6 +++++ .../api/org/webrtc/LibvpxVp9Encoder.java | 6 +++++ .../webrtc/SoftwareVideoEncoderFactory.java | 14 +++++++--- sdk/android/api/org/webrtc/VideoEncoder.java | 11 ++++++-- .../api/org/webrtc/VideoEncoderFallback.java | 7 +++++ .../org/webrtc/WrappedNativeVideoEncoder.java | 4 ++- .../codecs/wrapper_unittest.cc | 6 ++++- sdk/android/src/jni/libaom_av1_encoder.cc | 8 ++++++ .../src/jni/software_video_encoder_factory.cc | 22 +++++++++++++++ .../src/jni/video_encoder_factory_wrapper.cc | 13 +++++++++ .../src/jni/video_encoder_factory_wrapper.h | 4 +++ sdk/android/src/jni/video_encoder_fallback.cc | 27 ++++++++++++++++--- sdk/android/src/jni/video_encoder_wrapper.cc | 14 ++++++++++ sdk/android/src/jni/video_encoder_wrapper.h | 6 +++++ sdk/android/src/jni/vp8_codec.cc | 6 +++++ sdk/android/src/jni/vp9_codec.cc | 7 +++++ 18 files changed, 160 insertions(+), 11 deletions(-) diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 0efe76a497..8858d21b60 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -869,6 +869,7 @@ if (current_os == "linux" || is_android) { ":base_jni", ":generated_libvpx_vp9_jni", ":video_jni", + "../../api/environment", "../../modules/video_coding:webrtc_vp9", ] } @@ -881,6 +882,7 @@ if (current_os == "linux" || is_android) { ":base_jni", ":generated_libaom_av1_encoder_jni", ":video_jni", + "../../api/environment", "../../modules/video_coding/codecs/av1:libaom_av1_encoder", ] } @@ -1645,6 +1647,8 @@ if (is_android) { "../../api:enable_media_with_defaults", "../../api:field_trials_view", "../../api:scoped_refptr", + "../../api/environment", + "../../api/environment:environment_factory", "../../api/rtc_event_log:rtc_event_log_factory", "../../api/task_queue:default_task_queue_factory", "../../api/video:video_frame", diff --git a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java index 569a719f44..46ea9851c4 100644 --- a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java +++ b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java @@ -16,7 +16,13 @@ public class LibaomAv1Encoder extends WrappedNativeVideoEncoder { return nativeCreateEncoder(); } + @Override + public long createNative(long webrtcEnvRef) { + return nativeCreate(webrtcEnvRef); + } + static native long nativeCreateEncoder(); + static native long nativeCreate(long webrtcEnvRef); @Override public boolean isHardwareEncoder() { diff --git a/sdk/android/api/org/webrtc/LibvpxVp8Encoder.java b/sdk/android/api/org/webrtc/LibvpxVp8Encoder.java index 4be9e52c14..a47df2577e 100644 --- a/sdk/android/api/org/webrtc/LibvpxVp8Encoder.java +++ b/sdk/android/api/org/webrtc/LibvpxVp8Encoder.java @@ -16,7 +16,13 @@ public class LibvpxVp8Encoder extends WrappedNativeVideoEncoder { return nativeCreateEncoder(); } + @Override + public long createNative(long webrtcEnvRef) { + return nativeCreate(webrtcEnvRef); + } + static native long nativeCreateEncoder(); + static native long nativeCreate(long webrtcEnvRef); @Override public boolean isHardwareEncoder() { diff --git a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java index 1211ae93fb..de242bbd00 100644 --- a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java +++ b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java @@ -16,7 +16,13 @@ public class LibvpxVp9Encoder extends WrappedNativeVideoEncoder { return nativeCreateEncoder(); } + @Override + public long createNative(long webrtcEnvRef) { + return nativeCreate(webrtcEnvRef); + } + static native long nativeCreateEncoder(); + static native long nativeCreate(long webrtcEnvRef); @Override public boolean isHardwareEncoder() { diff --git a/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java index 7f4c457b97..b4e4142495 100644 --- a/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java @@ -26,8 +26,7 @@ public class SoftwareVideoEncoderFactory implements VideoEncoderFactory { @Nullable @Override public VideoEncoder createEncoder(VideoCodecInfo info) { - long nativeEncoder = nativeCreateEncoder(nativeFactory, info); - if (nativeEncoder == 0) { + if (!nativeIsSupported(nativeFactory, info)) { Logging.w(TAG, "Trying to create encoder for unsupported format. " + info); return null; } @@ -35,7 +34,12 @@ public class SoftwareVideoEncoderFactory implements VideoEncoderFactory { return new WrappedNativeVideoEncoder() { @Override public long createNativeVideoEncoder() { - return nativeEncoder; + return nativeCreateEncoder(nativeFactory, info); + } + + @Override + public long createNative(long webrtcEnvRef) { + return nativeCreate(nativeFactory, webrtcEnvRef, info); } @Override @@ -54,5 +58,9 @@ public class SoftwareVideoEncoderFactory implements VideoEncoderFactory { private static native long nativeCreateEncoder(long factory, VideoCodecInfo videoCodecInfo); + private static native boolean nativeIsSupported(long factory, VideoCodecInfo info); + + private static native long nativeCreate(long factory, long webrtcEnvRef, VideoCodecInfo info); + private static native List nativeGetSupportedCodecs(long factory); } diff --git a/sdk/android/api/org/webrtc/VideoEncoder.java b/sdk/android/api/org/webrtc/VideoEncoder.java index 0d8cf830ae..d2400012cb 100644 --- a/sdk/android/api/org/webrtc/VideoEncoder.java +++ b/sdk/android/api/org/webrtc/VideoEncoder.java @@ -310,11 +310,11 @@ public interface VideoEncoder { * encoder (e.g., an Android platform encoder), or alternatively 2) a native * encoder (e.g., a software encoder or a C++ encoder adapter). * - * For case 1), createNativeVideoEncoder() should return zero. + * For case 1), createNative() should return zero. * In this case, we expect the native library to call the encoder through * JNI using the Java interface declared below. * - * For case 2), createNativeVideoEncoder() should return a non-zero value. + * For case 2), createNative() should return a non-zero value. * In this case, we expect the native library to treat the returned value as * a raw pointer of type webrtc::VideoEncoder* (ownership is transferred to * the caller). The native library should then directly call the @@ -322,11 +322,18 @@ public interface VideoEncoder { * the Java interface methods declared below should thus throw an * UnsupportedOperationException. */ + @CalledByNative + @Deprecated default long createNativeVideoEncoder() { return 0; } + @CalledByNative + default long createNative(long webrtcEnvRef) { + return createNativeVideoEncoder(); + } + /** * Returns true if the encoder is backed by hardware. */ diff --git a/sdk/android/api/org/webrtc/VideoEncoderFallback.java b/sdk/android/api/org/webrtc/VideoEncoderFallback.java index fa36b7c989..0d0716b208 100644 --- a/sdk/android/api/org/webrtc/VideoEncoderFallback.java +++ b/sdk/android/api/org/webrtc/VideoEncoderFallback.java @@ -27,10 +27,17 @@ public class VideoEncoderFallback extends WrappedNativeVideoEncoder { return nativeCreateEncoder(fallback, primary); } + @Override + public long createNative(long webrtcEnvRef) { + return nativeCreate(webrtcEnvRef, fallback, primary); + } + @Override public boolean isHardwareEncoder() { return primary.isHardwareEncoder(); } private static native long nativeCreateEncoder(VideoEncoder fallback, VideoEncoder primary); + private static native long nativeCreate( + long webrtcEnvRef, VideoEncoder fallback, VideoEncoder primary); } diff --git a/sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java b/sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java index 7d0908a6ac..801cba3775 100644 --- a/sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java +++ b/sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java @@ -14,7 +14,9 @@ package org.webrtc; * Wraps a native webrtc::VideoEncoder. */ public abstract class WrappedNativeVideoEncoder implements VideoEncoder { - @Override public abstract long createNativeVideoEncoder(); + // TODO: bugs.webrtc.org/15860 - uncomment when implemented by all + // derived classes instead of the createNativeVideoEncoder + // @Override public abstract long createNative(long webrtcEnvRef); @Override public abstract boolean isHardwareEncoder(); @Override diff --git a/sdk/android/native_unittests/codecs/wrapper_unittest.cc b/sdk/android/native_unittests/codecs/wrapper_unittest.cc index 6a43480164..c9fced52f9 100644 --- a/sdk/android/native_unittests/codecs/wrapper_unittest.cc +++ b/sdk/android/native_unittests/codecs/wrapper_unittest.cc @@ -13,6 +13,8 @@ #include #include "absl/memory/memory.h" +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "media/base/media_constants.h" #include "sdk/android/generated_native_unittests_jni/CodecsWrapperTestHelper_jni.h" #include "sdk/android/src/jni/video_encoder_wrapper.h" @@ -40,8 +42,10 @@ TEST(JavaCodecsWrapperTest, JavaToNativeResolutionBitrateLimits) { JNIEnv* env = AttachCurrentThreadIfNeeded(); ScopedJavaLocalRef j_fake_encoder = jni::Java_CodecsWrapperTestHelper_createFakeVideoEncoder(env); + const Environment webrtc_env = CreateEnvironment(); - auto encoder = jni::JavaToNativeVideoEncoder(env, j_fake_encoder); + auto encoder = jni::JavaToNativeVideoEncoder( + env, j_fake_encoder, NativeToJavaPointer(&webrtc_env)); ASSERT_TRUE(encoder); // Check that the bitrate limits correctly passed from Java to native. diff --git a/sdk/android/src/jni/libaom_av1_encoder.cc b/sdk/android/src/jni/libaom_av1_encoder.cc index 96e0dda169..d597713ea9 100644 --- a/sdk/android/src/jni/libaom_av1_encoder.cc +++ b/sdk/android/src/jni/libaom_av1_encoder.cc @@ -12,6 +12,7 @@ #include +#include "api/environment/environment.h" #include "sdk/android/generated_libaom_av1_encoder_jni/LibaomAv1Encoder_jni.h" #include "sdk/android/src/jni/jni_helpers.h" @@ -22,5 +23,12 @@ static jlong JNI_LibaomAv1Encoder_CreateEncoder(JNIEnv* jni) { return jlongFromPointer(webrtc::CreateLibaomAv1Encoder().release()); } +jlong JNI_LibaomAv1Encoder_Create(JNIEnv* jni, jlong j_webrtc_env_ref) { + return NativeToJavaPointer( + CreateLibaomAv1Encoder( + *reinterpret_cast(j_webrtc_env_ref)) + .release()); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/software_video_encoder_factory.cc b/sdk/android/src/jni/software_video_encoder_factory.cc index 4b86960198..2533025a2e 100644 --- a/sdk/android/src/jni/software_video_encoder_factory.cc +++ b/sdk/android/src/jni/software_video_encoder_factory.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "api/environment/environment.h" #include "api/video_codecs/builtin_video_encoder_factory.h" #include "api/video_codecs/video_encoder.h" #include "sdk/android/generated_swcodecs_jni/SoftwareVideoEncoderFactory_jni.h" @@ -23,6 +24,27 @@ static jlong JNI_SoftwareVideoEncoderFactory_CreateFactory(JNIEnv* env) { CreateBuiltinVideoEncoderFactory().release()); } +jboolean JNI_SoftwareVideoEncoderFactory_IsSupported( + JNIEnv* env, + jlong j_factory, + const JavaParamRef& j_info) { + return VideoCodecInfoToSdpVideoFormat(env, j_info) + .IsCodecInList(reinterpret_cast(j_factory) + ->GetSupportedFormats()); +} + +jlong JNI_SoftwareVideoEncoderFactory_Create( + JNIEnv* env, + jlong j_factory, + jlong j_webrtc_env_ref, + const JavaParamRef& j_info) { + return NativeToJavaPointer( + reinterpret_cast(j_factory) + ->Create(*reinterpret_cast(j_webrtc_env_ref), + VideoCodecInfoToSdpVideoFormat(env, j_info)) + .release()); +} + static jlong JNI_SoftwareVideoEncoderFactory_CreateEncoder( JNIEnv* env, jlong j_factory, diff --git a/sdk/android/src/jni/video_encoder_factory_wrapper.cc b/sdk/android/src/jni/video_encoder_factory_wrapper.cc index 7df129b360..5c8ee71fa9 100644 --- a/sdk/android/src/jni/video_encoder_factory_wrapper.cc +++ b/sdk/android/src/jni/video_encoder_factory_wrapper.cc @@ -104,6 +104,19 @@ std::unique_ptr VideoEncoderFactoryWrapper::CreateVideoEncoder( return JavaToNativeVideoEncoder(jni, encoder); } +std::unique_ptr VideoEncoderFactoryWrapper::Create( + const Environment& env, + const SdpVideoFormat& format) { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + ScopedJavaLocalRef j_codec_info = + SdpVideoFormatToVideoCodecInfo(jni, format); + ScopedJavaLocalRef encoder = Java_VideoEncoderFactory_createEncoder( + jni, encoder_factory_, j_codec_info); + if (!encoder.obj()) + return nullptr; + return JavaToNativeVideoEncoder(jni, encoder, NativeToJavaPointer(&env)); +} + std::vector VideoEncoderFactoryWrapper::GetSupportedFormats() const { return supported_formats_; diff --git a/sdk/android/src/jni/video_encoder_factory_wrapper.h b/sdk/android/src/jni/video_encoder_factory_wrapper.h index bf6d1e3833..ba401202c8 100644 --- a/sdk/android/src/jni/video_encoder_factory_wrapper.h +++ b/sdk/android/src/jni/video_encoder_factory_wrapper.h @@ -15,6 +15,7 @@ #include +#include "api/environment/environment.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_encoder_factory.h" #include "sdk/android/src/jni/jni_helpers.h" @@ -33,6 +34,9 @@ class VideoEncoderFactoryWrapper : public VideoEncoderFactory { std::unique_ptr CreateVideoEncoder( const SdpVideoFormat& format) override; + std::unique_ptr Create(const Environment& env, + const SdpVideoFormat& format) override; + // Returns a list of supported codecs in order of preference. std::vector GetSupportedFormats() const override; diff --git a/sdk/android/src/jni/video_encoder_fallback.cc b/sdk/android/src/jni/video_encoder_fallback.cc index d581572abf..4f07c39a3b 100644 --- a/sdk/android/src/jni/video_encoder_fallback.cc +++ b/sdk/android/src/jni/video_encoder_fallback.cc @@ -10,6 +10,7 @@ #include +#include "api/environment/environment.h" #include "api/video_codecs/video_encoder_software_fallback_wrapper.h" #include "sdk/android/generated_video_jni/VideoEncoderFallback_jni.h" #include "sdk/android/src/jni/jni_helpers.h" @@ -27,12 +28,30 @@ static jlong JNI_VideoEncoderFallback_CreateEncoder( std::unique_ptr primary_encoder = JavaToNativeVideoEncoder(jni, j_primary_encoder); - VideoEncoder* nativeWrapper = - CreateVideoEncoderSoftwareFallbackWrapper(std::move(fallback_encoder), - std::move(primary_encoder)) + VideoEncoder* native_wrapper = + CreateVideoEncoderSoftwareFallbackWrapper( + std::move(fallback_encoder), std::move(primary_encoder), + /*prefer_temporal_support=*/false) .release(); - return jlongFromPointer(nativeWrapper); + return NativeToJavaPointer(native_wrapper); +} + +jlong JNI_VideoEncoderFallback_Create( + JNIEnv* jni, + jlong j_webrtc_env_ref, + const JavaParamRef& j_fallback_encoder, + const JavaParamRef& j_primary_encoder) { + std::unique_ptr fallback_encoder = + JavaToNativeVideoEncoder(jni, j_fallback_encoder, j_webrtc_env_ref); + std::unique_ptr primary_encoder = + JavaToNativeVideoEncoder(jni, j_primary_encoder, j_webrtc_env_ref); + + return NativeToJavaPointer(CreateVideoEncoderSoftwareFallbackWrapper( + std::move(fallback_encoder), + std::move(primary_encoder), + /*prefer_temporal_support=*/false) + .release()); } } // namespace jni diff --git a/sdk/android/src/jni/video_encoder_wrapper.cc b/sdk/android/src/jni/video_encoder_wrapper.cc index 8458ba6c3e..bc5650a774 100644 --- a/sdk/android/src/jni/video_encoder_wrapper.cc +++ b/sdk/android/src/jni/video_encoder_wrapper.cc @@ -12,6 +12,7 @@ #include +#include "absl/memory/memory.h" #include "common_video/h264/h264_common.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" @@ -457,6 +458,19 @@ ScopedJavaLocalRef VideoEncoderWrapper::ToJavaRateControlParameters( rc_parameters.framerate_fps); } +std::unique_ptr JavaToNativeVideoEncoder( + JNIEnv* jni, + const JavaRef& j_encoder, + jlong j_webrtc_env_ref) { + if (jlong native_encoder = + Java_VideoEncoder_createNative(jni, j_encoder, j_webrtc_env_ref); + native_encoder != 0) { + return absl::WrapUnique(reinterpret_cast(native_encoder)); + } else { + return std::make_unique(jni, j_encoder); + } +} + std::unique_ptr JavaToNativeVideoEncoder( JNIEnv* jni, const JavaRef& j_encoder) { diff --git a/sdk/android/src/jni/video_encoder_wrapper.h b/sdk/android/src/jni/video_encoder_wrapper.h index 04d70f3e28..3c4678b570 100644 --- a/sdk/android/src/jni/video_encoder_wrapper.h +++ b/sdk/android/src/jni/video_encoder_wrapper.h @@ -121,6 +121,12 @@ class VideoEncoderWrapper : public VideoEncoder { /* If the j_encoder is a wrapped native encoder, unwrap it. If it is not, * wrap it in a VideoEncoderWrapper. */ +std::unique_ptr JavaToNativeVideoEncoder( + JNIEnv* jni, + const JavaRef& j_encoder, + jlong webrtcEnvRef); + +// Deprecated, use variant above that requires webrtc::Environment std::unique_ptr JavaToNativeVideoEncoder( JNIEnv* jni, const JavaRef& j_encoder); diff --git a/sdk/android/src/jni/vp8_codec.cc b/sdk/android/src/jni/vp8_codec.cc index b33fece26e..2c1994ec89 100644 --- a/sdk/android/src/jni/vp8_codec.cc +++ b/sdk/android/src/jni/vp8_codec.cc @@ -23,6 +23,12 @@ static jlong JNI_LibvpxVp8Encoder_CreateEncoder(JNIEnv* jni) { return jlongFromPointer(VP8Encoder::Create().release()); } +jlong JNI_LibvpxVp8Encoder_Create(JNIEnv* jni, jlong j_webrtc_env_ref) { + return NativeToJavaPointer( + CreateVp8Encoder(*reinterpret_cast(j_webrtc_env_ref)) + .release()); +} + static jlong JNI_LibvpxVp8Decoder_CreateDecoder(JNIEnv* jni, jlong j_webrtc_env_ref) { return NativeToJavaPointer( diff --git a/sdk/android/src/jni/vp9_codec.cc b/sdk/android/src/jni/vp9_codec.cc index ad9ca793ce..fdafea83f5 100644 --- a/sdk/android/src/jni/vp9_codec.cc +++ b/sdk/android/src/jni/vp9_codec.cc @@ -10,6 +10,7 @@ #include +#include "api/environment/environment.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Decoder_jni.h" #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Encoder_jni.h" @@ -22,6 +23,12 @@ static jlong JNI_LibvpxVp9Encoder_CreateEncoder(JNIEnv* jni) { return jlongFromPointer(VP9Encoder::Create().release()); } +jlong JNI_LibvpxVp9Encoder_Create(JNIEnv* jni, jlong j_webrtc_env_ref) { + return NativeToJavaPointer( + CreateVp9Encoder(*reinterpret_cast(j_webrtc_env_ref)) + .release()); +} + static jboolean JNI_LibvpxVp9Encoder_IsSupported(JNIEnv* jni) { return !SupportedVP9Codecs().empty(); }