From dc1b9f179a9eed7a2dc7b1d915d38e97f156034d Mon Sep 17 00:00:00 2001 From: Anders Carlsson Date: Fri, 10 Nov 2017 13:15:04 +0100 Subject: [PATCH] Injectable software codecs for Android. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is similar to https://webrtc-review.googlesource.com/c/src/+/3620 for iOS. Using the new WebRtcMediaEngineFactory::Create API, the built-in software video codecs are no longer appended to the injected codecs. To be able to use the software codecs, they are exposed as Java classes through SoftwareVideoEncoderFactory etc. There is also a new DefaultVideoEncoderFactory used by AppRTCMobile. This factory tries to use hardware implementations where available, but falls back to using the injected software codecs. The HardwareVideoEncoderFactory is temporarily also falling back on the software codecs in its default configuration in order to maintain backwards compatibility. Bug: webrtc:7925 Change-Id: I3e8c5ed492ccd160aca968986ad217d7978a951c Reviewed-on: https://webrtc-review.googlesource.com/17480 Reviewed-by: Sami Kalliomäki Reviewed-by: Magnus Jedvert Commit-Queue: Anders Carlsson Cr-Commit-Position: refs/heads/master@{#20647} --- .../appspot/apprtc/PeerConnectionClient.java | 8 +- sdk/android/BUILD.gn | 36 ++++++ .../webrtc/DefaultVideoDecoderFactory.java | 31 +++++ .../webrtc/DefaultVideoEncoderFactory.java | 70 +++++++++++ .../webrtc/HardwareVideoDecoderFactory.java | 17 ++- .../webrtc/HardwareVideoEncoderFactory.java | 33 +++++- .../webrtc/SoftwareVideoDecoderFactory.java | 25 ++++ .../webrtc/SoftwareVideoEncoderFactory.java | 39 ++++++ .../api/org/webrtc/VideoCodecInfo.java | 9 +- .../api/org/webrtc/VideoDecoderFallback.java | 22 ++++ .../DefaultVideoEncoderFactoryTest.java | 111 ++++++++++++++++++ .../org/webrtc/HardwareVideoDecoderTest.java | 4 +- .../src/java/org/webrtc/VP8Decoder.java | 19 +++ .../src/java/org/webrtc/VP8Encoder.java | 19 +++ .../src/java/org/webrtc/VP9Decoder.java | 19 +++ .../src/java/org/webrtc/VP9Encoder.java | 19 +++ .../org/webrtc/WrappedNativeVideoDecoder.java | 52 ++++++++ .../org/webrtc/WrappedNativeVideoEncoder.java | 62 ++++++++++ sdk/android/src/jni/DEPS | 1 + .../src/jni/defaultvideoencoderfactory.cc | 54 +++++++++ sdk/android/src/jni/pc/media_jni.cc | 18 +++ sdk/android/src/jni/pc/media_jni.h | 11 ++ sdk/android/src/jni/pc/null_media_jni.cc | 11 ++ sdk/android/src/jni/pc/null_video_jni.cc | 28 ++++- .../src/jni/pc/ownedfactoryandthreads.h | 20 ++-- .../src/jni/pc/peerconnectionfactory_jni.cc | 59 ++++++++-- sdk/android/src/jni/pc/video_jni.cc | 88 ++++++++------ sdk/android/src/jni/pc/video_jni.h | 23 +++- sdk/android/src/jni/videocodecinfo.cc | 57 +++++++++ sdk/android/src/jni/videocodecinfo.h | 28 +++++ .../src/jni/videodecoderfactorywrapper.cc | 18 +-- .../src/jni/videodecoderfactorywrapper.h | 13 +- sdk/android/src/jni/videodecoderfallback.cc | 39 ++++++ .../src/jni/videoencoderfactorywrapper.cc | 83 +++++-------- .../src/jni/videoencoderfactorywrapper.h | 31 ++--- sdk/android/src/jni/vp8codec.cc | 34 ++++++ sdk/android/src/jni/vp9codec.cc | 34 ++++++ sdk/android/src/jni/wrappednativecodec.cc | 58 +++++++++ sdk/android/src/jni/wrappednativecodec.h | 38 ++++++ 39 files changed, 1171 insertions(+), 170 deletions(-) create mode 100644 sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java create mode 100644 sdk/android/api/org/webrtc/DefaultVideoEncoderFactory.java create mode 100644 sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java create mode 100644 sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java create mode 100644 sdk/android/api/org/webrtc/VideoDecoderFallback.java create mode 100644 sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java create mode 100644 sdk/android/src/java/org/webrtc/VP8Decoder.java create mode 100644 sdk/android/src/java/org/webrtc/VP8Encoder.java create mode 100644 sdk/android/src/java/org/webrtc/VP9Decoder.java create mode 100644 sdk/android/src/java/org/webrtc/VP9Encoder.java create mode 100644 sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java create mode 100644 sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java create mode 100644 sdk/android/src/jni/defaultvideoencoderfactory.cc create mode 100644 sdk/android/src/jni/videocodecinfo.cc create mode 100644 sdk/android/src/jni/videocodecinfo.h create mode 100644 sdk/android/src/jni/videodecoderfallback.cc create mode 100644 sdk/android/src/jni/vp8codec.cc create mode 100644 sdk/android/src/jni/vp9codec.cc create mode 100644 sdk/android/src/jni/wrappednativecodec.cc create mode 100644 sdk/android/src/jni/wrappednativecodec.h diff --git a/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java b/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java index a79a5a888d..156d06f952 100644 --- a/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java +++ b/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java @@ -35,9 +35,9 @@ import org.webrtc.AudioSource; import org.webrtc.AudioTrack; import org.webrtc.CameraVideoCapturer; import org.webrtc.DataChannel; +import org.webrtc.DefaultVideoDecoderFactory; +import org.webrtc.DefaultVideoEncoderFactory; import org.webrtc.EglBase; -import org.webrtc.HardwareVideoDecoderFactory; -import org.webrtc.HardwareVideoEncoderFactory; import org.webrtc.IceCandidate; import org.webrtc.Logging; import org.webrtc.MediaConstraints; @@ -521,9 +521,9 @@ public class PeerConnectionClient { final boolean enableH264HighProfile = VIDEO_CODEC_H264_HIGH.equals(peerConnectionParameters.videoCodec); factory = new PeerConnectionFactory(options, - new HardwareVideoEncoderFactory(rootEglBase.getEglBaseContext(), + new DefaultVideoEncoderFactory(rootEglBase.getEglBaseContext(), true /* enableIntelVp8Encoder */, enableH264HighProfile), - new HardwareVideoDecoderFactory(rootEglBase.getEglBaseContext())); + new DefaultVideoDecoderFactory(rootEglBase.getEglBaseContext())); Log.d(TAG, "Peer connection factory created."); } diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 28546afcfd..80b716415b 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -107,6 +107,8 @@ generate_jni("generated_video_jni") { "api/org/webrtc/VideoEncoder.java", "api/org/webrtc/VideoSink.java", "src/java/org/webrtc/VideoEncoderWrapper.java", + "src/java/org/webrtc/WrappedNativeVideoDecoder.java", + "src/java/org/webrtc/WrappedNativeVideoEncoder.java", ] jni_package = "" jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h" @@ -122,6 +124,7 @@ rtc_static_library("video_jni") { "src/jni/androidvideotracksource.cc", "src/jni/androidvideotracksource.h", "src/jni/androidvideotracksource_jni.cc", + "src/jni/defaultvideoencoderfactory.cc", "src/jni/jni_generator_helper.h", "src/jni/native_handle_impl.cc", "src/jni/native_handle_impl.h", @@ -131,8 +134,11 @@ rtc_static_library("video_jni") { "src/jni/surfacetexturehelper_jni.cc", "src/jni/surfacetexturehelper_jni.h", "src/jni/video_renderer_jni.cc", + "src/jni/videocodecinfo.cc", + "src/jni/videocodecinfo.h", "src/jni/videodecoderfactorywrapper.cc", "src/jni/videodecoderfactorywrapper.h", + "src/jni/videodecoderfallback.cc", "src/jni/videodecoderwrapper.cc", "src/jni/videodecoderwrapper.h", "src/jni/videoencoderfactorywrapper.cc", @@ -142,8 +148,12 @@ rtc_static_library("video_jni") { "src/jni/videofilerenderer_jni.cc", "src/jni/videoframe_jni.cc", "src/jni/videotrack_jni.cc", + "src/jni/vp8codec.cc", + "src/jni/vp9codec.cc", "src/jni/wrapped_native_i420_buffer.cc", "src/jni/wrapped_native_i420_buffer.h", + "src/jni/wrappednativecodec.cc", + "src/jni/wrappednativecodec.h", "src/jni/yuvhelper.cc", ] @@ -182,6 +192,8 @@ rtc_static_library("video_jni") { "../../modules/utility:utility", "../../modules/video_coding:codec_globals_headers", "../../modules/video_coding:video_coding_utility", + "../../modules/video_coding:webrtc_vp8", + "../../modules/video_coding:webrtc_vp9", "../../rtc_base:rtc_base", "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_task_queue", @@ -218,9 +230,11 @@ rtc_static_library("media_jni") { deps = [ ":base_jni", + "../../api/video_codecs:video_codecs_api", "../../call:call_interfaces", "../../logging:rtc_event_log_api", "../../media:rtc_audio_video", + "../../modules/audio_device:audio_device", "../../modules/audio_processing:audio_processing", ] @@ -241,6 +255,14 @@ rtc_static_library("null_media_jni") { deps = [ ":base_jni", ] + + if (is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ + "//build/config/clang:extra_warnings", + "//build/config/clang:find_bad_constructs", + ] + } } rtc_static_library("peerconnection_jni") { @@ -303,8 +325,10 @@ rtc_static_library("peerconnection_jni") { deps = [ ":base_jni", "../..:webrtc_common", + "../../api/video_codecs:video_codecs_api", "../../media:rtc_data", "../../media:rtc_media_base", + "../../modules/audio_device:audio_device", "../../modules/utility:utility", "../../pc:peerconnection", "../../rtc_base:rtc_base", @@ -413,6 +437,8 @@ rtc_android_library("libjingle_peerconnection_java") { "api/org/webrtc/CameraVideoCapturer.java", "api/org/webrtc/DataChannel.java", "api/org/webrtc/DefaultAudioProcessingFactory.java", + "api/org/webrtc/DefaultVideoDecoderFactory.java", + "api/org/webrtc/DefaultVideoEncoderFactory.java", "api/org/webrtc/DtmfSender.java", "api/org/webrtc/EglBase.java", "api/org/webrtc/EglRenderer.java", @@ -448,6 +474,8 @@ rtc_android_library("libjingle_peerconnection_java") { "api/org/webrtc/ScreenCapturerAndroid.java", "api/org/webrtc/SdpObserver.java", "api/org/webrtc/SessionDescription.java", + "api/org/webrtc/SoftwareVideoDecoderFactory.java", + "api/org/webrtc/SoftwareVideoEncoderFactory.java", "api/org/webrtc/StatsObserver.java", "api/org/webrtc/StatsReport.java", "api/org/webrtc/SurfaceTextureHelper.java", @@ -459,6 +487,7 @@ rtc_android_library("libjingle_peerconnection_java") { "api/org/webrtc/VideoCodecStatus.java", "api/org/webrtc/VideoDecoder.java", "api/org/webrtc/VideoDecoderFactory.java", + "api/org/webrtc/VideoDecoderFallback.java", "api/org/webrtc/VideoEncoder.java", "api/org/webrtc/VideoEncoderFactory.java", "api/org/webrtc/VideoFileRenderer.java", @@ -497,6 +526,12 @@ rtc_android_library("libjingle_peerconnection_java") { "src/java/org/webrtc/VideoDecoderWrapperCallback.java", "src/java/org/webrtc/VideoEncoderWrapper.java", "src/java/org/webrtc/WrappedNativeI420Buffer.java", + "src/java/org/webrtc/WrappedNativeVideoEncoder.java", + "src/java/org/webrtc/WrappedNativeVideoDecoder.java", + "src/java/org/webrtc/VP8Encoder.java", + "src/java/org/webrtc/VP8Decoder.java", + "src/java/org/webrtc/VP9Encoder.java", + "src/java/org/webrtc/VP9Decoder.java", ] deps = [ @@ -524,6 +559,7 @@ if (rtc_include_tests) { "instrumentationtests/src/org/webrtc/Camera2CapturerTest.java", "instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java", "instrumentationtests/src/org/webrtc/DefaultAudioProcessingFactoryTest.java", + "instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java", "instrumentationtests/src/org/webrtc/EglRendererTest.java", "instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java", "instrumentationtests/src/org/webrtc/GlRectDrawerTest.java", diff --git a/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java b/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java new file mode 100644 index 0000000000..561e7a51ff --- /dev/null +++ b/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class DefaultVideoDecoderFactory implements VideoDecoderFactory { + private final HardwareVideoDecoderFactory hardwareVideoDecoderFactory; + private final SoftwareVideoDecoderFactory softwareVideoDecoderFactory; + + public DefaultVideoDecoderFactory(EglBase.Context eglContext) { + hardwareVideoDecoderFactory = + new HardwareVideoDecoderFactory(eglContext, false /* fallbackToSoftware */); + softwareVideoDecoderFactory = new SoftwareVideoDecoderFactory(); + } + + @Override + public VideoDecoder createDecoder(String codecType) { + VideoDecoder decoder = hardwareVideoDecoderFactory.createDecoder(codecType); + if (decoder != null) { + return decoder; + } + return softwareVideoDecoderFactory.createDecoder(codecType); + } +} diff --git a/sdk/android/api/org/webrtc/DefaultVideoEncoderFactory.java b/sdk/android/api/org/webrtc/DefaultVideoEncoderFactory.java new file mode 100644 index 0000000000..db6af87e7b --- /dev/null +++ b/sdk/android/api/org/webrtc/DefaultVideoEncoderFactory.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DefaultVideoEncoderFactory implements VideoEncoderFactory { + private final VideoEncoderFactory hardwareVideoEncoderFactory; + private final VideoEncoderFactory softwareVideoEncoderFactory; + + public DefaultVideoEncoderFactory( + EglBase.Context eglContext, boolean enableIntelVp8Encoder, boolean enableH264HighProfile) { + hardwareVideoEncoderFactory = new HardwareVideoEncoderFactory( + eglContext, enableIntelVp8Encoder, enableH264HighProfile, false /* fallbackToSoftware */); + softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory(); + } + + /* This is used for testing. */ + DefaultVideoEncoderFactory(VideoEncoderFactory hardwareVideoEncoderFactory) { + this.hardwareVideoEncoderFactory = hardwareVideoEncoderFactory; + softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory(); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + List hardwareSupportedCodecs = + Arrays.asList(hardwareVideoEncoderFactory.getSupportedCodecs()); + if (containsSameCodec(hardwareSupportedCodecs, info)) { + return hardwareVideoEncoderFactory.createEncoder(info); + } else { + return softwareVideoEncoderFactory.createEncoder(info); + } + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + List supportedCodecInfos = new ArrayList<>(); + + supportedCodecInfos.addAll(Arrays.asList(softwareVideoEncoderFactory.getSupportedCodecs())); + + for (VideoCodecInfo info : hardwareVideoEncoderFactory.getSupportedCodecs()) { + if (!containsSameCodec(supportedCodecInfos, info)) { + supportedCodecInfos.add(info); + } + } + + return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); + } + + private static boolean containsSameCodec(List infos, VideoCodecInfo info) { + for (VideoCodecInfo otherInfo : infos) { + if (isSameCodec(info, otherInfo)) { + return true; + } + } + return false; + } + + private static native boolean isSameCodec(VideoCodecInfo info1, VideoCodecInfo info2); +} diff --git a/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java index 5f86e0a8ec..c14ec5f1d8 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java @@ -15,7 +15,6 @@ import static org.webrtc.MediaCodecUtils.INTEL_PREFIX; import static org.webrtc.MediaCodecUtils.NVIDIA_PREFIX; import static org.webrtc.MediaCodecUtils.QCOM_PREFIX; -import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecList; @@ -27,6 +26,7 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory { private static final String TAG = "HardwareVideoDecoderFactory"; private final EglBase.Context sharedContext; + private final boolean fallbackToSoftware; /** Creates a HardwareVideoDecoderFactory that does not use surface textures. */ @Deprecated // Not removed yet to avoid breaking callers. @@ -39,7 +39,12 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory { * shared context. The context may be null. If it is null, then surface support is disabled. */ public HardwareVideoDecoderFactory(EglBase.Context sharedContext) { + this(sharedContext, true /* fallbackToSoftware */); + } + + HardwareVideoDecoderFactory(EglBase.Context sharedContext, boolean fallbackToSoftware) { this.sharedContext = sharedContext; + this.fallbackToSoftware = fallbackToSoftware; } @Override @@ -48,7 +53,15 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory { MediaCodecInfo info = findCodecForType(type); if (info == null) { - return null; // No support for this codec type. + // No hardware support for this type. + // TODO(andersc): This is for backwards compatibility. Remove when clients have migrated to + // new DefaultVideoEncoderFactory. + if (fallbackToSoftware) { + SoftwareVideoDecoderFactory softwareVideoDecoderFactory = new SoftwareVideoDecoderFactory(); + return softwareVideoDecoderFactory.createDecoder(codecType); + } else { + return null; + } } CodecCapabilities capabilities = info.getCapabilitiesForType(type.mimeType()); diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java index 4b561d4447..02a96f8969 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java @@ -42,9 +42,16 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory { private final EglBase14.Context sharedContext; private final boolean enableIntelVp8Encoder; private final boolean enableH264HighProfile; + private final boolean fallbackToSoftware; public HardwareVideoEncoderFactory( EglBase.Context sharedContext, boolean enableIntelVp8Encoder, boolean enableH264HighProfile) { + this( + sharedContext, enableIntelVp8Encoder, enableH264HighProfile, true /* fallbackToSoftware */); + } + + HardwareVideoEncoderFactory(EglBase.Context sharedContext, boolean enableIntelVp8Encoder, + boolean enableH264HighProfile, boolean fallbackToSoftware) { // Texture mode requires EglBase14. if (sharedContext instanceof EglBase14.Context) { this.sharedContext = (EglBase14.Context) sharedContext; @@ -54,6 +61,7 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory { } this.enableIntelVp8Encoder = enableIntelVp8Encoder; this.enableH264HighProfile = enableH264HighProfile; + this.fallbackToSoftware = fallbackToSoftware; } @Deprecated @@ -67,7 +75,15 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory { MediaCodecInfo info = findCodecForType(type); if (info == null) { - return null; // No support for this type. + // No hardware support for this type. + // TODO(andersc): This is for backwards compatibility. Remove when clients have migrated to + // new DefaultVideoEncoderFactory. + if (fallbackToSoftware) { + SoftwareVideoEncoderFactory softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory(); + return softwareVideoEncoderFactory.createEncoder(input); + } else { + return null; + } } String codecName = info.getName(); @@ -93,12 +109,23 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory { if (codec != null) { String name = type.name(); if (type == VideoCodecType.H264 && isH264HighProfileSupported(codec)) { - supportedCodecInfos.add(new VideoCodecInfo(0, name, getCodecProperties(type, true))); + supportedCodecInfos.add(new VideoCodecInfo(name, getCodecProperties(type, true))); } - supportedCodecInfos.add(new VideoCodecInfo(0, name, getCodecProperties(type, false))); + supportedCodecInfos.add(new VideoCodecInfo(name, getCodecProperties(type, false))); } } + + // TODO(andersc): This is for backwards compatibility. Remove when clients have migrated to + // new DefaultVideoEncoderFactory. + if (fallbackToSoftware) { + for (VideoCodecInfo info : SoftwareVideoEncoderFactory.supportedCodecs()) { + if (!supportedCodecInfos.contains(info)) { + supportedCodecInfos.add(info); + } + } + } + return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); } diff --git a/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java b/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java new file mode 100644 index 0000000000..52c515f6c7 --- /dev/null +++ b/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class SoftwareVideoDecoderFactory implements VideoDecoderFactory { + @Override + public VideoDecoder createDecoder(String codecType) { + if (codecType.equalsIgnoreCase("VP8")) { + return new VP8Decoder(); + } + if (codecType.equalsIgnoreCase("VP9")) { + return new VP9Decoder(); + } + + return null; + } +} diff --git a/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java new file mode 100644 index 0000000000..8fb112e6cf --- /dev/null +++ b/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java @@ -0,0 +1,39 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.util.HashMap; + +public class SoftwareVideoEncoderFactory implements VideoEncoderFactory { + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + if (info.name.equalsIgnoreCase("VP8")) { + return new VP8Encoder(); + } + if (info.name.equalsIgnoreCase("VP9")) { + return new VP9Encoder(); + } + + return null; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return supportedCodecs(); + } + + public static VideoCodecInfo[] supportedCodecs() { + VideoCodecInfo vp8Info = new VideoCodecInfo("VP8", new HashMap<>()); + VideoCodecInfo vp9Info = new VideoCodecInfo("VP9", new HashMap<>()); + + return new VideoCodecInfo[] {vp8Info, vp9Info}; + } +} diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java index a2ed5fc4b1..208a99203b 100644 --- a/sdk/android/api/org/webrtc/VideoCodecInfo.java +++ b/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -29,10 +29,17 @@ public class VideoCodecInfo { public static final String H264_CONSTRAINED_BASELINE_3_1 = H264_PROFILE_CONSTRAINED_BASELINE + H264_LEVEL_3_1; - public final int payload; public final String name; public final Map params; + @Deprecated public final int payload; + public VideoCodecInfo(String name, Map params) { + this.payload = 0; + this.name = name; + this.params = params; + } + + @Deprecated public VideoCodecInfo(int payload, String name, Map params) { this.payload = payload; this.name = name; diff --git a/sdk/android/api/org/webrtc/VideoDecoderFallback.java b/sdk/android/api/org/webrtc/VideoDecoderFallback.java new file mode 100644 index 0000000000..7c4d8caa0e --- /dev/null +++ b/sdk/android/api/org/webrtc/VideoDecoderFallback.java @@ -0,0 +1,22 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * A combined video decoder that falls back on a secondary decoder if the primary decoder fails. + */ +public class VideoDecoderFallback extends WrappedNativeVideoDecoder { + public VideoDecoderFallback(VideoDecoder fallback, VideoDecoder primary) { + super(createNativeDecoder(fallback, primary)); + } + + private static native long createNativeDecoder(VideoDecoder fallback, VideoDecoder primary); +} diff --git a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java new file mode 100644 index 0000000000..d1a3b6629b --- /dev/null +++ b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java @@ -0,0 +1,111 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import static org.junit.Assert.assertEquals; + +import android.support.test.filters.SmallTest; +import android.util.Log; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import org.chromium.base.test.BaseJUnit4ClassRunner; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link DefaultVideoEncoderFactory}. */ +@RunWith(BaseJUnit4ClassRunner.class) +public class DefaultVideoEncoderFactoryTest { + static class CustomHardwareVideoEncoderFactory implements VideoEncoderFactory { + private ArrayList codecs = new ArrayList<>(); + + public CustomHardwareVideoEncoderFactory(boolean includeVP8, boolean includeH264High) { + if (includeVP8) { + codecs.add(new VideoCodecInfo("VP8", new HashMap<>())); + } + codecs.add(new VideoCodecInfo("VP9", new HashMap<>())); + + HashMap baselineParams = new HashMap(); + baselineParams.put("profile-level-id", "42e01f"); + baselineParams.put("level-asymmetry-allowed", "1"); + baselineParams.put("packetization-mode", "1"); + codecs.add(new VideoCodecInfo("H264", baselineParams)); + + if (includeH264High) { + HashMap highParams = new HashMap(); + highParams.put("profile-level-id", "640c1f"); + highParams.put("level-asymmetry-allowed", "1"); + highParams.put("packetization-mode", "1"); + codecs.add(new VideoCodecInfo("H264", highParams)); + } + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + return null; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return codecs.toArray(new VideoCodecInfo[codecs.size()]); + } + } + + @Before + public void setUp() { + NativeLibrary.initialize(new NativeLibrary.DefaultLoader()); + } + + @SmallTest + @Test + public void testGetSupportedCodecsWithHardwareH264HighProfile() { + VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, true); + DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory); + VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs(); + assertEquals(4, videoCodecs.length); + assertEquals("VP8", videoCodecs[0].name); + assertEquals("VP9", videoCodecs[1].name); + assertEquals("H264", videoCodecs[2].name); + assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id")); + assertEquals("H264", videoCodecs[3].name); + assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id")); + } + + @SmallTest + @Test + public void testGetSupportedCodecsWithoutHardwareH264HighProfile() { + VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, false); + DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory); + VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs(); + assertEquals(3, videoCodecs.length); + assertEquals("VP8", videoCodecs[0].name); + assertEquals("VP9", videoCodecs[1].name); + assertEquals("H264", videoCodecs[2].name); + assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id")); + } + + @SmallTest + @Test + public void testGetSupportedCodecsWithoutHardwareVP8() { + VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(false, true); + DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory); + VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs(); + assertEquals(4, videoCodecs.length); + assertEquals("VP8", videoCodecs[0].name); + assertEquals("VP9", videoCodecs[1].name); + assertEquals("H264", videoCodecs[2].name); + assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id")); + assertEquals("H264", videoCodecs[3].name); + assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id")); + } +} diff --git a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java b/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java index 4260397ecf..f680102a7c 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java @@ -127,8 +127,8 @@ public final class HardwareVideoDecoderTest { private void encodeTestFrames() { VideoEncoderFactory encoderFactory = new HardwareVideoEncoderFactory( eglBase.getEglBaseContext(), ENABLE_INTEL_VP8_ENCODER, ENABLE_H264_HIGH_PROFILE); - VideoEncoder encoder = encoderFactory.createEncoder( - new VideoCodecInfo(0 /* payload*/, codecType, new HashMap<>())); + VideoEncoder encoder = + encoderFactory.createEncoder(new VideoCodecInfo(codecType, new HashMap<>())); HardwareVideoEncoderTest.MockEncoderCallback encodeCallback = new HardwareVideoEncoderTest.MockEncoderCallback(); assertEquals(VideoCodecStatus.OK, encoder.initEncode(ENCODER_SETTINGS, encodeCallback)); diff --git a/sdk/android/src/java/org/webrtc/VP8Decoder.java b/sdk/android/src/java/org/webrtc/VP8Decoder.java new file mode 100644 index 0000000000..474107b32b --- /dev/null +++ b/sdk/android/src/java/org/webrtc/VP8Decoder.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +class VP8Decoder extends WrappedNativeVideoDecoder { + VP8Decoder() { + super(createNativeDecoder()); + } + + private static native long createNativeDecoder(); +} diff --git a/sdk/android/src/java/org/webrtc/VP8Encoder.java b/sdk/android/src/java/org/webrtc/VP8Encoder.java new file mode 100644 index 0000000000..505a7a9aee --- /dev/null +++ b/sdk/android/src/java/org/webrtc/VP8Encoder.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +class VP8Encoder extends WrappedNativeVideoEncoder { + VP8Encoder() { + super(createNativeEncoder()); + } + + private static native long createNativeEncoder(); +} diff --git a/sdk/android/src/java/org/webrtc/VP9Decoder.java b/sdk/android/src/java/org/webrtc/VP9Decoder.java new file mode 100644 index 0000000000..96f2086534 --- /dev/null +++ b/sdk/android/src/java/org/webrtc/VP9Decoder.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +class VP9Decoder extends WrappedNativeVideoDecoder { + VP9Decoder() { + super(createNativeDecoder()); + } + + private static native long createNativeDecoder(); +} diff --git a/sdk/android/src/java/org/webrtc/VP9Encoder.java b/sdk/android/src/java/org/webrtc/VP9Encoder.java new file mode 100644 index 0000000000..203fcbf155 --- /dev/null +++ b/sdk/android/src/java/org/webrtc/VP9Encoder.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +class VP9Encoder extends WrappedNativeVideoEncoder { + VP9Encoder() { + super(createNativeEncoder()); + } + + private static native long createNativeEncoder(); +} diff --git a/sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java b/sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java new file mode 100644 index 0000000000..d94c4ada97 --- /dev/null +++ b/sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Wraps a native webrtc::VideoDecoder. + */ +class WrappedNativeVideoDecoder implements VideoDecoder { + private final long nativeDecoder; + + WrappedNativeVideoDecoder(long nativeDecoder) { + this.nativeDecoder = nativeDecoder; + } + + @CalledByNative + public long getNativeDecoder() { + return this.nativeDecoder; + } + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback decodeCallback) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public VideoCodecStatus release() { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public VideoCodecStatus decode(EncodedImage frame, DecodeInfo info) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public boolean getPrefersLateDecoding() { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public String getImplementationName() { + throw new UnsupportedOperationException("Not implemented."); + } +} diff --git a/sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java b/sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java new file mode 100644 index 0000000000..c0b145ea58 --- /dev/null +++ b/sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Wraps a native webrtc::VideoEncoder. + */ +class WrappedNativeVideoEncoder implements VideoEncoder { + private final long nativeEncoder; + + WrappedNativeVideoEncoder(long nativeEncoder) { + this.nativeEncoder = nativeEncoder; + } + + @CalledByNative + public long getNativeEncoder() { + return this.nativeEncoder; + } + + @Override + public VideoCodecStatus initEncode(Settings settings, Callback encodeCallback) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public VideoCodecStatus release() { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public VideoCodecStatus setChannelParameters(short packetLoss, long roundTripTimeMs) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public ScalingSettings getScalingSettings() { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public String getImplementationName() { + throw new UnsupportedOperationException("Not implemented."); + } +} diff --git a/sdk/android/src/jni/DEPS b/sdk/android/src/jni/DEPS index 4ac0aac23d..db95db649f 100644 --- a/sdk/android/src/jni/DEPS +++ b/sdk/android/src/jni/DEPS @@ -5,6 +5,7 @@ include_rules = [ "+logging/rtc_event_log/rtc_event_log_factory_interface.h", "+media/base", "+media/engine", + "+modules/audio_device/include/audio_device.h", "+modules/audio_processing/include/audio_processing.h", "+modules/include", "+modules/utility/include/jvm_android.h", diff --git a/sdk/android/src/jni/defaultvideoencoderfactory.cc b/sdk/android/src/jni/defaultvideoencoderfactory.cc new file mode 100644 index 0000000000..eb04c4fa56 --- /dev/null +++ b/sdk/android/src/jni/defaultvideoencoderfactory.cc @@ -0,0 +1,54 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "media/base/codec.h" +#include "media/base/h264_profile_level_id.h" +#include "media/base/mediaconstants.h" +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/videocodecinfo.h" + +namespace webrtc { +namespace jni { + +static bool IsSameH264Profile(const cricket::CodecParameterMap& params1, + const cricket::CodecParameterMap& params2) { + const rtc::Optional profile_level_id = + H264::ParseSdpProfileLevelId(params1); + const rtc::Optional other_profile_level_id = + H264::ParseSdpProfileLevelId(params2); + // Compare H264 profiles, but not levels. + return profile_level_id && other_profile_level_id && + profile_level_id->profile == other_profile_level_id->profile; +} + +JNI_FUNCTION_DECLARATION(jboolean, + DefaultVideoEncoderFactory_isSameCodec, + JNIEnv* jni, + jclass, + jobject info1, + jobject info2) { + cricket::VideoCodec codec1 = + cricket::VideoCodec(VideoCodecInfoToSdpVideoFormat(jni, info1)); + cricket::VideoCodec codec2 = + cricket::VideoCodec(VideoCodecInfoToSdpVideoFormat(jni, info2)); + + if (!cricket::CodecNamesEq(codec1.name, codec2.name)) + return false; + if (cricket::CodecNamesEq(codec1.name.c_str(), cricket::kH264CodecName) && + !IsSameH264Profile(codec1.params, codec2.params)) { + return false; + } + return true; +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/media_jni.cc b/sdk/android/src/jni/pc/media_jni.cc index 32b9020c56..8450775300 100644 --- a/sdk/android/src/jni/pc/media_jni.cc +++ b/sdk/android/src/jni/pc/media_jni.cc @@ -9,9 +9,12 @@ */ #include "sdk/android/src/jni/pc/media_jni.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" #include "call/callfactoryinterface.h" #include "logging/rtc_event_log/rtc_event_log_factory_interface.h" #include "media/engine/webrtcmediaengine.h" +#include "modules/audio_device/include/audio_device.h" #include "modules/audio_processing/include/audio_processing.h" namespace webrtc { @@ -38,5 +41,20 @@ cricket::MediaEngineInterface* CreateMediaEngine( video_decoder_factory, audio_mixer, audio_processor); } +cricket::MediaEngineInterface* CreateMediaEngine( + rtc::scoped_refptr adm, + rtc::scoped_refptr audio_encoder_factory, + rtc::scoped_refptr audio_decoder_factory, + std::unique_ptr video_encoder_factory, + std::unique_ptr video_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processor) { + return cricket::WebRtcMediaEngineFactory::Create( + adm, audio_encoder_factory, audio_decoder_factory, + std::move(video_encoder_factory), std::move(video_decoder_factory), + audio_mixer, audio_processor) + .release(); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/pc/media_jni.h b/sdk/android/src/jni/pc/media_jni.h index 13a8bb5000..6ed0e7e1d5 100644 --- a/sdk/android/src/jni/pc/media_jni.h +++ b/sdk/android/src/jni/pc/media_jni.h @@ -21,6 +21,8 @@ class AudioDecoderFactory; class RtcEventLogFactoryInterface; class AudioMixer; class AudioProcessing; +class VideoEncoderFactory; +class VideoDecoderFactory; } // namespace webrtc namespace cricket { @@ -44,6 +46,15 @@ cricket::MediaEngineInterface* CreateMediaEngine( rtc::scoped_refptr audio_mixer, rtc::scoped_refptr audio_processor); +cricket::MediaEngineInterface* CreateMediaEngine( + rtc::scoped_refptr adm, + rtc::scoped_refptr audio_encoder_factory, + rtc::scoped_refptr audio_decoder_factory, + std::unique_ptr video_encoder_factory, + std::unique_ptr video_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processor); + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/pc/null_media_jni.cc b/sdk/android/src/jni/pc/null_media_jni.cc index 8a39028164..25e5124771 100644 --- a/sdk/android/src/jni/pc/null_media_jni.cc +++ b/sdk/android/src/jni/pc/null_media_jni.cc @@ -32,5 +32,16 @@ cricket::MediaEngineInterface* CreateMediaEngine( return nullptr; } +cricket::MediaEngineInterface* CreateMediaEngine( + rtc::scoped_refptr adm, + rtc::scoped_refptr audio_encoder_factory, + rtc::scoped_refptr audio_decoder_factory, + std::unique_ptr video_encoder_factory, + std::unique_ptr video_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processor) { + return nullptr; +} + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/pc/null_video_jni.cc b/sdk/android/src/jni/pc/null_video_jni.cc index 818cc9cd32..88ed665b14 100644 --- a/sdk/android/src/jni/pc/null_video_jni.cc +++ b/sdk/android/src/jni/pc/null_video_jni.cc @@ -13,15 +13,31 @@ namespace webrtc { namespace jni { -cricket::WebRtcVideoEncoderFactory* CreateVideoEncoderFactory( - JNIEnv* jni, - jobject j_encoder_factory) { +VideoEncoderFactory* CreateVideoEncoderFactory(JNIEnv* jni, + jobject j_encoder_factory) { return nullptr; } -cricket::WebRtcVideoDecoderFactory* CreateVideoDecoderFactory( - JNIEnv* jni, - jobject j_decoder_factory) { +VideoDecoderFactory* CreateVideoDecoderFactory(JNIEnv* jni, + jobject j_decoder_factory) { + return nullptr; +} + +cricket::WebRtcVideoEncoderFactory* CreateLegacyVideoEncoderFactory() { + return nullptr; +} + +cricket::WebRtcVideoDecoderFactory* CreateLegacyVideoDecoderFactory() { + return nullptr; +} + +VideoEncoderFactory* WrapLegacyVideoEncoderFactory( + cricket::WebRtcVideoEncoderFactory* legacy_encoder_factory) { + return nullptr; +} + +VideoDecoderFactory* WrapLegacyVideoDecoderFactory( + cricket::WebRtcVideoDecoderFactory* legacy_decoder_factory) { return nullptr; } diff --git a/sdk/android/src/jni/pc/ownedfactoryandthreads.h b/sdk/android/src/jni/pc/ownedfactoryandthreads.h index c73d7d7fd8..e4d1cf6b18 100644 --- a/sdk/android/src/jni/pc/ownedfactoryandthreads.h +++ b/sdk/android/src/jni/pc/ownedfactoryandthreads.h @@ -38,15 +38,15 @@ class OwnedFactoryAndThreads { OwnedFactoryAndThreads(std::unique_ptr network_thread, std::unique_ptr worker_thread, std::unique_ptr signaling_thread, - WebRtcVideoEncoderFactory* encoder_factory, - WebRtcVideoDecoderFactory* decoder_factory, + WebRtcVideoEncoderFactory* legacy_encoder_factory, + WebRtcVideoDecoderFactory* legacy_decoder_factory, rtc::NetworkMonitorFactory* network_monitor_factory, PeerConnectionFactoryInterface* factory) : network_thread_(std::move(network_thread)), worker_thread_(std::move(worker_thread)), signaling_thread_(std::move(signaling_thread)), - encoder_factory_(encoder_factory), - decoder_factory_(decoder_factory), + legacy_encoder_factory_(legacy_encoder_factory), + legacy_decoder_factory_(legacy_decoder_factory), network_monitor_factory_(network_monitor_factory), factory_(factory) {} @@ -55,8 +55,12 @@ class OwnedFactoryAndThreads { PeerConnectionFactoryInterface* factory() { return factory_; } Thread* signaling_thread() { return signaling_thread_.get(); } Thread* worker_thread() { return worker_thread_.get(); } - WebRtcVideoEncoderFactory* encoder_factory() { return encoder_factory_; } - WebRtcVideoDecoderFactory* decoder_factory() { return decoder_factory_; } + WebRtcVideoEncoderFactory* legacy_encoder_factory() { + return legacy_encoder_factory_; + } + WebRtcVideoDecoderFactory* legacy_decoder_factory() { + return legacy_decoder_factory_; + } rtc::NetworkMonitorFactory* network_monitor_factory() { return network_monitor_factory_; } @@ -69,8 +73,8 @@ class OwnedFactoryAndThreads { const std::unique_ptr network_thread_; const std::unique_ptr worker_thread_; const std::unique_ptr signaling_thread_; - WebRtcVideoEncoderFactory* encoder_factory_; - WebRtcVideoDecoderFactory* decoder_factory_; + WebRtcVideoEncoderFactory* legacy_encoder_factory_; + WebRtcVideoDecoderFactory* legacy_decoder_factory_; rtc::NetworkMonitorFactory* network_monitor_factory_; PeerConnectionFactoryInterface* factory_; // Const after ctor except dtor. }; diff --git a/sdk/android/src/jni/pc/peerconnectionfactory_jni.cc b/sdk/android/src/jni/pc/peerconnectionfactory_jni.cc index b335cff620..a1b9b5e85a 100644 --- a/sdk/android/src/jni/pc/peerconnectionfactory_jni.cc +++ b/sdk/android/src/jni/pc/peerconnectionfactory_jni.cc @@ -12,7 +12,10 @@ #include #include "api/peerconnectioninterface.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" #include "media/base/mediaengine.h" +#include "modules/audio_device/include/audio_device.h" #include "modules/utility/include/jvm_android.h" // We don't depend on the audio processing module implementation. // The user may pass in a nullptr. @@ -161,8 +164,6 @@ jlong CreatePeerConnectionFactoryForJava( signaling_thread->SetName("signaling_thread", NULL); RTC_CHECK(signaling_thread->Start()) << "Failed to start thread"; - cricket::WebRtcVideoEncoderFactory* video_encoder_factory = nullptr; - cricket::WebRtcVideoDecoderFactory* video_decoder_factory = nullptr; rtc::NetworkMonitorFactory* network_monitor_factory = nullptr; auto audio_encoder_factory = CreateAudioEncoderFactory(); auto audio_decoder_factory = CreateAudioDecoderFactory(); @@ -173,10 +174,6 @@ jlong CreatePeerConnectionFactoryForJava( options = JavaToNativePeerConnectionFactoryOptions(jni, joptions); } - if (video_hw_acceleration_enabled) { - video_encoder_factory = CreateVideoEncoderFactory(jni, jencoder_factory); - video_decoder_factory = CreateVideoDecoderFactory(jni, jdecoder_factory); - } // Do not create network_monitor_factory only if the options are // provided and disable_network_monitor therein is set to true. if (!(has_options && options.disable_network_monitor)) { @@ -189,9 +186,49 @@ jlong CreatePeerConnectionFactoryForJava( std::unique_ptr call_factory(CreateCallFactory()); std::unique_ptr rtc_event_log_factory( CreateRtcEventLogFactory()); - std::unique_ptr media_engine(CreateMediaEngine( - adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory, - video_decoder_factory, audio_mixer, audio_processor)); + + cricket::WebRtcVideoEncoderFactory* legacy_video_encoder_factory = nullptr; + cricket::WebRtcVideoDecoderFactory* legacy_video_decoder_factory = nullptr; + std::unique_ptr media_engine; + if (jencoder_factory == nullptr && jdecoder_factory == nullptr) { + // This uses the legacy API, which automatically uses the internal SW + // codecs in WebRTC. + if (video_hw_acceleration_enabled) { + legacy_video_encoder_factory = CreateLegacyVideoEncoderFactory(); + legacy_video_decoder_factory = CreateLegacyVideoDecoderFactory(); + } + media_engine.reset(CreateMediaEngine( + adm, audio_encoder_factory, audio_decoder_factory, + legacy_video_encoder_factory, legacy_video_decoder_factory, audio_mixer, + audio_processor)); + } else { + // This uses the new API, does not automatically include software codecs. + std::unique_ptr video_encoder_factory = nullptr; + if (jencoder_factory == nullptr) { + legacy_video_encoder_factory = CreateLegacyVideoEncoderFactory(); + video_encoder_factory = std::unique_ptr( + WrapLegacyVideoEncoderFactory(legacy_video_encoder_factory)); + } else { + video_encoder_factory = std::unique_ptr( + CreateVideoEncoderFactory(jni, jencoder_factory)); + } + + std::unique_ptr video_decoder_factory = nullptr; + if (jdecoder_factory == nullptr) { + legacy_video_decoder_factory = CreateLegacyVideoDecoderFactory(); + video_decoder_factory = std::unique_ptr( + WrapLegacyVideoDecoderFactory(legacy_video_decoder_factory)); + } else { + video_decoder_factory = std::unique_ptr( + CreateVideoDecoderFactory(jni, jdecoder_factory)); + } + + rtc::scoped_refptr adm_scoped = nullptr; + media_engine.reset(CreateMediaEngine( + adm_scoped, audio_encoder_factory, audio_decoder_factory, + std::move(video_encoder_factory), std::move(video_decoder_factory), + audio_mixer, audio_processor)); + } rtc::scoped_refptr factory( CreateModularPeerConnectionFactory( @@ -207,8 +244,8 @@ jlong CreatePeerConnectionFactoryForJava( } OwnedFactoryAndThreads* owned_factory = new OwnedFactoryAndThreads( std::move(network_thread), std::move(worker_thread), - std::move(signaling_thread), video_encoder_factory, video_decoder_factory, - network_monitor_factory, factory.release()); + std::move(signaling_thread), legacy_video_encoder_factory, + legacy_video_decoder_factory, network_monitor_factory, factory.release()); owned_factory->InvokeJavaCallbacksOnFactoryThreads(); return jlongFromPointer(owned_factory); } diff --git a/sdk/android/src/jni/pc/video_jni.cc b/sdk/android/src/jni/pc/video_jni.cc index 96ee33935d..f7f60ff659 100644 --- a/sdk/android/src/jni/pc/video_jni.cc +++ b/sdk/android/src/jni/pc/video_jni.cc @@ -10,7 +10,10 @@ #include +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" #include "api/videosourceproxy.h" +#include "media/engine/convert_legacy_video_factory.h" #include "media/engine/webrtcvideodecoderfactory.h" #include "media/engine/webrtcvideoencoderfactory.h" #include "rtc_base/logging.h" @@ -26,35 +29,38 @@ namespace webrtc { namespace jni { -// TODO(sakal): Remove this once MediaCodecVideoDecoder/Encoder are no longer -// used and all applications inject their own codecs. -// This is semi broken if someone wants to create multiple peerconnection -// factories. -static bool use_media_codec_encoder_factory; -static bool use_media_codec_decoder_factory; - -cricket::WebRtcVideoEncoderFactory* CreateVideoEncoderFactory( - JNIEnv* jni, - jobject j_encoder_factory) { - use_media_codec_encoder_factory = j_encoder_factory == nullptr; - - if (use_media_codec_encoder_factory) { - return new MediaCodecVideoEncoderFactory(); - } else { - return new VideoEncoderFactoryWrapper(jni, j_encoder_factory); - } +VideoEncoderFactory* CreateVideoEncoderFactory(JNIEnv* jni, + jobject j_encoder_factory) { + return new VideoEncoderFactoryWrapper(jni, j_encoder_factory); } -cricket::WebRtcVideoDecoderFactory* CreateVideoDecoderFactory( - JNIEnv* jni, - jobject j_decoder_factory) { - use_media_codec_decoder_factory = j_decoder_factory == nullptr; +VideoDecoderFactory* CreateVideoDecoderFactory(JNIEnv* jni, + jobject j_decoder_factory) { + return new VideoDecoderFactoryWrapper(jni, j_decoder_factory); +} - if (use_media_codec_decoder_factory) { - return new MediaCodecVideoDecoderFactory(); - } else { - return new VideoDecoderFactoryWrapper(jni, j_decoder_factory); - } +cricket::WebRtcVideoEncoderFactory* CreateLegacyVideoEncoderFactory() { + return new MediaCodecVideoEncoderFactory(); +} + +cricket::WebRtcVideoDecoderFactory* CreateLegacyVideoDecoderFactory() { + return new MediaCodecVideoDecoderFactory(); +} + +VideoEncoderFactory* WrapLegacyVideoEncoderFactory( + cricket::WebRtcVideoEncoderFactory* legacy_encoder_factory) { + return ConvertVideoEncoderFactory( + std::unique_ptr( + legacy_encoder_factory)) + .release(); +} + +VideoDecoderFactory* WrapLegacyVideoDecoderFactory( + cricket::WebRtcVideoDecoderFactory* legacy_decoder_factory) { + return ConvertVideoDecoderFactory( + std::unique_ptr( + legacy_decoder_factory)) + .release(); } jobject GetJavaSurfaceTextureHelper( @@ -114,21 +120,25 @@ JNI_FUNCTION_DECLARATION( jclass j_eglbase14_context_class = FindClass(jni, "org/webrtc/EglBase14$Context"); - MediaCodecVideoEncoderFactory* encoder_factory = - static_cast( - owned_factory->encoder_factory()); - if (use_media_codec_encoder_factory && encoder_factory && - jni->IsInstanceOf(local_egl_context, j_eglbase14_context_class)) { - RTC_LOG(LS_INFO) << "Set EGL context for HW encoding."; - encoder_factory->SetEGLContext(jni, local_egl_context); + if (owned_factory->legacy_encoder_factory()) { + MediaCodecVideoEncoderFactory* encoder_factory = + static_cast( + owned_factory->legacy_encoder_factory()); + if (encoder_factory && + jni->IsInstanceOf(local_egl_context, j_eglbase14_context_class)) { + RTC_LOG(LS_INFO) << "Set EGL context for HW encoding."; + encoder_factory->SetEGLContext(jni, local_egl_context); + } } - MediaCodecVideoDecoderFactory* decoder_factory = - static_cast( - owned_factory->decoder_factory()); - if (use_media_codec_decoder_factory && decoder_factory) { - RTC_LOG(LS_INFO) << "Set EGL context for HW decoding."; - decoder_factory->SetEGLContext(jni, remote_egl_context); + if (owned_factory->legacy_decoder_factory()) { + MediaCodecVideoDecoderFactory* decoder_factory = + static_cast( + owned_factory->legacy_decoder_factory()); + if (decoder_factory) { + RTC_LOG(LS_INFO) << "Set EGL context for HW decoding."; + decoder_factory->SetEGLContext(jni, remote_egl_context); + } } } diff --git a/sdk/android/src/jni/pc/video_jni.h b/sdk/android/src/jni/pc/video_jni.h index 0c2ccdae09..a73c093d11 100644 --- a/sdk/android/src/jni/pc/video_jni.h +++ b/sdk/android/src/jni/pc/video_jni.h @@ -20,18 +20,29 @@ class WebRtcVideoEncoderFactory; class WebRtcVideoDecoderFactory; } // namespace cricket +namespace webrtc { +class VideoEncoderFactory; +class VideoDecoderFactory; +} // namespace webrtc + namespace webrtc { namespace jni { class SurfaceTextureHelper; -cricket::WebRtcVideoEncoderFactory* CreateVideoEncoderFactory( - JNIEnv* jni, - jobject j_encoder_factory); +VideoEncoderFactory* CreateVideoEncoderFactory(JNIEnv* jni, + jobject j_encoder_factory); -cricket::WebRtcVideoDecoderFactory* CreateVideoDecoderFactory( - JNIEnv* jni, - jobject j_decoder_factory); +VideoDecoderFactory* CreateVideoDecoderFactory(JNIEnv* jni, + jobject j_decoder_factory); + +cricket::WebRtcVideoEncoderFactory* CreateLegacyVideoEncoderFactory(); +cricket::WebRtcVideoDecoderFactory* CreateLegacyVideoDecoderFactory(); + +VideoEncoderFactory* WrapLegacyVideoEncoderFactory( + cricket::WebRtcVideoEncoderFactory* legacy_encoder_factory); +VideoDecoderFactory* WrapLegacyVideoDecoderFactory( + cricket::WebRtcVideoDecoderFactory* legacy_decoder_factory); jobject GetJavaSurfaceTextureHelper( const rtc::scoped_refptr& surface_texture_helper); diff --git a/sdk/android/src/jni/videocodecinfo.cc b/sdk/android/src/jni/videocodecinfo.cc new file mode 100644 index 0000000000..2566673dfe --- /dev/null +++ b/sdk/android/src/jni/videocodecinfo.cc @@ -0,0 +1,57 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/android/src/jni/videocodecinfo.h" + +#include "sdk/android/src/jni/classreferenceholder.h" +#include "sdk/android/src/jni/jni_helpers.h" + +namespace webrtc { +namespace jni { + +SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni, jobject j_info) { + jclass video_codec_info_class = FindClass(jni, "org/webrtc/VideoCodecInfo"); + jfieldID name_field = + GetFieldID(jni, video_codec_info_class, "name", "Ljava/lang/String;"); + jfieldID params_field = + GetFieldID(jni, video_codec_info_class, "params", "Ljava/util/Map;"); + + jobject j_params = jni->GetObjectField(j_info, params_field); + jstring j_name = + static_cast(jni->GetObjectField(j_info, name_field)); + + return SdpVideoFormat(JavaToStdString(jni, j_name), + JavaToStdMapStrings(jni, j_params)); +} + +jobject SdpVideoFormatToVideoCodecInfo(JNIEnv* jni, + const SdpVideoFormat& format) { + jclass hash_map_class = jni->FindClass("java/util/HashMap"); + jmethodID hash_map_constructor = + jni->GetMethodID(hash_map_class, "", "()V"); + jmethodID put_method = jni->GetMethodID( + hash_map_class, "put", + "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); + jclass video_codec_info_class = FindClass(jni, "org/webrtc/VideoCodecInfo"); + jmethodID video_codec_info_constructor = jni->GetMethodID( + video_codec_info_class, "", "(Ljava/lang/String;Ljava/util/Map;)V"); + + jobject j_params = jni->NewObject(hash_map_class, hash_map_constructor); + for (auto const& param : format.parameters) { + jni->CallObjectMethod(j_params, put_method, + JavaStringFromStdString(jni, param.first), + JavaStringFromStdString(jni, param.second)); + } + return jni->NewObject(video_codec_info_class, video_codec_info_constructor, + JavaStringFromStdString(jni, format.name), j_params); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/videocodecinfo.h b/sdk/android/src/jni/videocodecinfo.h new file mode 100644 index 0000000000..ea72273a55 --- /dev/null +++ b/sdk/android/src/jni/videocodecinfo.h @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_ANDROID_SRC_JNI_VIDEOCODECINFO_H_ +#define SDK_ANDROID_SRC_JNI_VIDEOCODECINFO_H_ + +#include + +#include "api/video_codecs/sdp_video_format.h" + +namespace webrtc { +namespace jni { + +SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni, jobject info); +jobject SdpVideoFormatToVideoCodecInfo(JNIEnv* jni, + const SdpVideoFormat& format); + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_VIDEOCODECINFO_H_ diff --git a/sdk/android/src/jni/videodecoderfactorywrapper.cc b/sdk/android/src/jni/videodecoderfactorywrapper.cc index b1b784c63c..f976584a1a 100644 --- a/sdk/android/src/jni/videodecoderfactorywrapper.cc +++ b/sdk/android/src/jni/videodecoderfactorywrapper.cc @@ -10,10 +10,11 @@ #include "sdk/android/src/jni/videodecoderfactorywrapper.h" +#include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder.h" #include "common_types.h" // NOLINT(build/include) #include "rtc_base/logging.h" -#include "sdk/android/src/jni/videodecoderwrapper.h" +#include "sdk/android/src/jni/wrappednativecodec.h" namespace webrtc { namespace jni { @@ -27,19 +28,20 @@ VideoDecoderFactoryWrapper::VideoDecoderFactoryWrapper(JNIEnv* jni, "(Ljava/lang/String;)Lorg/webrtc/VideoDecoder;"); } -VideoDecoder* VideoDecoderFactoryWrapper::CreateVideoDecoderWithParams( - const cricket::VideoCodec& codec, - cricket::VideoDecoderParams params) { +std::unique_ptr VideoDecoderFactoryWrapper::CreateVideoDecoder( + const SdpVideoFormat& format) { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); - jstring name = JavaStringFromStdString(jni, codec.name); + jstring name = JavaStringFromStdString(jni, format.name); jobject decoder = jni->CallObjectMethod(*decoder_factory_, create_decoder_method_, name); - return decoder != nullptr ? new VideoDecoderWrapper(jni, decoder) : nullptr; + return decoder != nullptr ? WrapOrUnwrapVideoDecoder(jni, decoder) : nullptr; } -void VideoDecoderFactoryWrapper::DestroyVideoDecoder(VideoDecoder* decoder) { - delete decoder; +std::vector VideoDecoderFactoryWrapper::GetSupportedFormats() + const { + // TODO(andersc): VideoDecoderFactory.java does not have this method. + return std::vector(); } } // namespace jni diff --git a/sdk/android/src/jni/videodecoderfactorywrapper.h b/sdk/android/src/jni/videodecoderfactorywrapper.h index 0ad3ff230a..3ba5ab69a0 100644 --- a/sdk/android/src/jni/videodecoderfactorywrapper.h +++ b/sdk/android/src/jni/videodecoderfactorywrapper.h @@ -13,7 +13,7 @@ #include -#include "media/engine/webrtcvideodecoderfactory.h" +#include "api/video_codecs/video_decoder_factory.h" #include "sdk/android/src/jni/jni_helpers.h" namespace webrtc { @@ -21,16 +21,13 @@ namespace jni { // Wrapper for Java VideoDecoderFactory class. Delegates method calls through // JNI and wraps the decoder inside VideoDecoderWrapper. -class VideoDecoderFactoryWrapper : public cricket::WebRtcVideoDecoderFactory { +class VideoDecoderFactoryWrapper : public VideoDecoderFactory { public: VideoDecoderFactoryWrapper(JNIEnv* jni, jobject decoder_factory); - // Caller takes the ownership of the returned object and it should be released - // by calling DestroyVideoDecoder(). - VideoDecoder* CreateVideoDecoderWithParams( - const cricket::VideoCodec& codec, - cricket::VideoDecoderParams params) override; - void DestroyVideoDecoder(VideoDecoder* decoder) override; + std::vector GetSupportedFormats() const override; + std::unique_ptr CreateVideoDecoder( + const SdpVideoFormat& format) override; private: const ScopedGlobalRef decoder_factory_; diff --git a/sdk/android/src/jni/videodecoderfallback.cc b/sdk/android/src/jni/videodecoderfallback.cc new file mode 100644 index 0000000000..594cde24cc --- /dev/null +++ b/sdk/android/src/jni/videodecoderfallback.cc @@ -0,0 +1,39 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "media/engine/videodecodersoftwarefallbackwrapper.h" +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/wrappednativecodec.h" + +namespace webrtc { +namespace jni { + +JNI_FUNCTION_DECLARATION(jlong, + VideoDecoderFallback_createNativeDecoder, + JNIEnv* jni, + jclass, + jobject j_fallback_decoder, + jobject j_primary_decoder) { + std::unique_ptr fallback_decoder = + WrapOrUnwrapVideoDecoder(jni, j_fallback_decoder); + std::unique_ptr primary_decoder = + WrapOrUnwrapVideoDecoder(jni, j_primary_decoder); + + VideoDecoderSoftwareFallbackWrapper* nativeWrapper = + new VideoDecoderSoftwareFallbackWrapper(std::move(fallback_decoder), + std::move(primary_decoder)); + + return jlongFromPointer(nativeWrapper); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/videoencoderfactorywrapper.cc b/sdk/android/src/jni/videoencoderfactorywrapper.cc index 5bb904769a..3171a4e032 100644 --- a/sdk/android/src/jni/videoencoderfactorywrapper.cc +++ b/sdk/android/src/jni/videoencoderfactorywrapper.cc @@ -13,17 +13,16 @@ #include "api/video_codecs/video_encoder.h" #include "common_types.h" // NOLINT(build/include) #include "rtc_base/logging.h" -#include "sdk/android/src/jni/classreferenceholder.h" -#include "sdk/android/src/jni/videoencoderwrapper.h" +#include "sdk/android/src/jni/class_loader.h" +#include "sdk/android/src/jni/videocodecinfo.h" +#include "sdk/android/src/jni/wrappednativecodec.h" namespace webrtc { namespace jni { VideoEncoderFactoryWrapper::VideoEncoderFactoryWrapper(JNIEnv* jni, jobject encoder_factory) - : video_codec_info_class_(jni, FindClass(jni, "org/webrtc/VideoCodecInfo")), - hash_map_class_(jni, jni->FindClass("java/util/HashMap")), - encoder_factory_(jni, encoder_factory) { + : encoder_factory_(jni, encoder_factory) { jclass encoder_factory_class = jni->GetObjectClass(*encoder_factory_); create_encoder_method_ = jni->GetMethodID( encoder_factory_class, "createEncoder", @@ -32,71 +31,53 @@ VideoEncoderFactoryWrapper::VideoEncoderFactoryWrapper(JNIEnv* jni, jni->GetMethodID(encoder_factory_class, "getSupportedCodecs", "()[Lorg/webrtc/VideoCodecInfo;"); - video_codec_info_constructor_ = - jni->GetMethodID(*video_codec_info_class_, "", - "(ILjava/lang/String;Ljava/util/Map;)V"); - payload_field_ = jni->GetFieldID(*video_codec_info_class_, "payload", "I"); - name_field_ = - jni->GetFieldID(*video_codec_info_class_, "name", "Ljava/lang/String;"); - params_field_ = - jni->GetFieldID(*video_codec_info_class_, "params", "Ljava/util/Map;"); - - hash_map_constructor_ = jni->GetMethodID(*hash_map_class_, "", "()V"); - put_method_ = jni->GetMethodID( - *hash_map_class_, "put", - "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); - - supported_codecs_ = GetSupportedCodecs(jni); + supported_formats_ = GetSupportedFormats(jni); } -VideoEncoder* VideoEncoderFactoryWrapper::CreateVideoEncoder( - const cricket::VideoCodec& codec) { +std::unique_ptr VideoEncoderFactoryWrapper::CreateVideoEncoder( + const SdpVideoFormat& format) { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); - jobject j_codec_info = ToJavaCodecInfo(jni, codec); + jobject j_codec_info = SdpVideoFormatToVideoCodecInfo(jni, format); jobject encoder = jni->CallObjectMethod(*encoder_factory_, create_encoder_method_, j_codec_info); - return encoder != nullptr ? new VideoEncoderWrapper(jni, encoder) : nullptr; + return encoder != nullptr ? WrapOrUnwrapVideoEncoder(jni, encoder) : nullptr; } -jobject VideoEncoderFactoryWrapper::ToJavaCodecInfo( - JNIEnv* jni, - const cricket::VideoCodec& codec) { - jobject j_params = jni->NewObject(*hash_map_class_, hash_map_constructor_); - for (auto const& param : codec.params) { - jni->CallObjectMethod(j_params, put_method_, - JavaStringFromStdString(jni, param.first), - JavaStringFromStdString(jni, param.second)); - } - return jni->NewObject(*video_codec_info_class_, video_codec_info_constructor_, - codec.id, JavaStringFromStdString(jni, codec.name), - j_params); +VideoEncoderFactory::CodecInfo VideoEncoderFactoryWrapper::QueryVideoEncoder( + const SdpVideoFormat& format) const { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + ScopedLocalRefFrame local_ref_frame(jni); + + jobject j_codec_info = SdpVideoFormatToVideoCodecInfo(jni, format); + jobject encoder = jni->CallObjectMethod(*encoder_factory_, + create_encoder_method_, j_codec_info); + + jclass wrapped_native_encoder_class = + GetClass(jni, "org/webrtc/WrappedNativeVideoEncoder"); + + CodecInfo codec_info; + // Check if this is a wrapped native software encoder implementation. + codec_info.is_hardware_accelerated = + !jni->IsInstanceOf(encoder, wrapped_native_encoder_class); + codec_info.has_internal_source = false; + return codec_info; } -std::vector VideoEncoderFactoryWrapper::GetSupportedCodecs( +std::vector VideoEncoderFactoryWrapper::GetSupportedFormats( JNIEnv* jni) const { const jobjectArray j_supported_codecs = static_cast( jni->CallObjectMethod(*encoder_factory_, get_supported_codecs_method_)); const jsize supported_codecs_count = jni->GetArrayLength(j_supported_codecs); - std::vector supported_codecs; - supported_codecs.resize(supported_codecs_count); + std::vector supported_formats; for (jsize i = 0; i < supported_codecs_count; i++) { jobject j_supported_codec = jni->GetObjectArrayElement(j_supported_codecs, i); - int payload = jni->GetIntField(j_supported_codec, payload_field_); - jobject j_params = jni->GetObjectField(j_supported_codec, params_field_); - jstring j_name = static_cast( - jni->GetObjectField(j_supported_codec, name_field_)); - supported_codecs[i] = - cricket::VideoCodec(payload, JavaToStdString(jni, j_name)); - supported_codecs[i].params = JavaToStdMapStrings(jni, j_params); + supported_formats.push_back( + VideoCodecInfoToSdpVideoFormat(jni, j_supported_codec)); } - return supported_codecs; -} - -void VideoEncoderFactoryWrapper::DestroyVideoEncoder(VideoEncoder* encoder) { - delete encoder; + return supported_formats; } } // namespace jni diff --git a/sdk/android/src/jni/videoencoderfactorywrapper.h b/sdk/android/src/jni/videoencoderfactorywrapper.h index 509f51b7b3..26174be292 100644 --- a/sdk/android/src/jni/videoencoderfactorywrapper.h +++ b/sdk/android/src/jni/videoencoderfactorywrapper.h @@ -14,7 +14,8 @@ #include #include -#include "media/engine/webrtcvideoencoderfactory.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder_factory.h" #include "sdk/android/src/jni/jni_helpers.h" namespace webrtc { @@ -22,41 +23,29 @@ namespace jni { // Wrapper for Java VideoEncoderFactory class. Delegates method calls through // JNI and wraps the encoder inside VideoEncoderWrapper. -class VideoEncoderFactoryWrapper : public cricket::WebRtcVideoEncoderFactory { +class VideoEncoderFactoryWrapper : public VideoEncoderFactory { public: VideoEncoderFactoryWrapper(JNIEnv* jni, jobject encoder_factory); - // Caller takes the ownership of the returned object and it should be released - // by calling DestroyVideoEncoder(). - VideoEncoder* CreateVideoEncoder(const cricket::VideoCodec& codec) override; + std::unique_ptr CreateVideoEncoder( + const SdpVideoFormat& format) override; // Returns a list of supported codecs in order of preference. - const std::vector& supported_codecs() const override { - return supported_codecs_; + std::vector GetSupportedFormats() const override { + return supported_formats_; } - void DestroyVideoEncoder(VideoEncoder* encoder) override; + CodecInfo QueryVideoEncoder(const SdpVideoFormat& format) const override; private: - std::vector GetSupportedCodecs(JNIEnv* jni) const; - jobject ToJavaCodecInfo(JNIEnv* jni, const cricket::VideoCodec& codec); + std::vector GetSupportedFormats(JNIEnv* jni) const; - const ScopedGlobalRef video_codec_info_class_; - const ScopedGlobalRef hash_map_class_; const ScopedGlobalRef encoder_factory_; jmethodID create_encoder_method_; jmethodID get_supported_codecs_method_; - jmethodID video_codec_info_constructor_; - jfieldID payload_field_; - jfieldID name_field_; - jfieldID params_field_; - - jmethodID hash_map_constructor_; - jmethodID put_method_; - - std::vector supported_codecs_; + std::vector supported_formats_; }; } // namespace jni diff --git a/sdk/android/src/jni/vp8codec.cc b/sdk/android/src/jni/vp8codec.cc new file mode 100644 index 0000000000..90182a1d17 --- /dev/null +++ b/sdk/android/src/jni/vp8codec.cc @@ -0,0 +1,34 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "sdk/android/src/jni/jni_helpers.h" + +namespace webrtc { +namespace jni { + +JNI_FUNCTION_DECLARATION(jlong, + VP8Encoder_createNativeEncoder, + JNIEnv* jni, + jclass) { + return jlongFromPointer(VP8Encoder::Create()); +} + +JNI_FUNCTION_DECLARATION(jlong, + VP8Decoder_createNativeDecoder, + JNIEnv* jni, + jclass) { + return jlongFromPointer(VP8Decoder::Create()); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/vp9codec.cc b/sdk/android/src/jni/vp9codec.cc new file mode 100644 index 0000000000..6ad774e9b0 --- /dev/null +++ b/sdk/android/src/jni/vp9codec.cc @@ -0,0 +1,34 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "sdk/android/src/jni/jni_helpers.h" + +namespace webrtc { +namespace jni { + +JNI_FUNCTION_DECLARATION(jlong, + VP9Encoder_createNativeEncoder, + JNIEnv* jni, + jclass) { + return jlongFromPointer(VP9Encoder::Create()); +} + +JNI_FUNCTION_DECLARATION(jlong, + VP9Decoder_createNativeDecoder, + JNIEnv* jni, + jclass) { + return jlongFromPointer(VP9Decoder::Create()); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/wrappednativecodec.cc b/sdk/android/src/jni/wrappednativecodec.cc new file mode 100644 index 0000000000..3cdaf7c623 --- /dev/null +++ b/sdk/android/src/jni/wrappednativecodec.cc @@ -0,0 +1,58 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/android/src/jni/wrappednativecodec.h" + +#include "sdk/android/generated_video_jni/jni/WrappedNativeVideoDecoder_jni.h" +#include "sdk/android/generated_video_jni/jni/WrappedNativeVideoEncoder_jni.h" +#include "sdk/android/src/jni/class_loader.h" +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/videodecoderwrapper.h" +#include "sdk/android/src/jni/videoencoderwrapper.h" + +namespace webrtc { +namespace jni { + +std::unique_ptr WrapOrUnwrapVideoDecoder(JNIEnv* jni, + jobject j_decoder) { + jclass wrapped_native_decoder_class = + GetClass(jni, "org/webrtc/WrappedNativeVideoDecoder"); + + VideoDecoder* decoder; + if (jni->IsInstanceOf(j_decoder, wrapped_native_decoder_class)) { + jlong native_decoder = + Java_WrappedNativeVideoDecoder_getNativeDecoder(jni, j_decoder); + decoder = reinterpret_cast(native_decoder); + } else { + decoder = new VideoDecoderWrapper(jni, j_decoder); + } + + return std::unique_ptr(decoder); +} + +std::unique_ptr WrapOrUnwrapVideoEncoder(JNIEnv* jni, + jobject j_encoder) { + jclass wrapped_native_encoder_class = + GetClass(jni, "org/webrtc/WrappedNativeVideoEncoder"); + + VideoEncoder* encoder; + if (jni->IsInstanceOf(j_encoder, wrapped_native_encoder_class)) { + jlong native_encoder = + Java_WrappedNativeVideoEncoder_getNativeEncoder(jni, j_encoder); + encoder = reinterpret_cast(native_encoder); + } else { + encoder = new VideoEncoderWrapper(jni, j_encoder); + } + + return std::unique_ptr(encoder); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/wrappednativecodec.h b/sdk/android/src/jni/wrappednativecodec.h new file mode 100644 index 0000000000..222610a403 --- /dev/null +++ b/sdk/android/src/jni/wrappednativecodec.h @@ -0,0 +1,38 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_ +#define SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_ + +#include +#include + +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" + +namespace webrtc { +namespace jni { + +/* If the j_decoder is a wrapped native decoder, unwrap it. If it is not, + * wrap it in a VideoDecoderWrapper. + */ +std::unique_ptr WrapOrUnwrapVideoDecoder(JNIEnv* jni, + jobject j_decoder); + +/* If the j_encoder is a wrapped native encoder, unwrap it. If it is not, + * wrap it in a VideoEncoderWrapper. + */ +std::unique_ptr WrapOrUnwrapVideoEncoder(JNIEnv* jni, + jobject j_encoder); + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_