From 389d2261c3dccbb44cbad472f2689106aa6ad493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sami=20Kalliom=C3=A4ki?= Date: Wed, 5 Sep 2018 16:38:57 +0200 Subject: [PATCH] Add support for platform software video decoder implementations. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also enables support for all hardware implementations. Renames HardwareVideoDecoderFactory to MediaCodecVideoDecoderFactory. Renames HardwareVideoDecoder to AndroidVideoDecoder. Bug: webrtc:8538 Change-Id: I9b351f387526af4da61fb07c07fb4285bd833e19 Reviewed-on: https://webrtc-review.googlesource.com/97680 Reviewed-by: Mirko Bonadei Reviewed-by: Magnus Jedvert Commit-Queue: Sami Kalliomäki Cr-Commit-Position: refs/heads/master@{#24586} --- BUILD.gn | 2 +- sdk/android/BUILD.gn | 12 +- .../webrtc/DefaultVideoDecoderFactory.java | 26 ++- .../webrtc/HardwareVideoDecoderFactory.java | 139 +-------------- .../PlatformSoftwareVideoDecoderFactory.java | 27 +++ ...droidVideoDecoderInstrumentationTest.java} | 8 +- .../org/webrtc/HardwareVideoEncoderTest.java | 2 +- ...oDecoder.java => AndroidVideoDecoder.java} | 29 ++-- .../src/java/org/webrtc/MediaCodecUtils.java | 1 + .../webrtc/MediaCodecVideoDecoderFactory.java | 159 ++++++++++++++++++ .../java/org/webrtc/MediaCodecWrapper.java | 2 +- ...Test.java => AndroidVideoDecoderTest.java} | 31 +--- .../tests/src/org/webrtc/CodecTestHelper.java | 4 +- 13 files changed, 258 insertions(+), 184 deletions(-) create mode 100644 sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java rename sdk/android/instrumentationtests/src/org/webrtc/{HardwareVideoDecoderTest.java => AndroidVideoDecoderInstrumentationTest.java} (96%) rename sdk/android/src/java/org/webrtc/{HardwareVideoDecoder.java => AndroidVideoDecoder.java} (96%) create mode 100644 sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java rename sdk/android/tests/src/org/webrtc/{HardwareVideoDecoderTest.java => AndroidVideoDecoderTest.java} (93%) diff --git a/BUILD.gn b/BUILD.gn index 7c8a5772c3..0fd2a2ae59 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -576,12 +576,12 @@ if (rtc_include_tests) { "examples/androidjunit/src/org/appspot/apprtc/BluetoothManagerTest.java", "examples/androidjunit/src/org/appspot/apprtc/DirectRTCClientTest.java", "examples/androidjunit/src/org/appspot/apprtc/TCPChannelClientTest.java", + "sdk/android/tests/src/org/webrtc/AndroidVideoDecoderTest.java", "sdk/android/tests/src/org/webrtc/CameraEnumerationTest.java", "sdk/android/tests/src/org/webrtc/CodecTestHelper.java", "sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java", "sdk/android/tests/src/org/webrtc/GlGenericDrawerTest.java", "sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java", - "sdk/android/tests/src/org/webrtc/HardwareVideoDecoderTest.java", "sdk/android/tests/src/org/webrtc/ScalingSettingsTest.java", ] diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 3b735e0be4..ac94f97930 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -385,16 +385,18 @@ if (is_android) { java_files = [ "api/org/webrtc/HardwareVideoDecoderFactory.java", "api/org/webrtc/HardwareVideoEncoderFactory.java", + "api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java", + "src/java/org/webrtc/AndroidVideoDecoder.java", "src/java/org/webrtc/BaseBitrateAdjuster.java", "src/java/org/webrtc/BitrateAdjuster.java", "src/java/org/webrtc/DynamicBitrateAdjuster.java", "src/java/org/webrtc/FramerateBitrateAdjuster.java", - "src/java/org/webrtc/HardwareVideoDecoder.java", "src/java/org/webrtc/HardwareVideoEncoder.java", + "src/java/org/webrtc/MediaCodecUtils.java", + "src/java/org/webrtc/MediaCodecVideoDecoderFactory.java", "src/java/org/webrtc/MediaCodecWrapper.java", "src/java/org/webrtc/MediaCodecWrapperFactory.java", "src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java", - "src/java/org/webrtc/MediaCodecUtils.java", "src/java/org/webrtc/NV12Buffer.java", "src/java/org/webrtc/VideoCodecType.java", ] @@ -1253,17 +1255,15 @@ if (is_android) { android_manifest = "instrumentationtests/AndroidManifest.xml" java_files = [ + "instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java", "instrumentationtests/src/org/webrtc/Camera1CapturerUsingByteBufferTest.java", "instrumentationtests/src/org/webrtc/Camera1CapturerUsingTextureTest.java", "instrumentationtests/src/org/webrtc/Camera2CapturerTest.java", "instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java", - "instrumentationtests/src/org/webrtc/TestConstants.java", "instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java", "instrumentationtests/src/org/webrtc/EglRendererTest.java", "instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java", "instrumentationtests/src/org/webrtc/GlRectDrawerTest.java", - "instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java", - "instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java", "instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java", "instrumentationtests/src/org/webrtc/LoggableTest.java", "instrumentationtests/src/org/webrtc/MediaCodecVideoEncoderTest.java", @@ -1273,7 +1273,9 @@ if (is_android) { "instrumentationtests/src/org/webrtc/RendererCommonTest.java", "instrumentationtests/src/org/webrtc/SurfaceTextureHelperTest.java", "instrumentationtests/src/org/webrtc/SurfaceViewRendererOnMeasureTest.java", + "instrumentationtests/src/org/webrtc/TestConstants.java", "instrumentationtests/src/org/webrtc/VideoFileRendererTest.java", + "instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java", "instrumentationtests/src/org/webrtc/WebRtcJniBootTest.java", "instrumentationtests/src/org/webrtc/YuvHelperTest.java", ] diff --git a/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java b/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java index 4e5c082fd8..4561a5ec23 100644 --- a/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java +++ b/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java @@ -15,25 +15,37 @@ import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; -/** Helper class that combines HW and SW decoders. */ +/** + * Helper class that combines HW and SW decoders. + */ public class DefaultVideoDecoderFactory implements VideoDecoderFactory { private final VideoDecoderFactory hardwareVideoDecoderFactory; private final VideoDecoderFactory softwareVideoDecoderFactory = new SoftwareVideoDecoderFactory(); + private final @Nullable VideoDecoderFactory platformSoftwareVideoDecoderFactory; - /** Create decoder factory using default hardware decoder factory. */ - public DefaultVideoDecoderFactory(EglBase.Context eglContext) { + /** + * Create decoder factory using default hardware decoder factory. + */ + public DefaultVideoDecoderFactory(@Nullable EglBase.Context eglContext) { this.hardwareVideoDecoderFactory = new HardwareVideoDecoderFactory(eglContext); + this.platformSoftwareVideoDecoderFactory = new PlatformSoftwareVideoDecoderFactory(eglContext); } - /** Create decoder factory using explicit hardware decoder factory. */ + /** + * Create decoder factory using explicit hardware decoder factory. + */ DefaultVideoDecoderFactory(VideoDecoderFactory hardwareVideoDecoderFactory) { this.hardwareVideoDecoderFactory = hardwareVideoDecoderFactory; + this.platformSoftwareVideoDecoderFactory = null; } @Override public @Nullable VideoDecoder createDecoder(VideoCodecInfo codecType) { - final VideoDecoder softwareDecoder = softwareVideoDecoderFactory.createDecoder(codecType); + VideoDecoder softwareDecoder = softwareVideoDecoderFactory.createDecoder(codecType); final VideoDecoder hardwareDecoder = hardwareVideoDecoderFactory.createDecoder(codecType); + if (softwareDecoder == null && platformSoftwareVideoDecoderFactory != null) { + softwareDecoder = platformSoftwareVideoDecoderFactory.createDecoder(codecType); + } if (hardwareDecoder != null && softwareDecoder != null) { // Both hardware and software supported, wrap it in a software fallback return new VideoDecoderFallback( @@ -48,6 +60,10 @@ public class DefaultVideoDecoderFactory implements VideoDecoderFactory { supportedCodecInfos.addAll(Arrays.asList(softwareVideoDecoderFactory.getSupportedCodecs())); supportedCodecInfos.addAll(Arrays.asList(hardwareVideoDecoderFactory.getSupportedCodecs())); + if (platformSoftwareVideoDecoderFactory != null) { + supportedCodecInfos.addAll( + Arrays.asList(platformSoftwareVideoDecoderFactory.getSupportedCodecs())); + } return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); } diff --git a/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java index 3e885830e4..80252e642f 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java @@ -10,26 +10,10 @@ package org.webrtc; -import static org.webrtc.MediaCodecUtils.EXYNOS_PREFIX; -import static org.webrtc.MediaCodecUtils.INTEL_PREFIX; -import static org.webrtc.MediaCodecUtils.NVIDIA_PREFIX; -import static org.webrtc.MediaCodecUtils.QCOM_PREFIX; - -import android.media.MediaCodecInfo; -import android.media.MediaCodecInfo.CodecCapabilities; -import android.media.MediaCodecList; -import android.os.Build; -import java.util.ArrayList; -import java.util.List; import javax.annotation.Nullable; /** Factory for Android hardware VideoDecoders. */ -@SuppressWarnings("deprecation") // API level 16 requires use of deprecated methods. -public class HardwareVideoDecoderFactory implements VideoDecoderFactory { - private static final String TAG = "HardwareVideoDecoderFactory"; - - private final EglBase.Context sharedContext; - +public class HardwareVideoDecoderFactory extends MediaCodecVideoDecoderFactory { /** Creates a HardwareVideoDecoderFactory that does not use surface textures. */ @Deprecated // Not removed yet to avoid breaking callers. public HardwareVideoDecoderFactory() { @@ -37,120 +21,13 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory { } /** - * Creates a HardwareVideoDecoderFactory that supports surface texture rendering using the given - * shared context. The context may be null. If it is null, then surface support is disabled. + * Creates a HardwareVideoDecoderFactory that supports surface texture rendering. + * + * @param sharedContext The textures generated will be accessible from this context. May be null, + * this disables texture support. */ - public HardwareVideoDecoderFactory(EglBase.Context sharedContext) { - this.sharedContext = sharedContext; - } - - @Nullable - @Override - public VideoDecoder createDecoder(VideoCodecInfo codecType) { - VideoCodecType type = VideoCodecType.valueOf(codecType.getName()); - MediaCodecInfo info = findCodecForType(type); - - if (info == null) { - return null; - } - - CodecCapabilities capabilities = info.getCapabilitiesForType(type.mimeType()); - return new HardwareVideoDecoder(new MediaCodecWrapperFactoryImpl(), info.getName(), type, - MediaCodecUtils.selectColorFormat(MediaCodecUtils.DECODER_COLOR_FORMATS, capabilities), - sharedContext); - } - - @Override - public VideoCodecInfo[] getSupportedCodecs() { - List supportedCodecInfos = new ArrayList(); - // Generate a list of supported codecs in order of preference: - // VP8, VP9, H264 (high profile), and H264 (baseline profile). - for (VideoCodecType type : - new VideoCodecType[] {VideoCodecType.VP8, VideoCodecType.VP9, VideoCodecType.H264}) { - MediaCodecInfo codec = findCodecForType(type); - if (codec != null) { - String name = type.name(); - if (type == VideoCodecType.H264 && isH264HighProfileSupported(codec)) { - supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); - } - - supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); - } - } - - return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); - } - - private @Nullable MediaCodecInfo findCodecForType(VideoCodecType type) { - // HW decoding is not supported on builds before KITKAT. - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { - return null; - } - - for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { - MediaCodecInfo info = null; - try { - info = MediaCodecList.getCodecInfoAt(i); - } catch (IllegalArgumentException e) { - Logging.e(TAG, "Cannot retrieve encoder codec info", e); - } - - if (info == null || info.isEncoder()) { - continue; - } - - if (isSupportedCodec(info, type)) { - return info; - } - } - return null; // No support for this type. - } - - // Returns true if the given MediaCodecInfo indicates a supported encoder for the given type. - private boolean isSupportedCodec(MediaCodecInfo info, VideoCodecType type) { - if (!MediaCodecUtils.codecSupportsType(info, type)) { - return false; - } - // Check for a supported color format. - if (MediaCodecUtils.selectColorFormat( - MediaCodecUtils.DECODER_COLOR_FORMATS, info.getCapabilitiesForType(type.mimeType())) - == null) { - return false; - } - return isHardwareSupported(info, type); - } - - private boolean isHardwareSupported(MediaCodecInfo info, VideoCodecType type) { - String name = info.getName(); - switch (type) { - case VP8: - // QCOM, Intel, Exynos, and Nvidia all supported for VP8. - return name.startsWith(QCOM_PREFIX) || name.startsWith(INTEL_PREFIX) - || name.startsWith(EXYNOS_PREFIX) || name.startsWith(NVIDIA_PREFIX); - case VP9: - // QCOM and Exynos supported for VP9. - return name.startsWith(QCOM_PREFIX) || name.startsWith(EXYNOS_PREFIX); - case H264: - // QCOM, Intel, and Exynos supported for H264. - return name.startsWith(QCOM_PREFIX) || name.startsWith(INTEL_PREFIX) - || name.startsWith(EXYNOS_PREFIX); - default: - return false; - } - } - - private boolean isH264HighProfileSupported(MediaCodecInfo info) { - String name = info.getName(); - // Support H.264 HP decoding on QCOM chips for Android L and above. - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP && name.startsWith(QCOM_PREFIX)) { - return true; - } - // Support H.264 HP decoding on Exynos chips for Android M and above. - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && name.startsWith(EXYNOS_PREFIX)) { - return true; - } - return false; + public HardwareVideoDecoderFactory(@Nullable EglBase.Context sharedContext) { + super(sharedContext, /* prefixWhitelist= */ new String[] {""}, + /* prefixBlacklist= */ MediaCodecUtils.SOFTWARE_IMPLEMENTATION_PREFIXES); } } diff --git a/sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java b/sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java new file mode 100644 index 0000000000..856db687ba --- /dev/null +++ b/sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import javax.annotation.Nullable; + +/** Factory for Android platform software VideoDecoders. */ +public class PlatformSoftwareVideoDecoderFactory extends MediaCodecVideoDecoderFactory { + /** + * Creates a PlatformSoftwareVideoDecoderFactory that supports surface texture rendering. + * + * @param sharedContext The textures generated will be accessible from this context. May be null, + * this disables texture support. + */ + public PlatformSoftwareVideoDecoderFactory(@Nullable EglBase.Context sharedContext) { + super(sharedContext, /* prefixWhitelist= */ MediaCodecUtils.SOFTWARE_IMPLEMENTATION_PREFIXES, + /* prefixBlacklist= */ new String[] {}); + } +} diff --git a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java similarity index 96% rename from sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java rename to sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java index 64a51f50c4..62e6022824 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java @@ -32,10 +32,10 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -/** Unit tests for {@link HardwareVideoDecoder}. */ +/** Unit tests for {@link AndroidVideoDecoder}. */ @RunWith(ParameterizedRunner.class) @UseRunnerDelegate(BaseJUnit4RunnerDelegate.class) -public final class HardwareVideoDecoderTest { +public final class AndroidVideoDecoderInstrumentationTest { @ClassParameter private static List CLASS_PARAMS = new ArrayList<>(); static { @@ -56,7 +56,7 @@ public final class HardwareVideoDecoderTest { private final VideoCodecInfo codecType; private final boolean useEglContext; - public HardwareVideoDecoderTest(String codecName, boolean useEglContext) { + public AndroidVideoDecoderInstrumentationTest(String codecName, boolean useEglContext) { if (codecName.equals("H264")) { this.codecType = H264Utils.DEFAULT_H264_BASELINE_PROFILE_CODEC; } else { @@ -65,7 +65,7 @@ public final class HardwareVideoDecoderTest { this.useEglContext = useEglContext; } - private static final String TAG = "HardwareVideoDecoderTest"; + private static final String TAG = "AndroidVideoDecoderInstrumentationTest"; private static final int TEST_FRAME_COUNT = 10; private static final int TEST_FRAME_WIDTH = 640; diff --git a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java b/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java index 527ab5ce94..fb27506590 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java @@ -79,7 +79,7 @@ public class HardwareVideoEncoderTest { // # Mock classes /** * Mock encoder callback that allows easy verification of the general properties of the encoded - * frame such as width and height. Also used from HardwareVideoDecoderTest. + * frame such as width and height. Also used from AndroidVideoDecoderInstrumentationTest. */ static class MockEncoderCallback implements VideoEncoder.Callback { private BlockingQueue frameQueue = new LinkedBlockingQueue<>(); diff --git a/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java b/sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java similarity index 96% rename from sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java rename to sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java index 6cc5379e25..a75682834a 100644 --- a/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java +++ b/sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java @@ -24,11 +24,15 @@ import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.webrtc.ThreadUtils.ThreadChecker; -/** Android hardware video decoder. */ +/** + * Android hardware video decoder. + */ @TargetApi(16) -@SuppressWarnings("deprecation") // Cannot support API 16 without using deprecated methods. -class HardwareVideoDecoder implements VideoDecoder, VideoSink { - private static final String TAG = "HardwareVideoDecoder"; +@SuppressWarnings("deprecation") +// Cannot support API 16 without using deprecated methods. +// TODO(sakal): Rename to MediaCodecVideoDecoder once the deprecated implementation is removed. +class AndroidVideoDecoder implements VideoDecoder, VideoSink { + private static final String TAG = "AndroidVideoDecoder"; // TODO(magjed): Use MediaFormat.KEY_* constants when part of the public API. private static final String MEDIA_FORMAT_KEY_STRIDE = "stride"; @@ -100,7 +104,7 @@ class HardwareVideoDecoder implements VideoDecoder, VideoSink { // on the decoder thread. private boolean keyFrameRequired; - private final EglBase.Context sharedContext; + private final @Nullable EglBase.Context sharedContext; // Valid and immutable while the decoder is running. @Nullable private SurfaceTextureHelper surfaceTextureHelper; @Nullable private Surface surface = null; @@ -126,11 +130,14 @@ class HardwareVideoDecoder implements VideoDecoder, VideoSink { // Valid and immutable while the decoder is running. @Nullable private MediaCodecWrapper codec = null; - HardwareVideoDecoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, - VideoCodecType codecType, int colorFormat, EglBase.Context sharedContext) { + AndroidVideoDecoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, + VideoCodecType codecType, int colorFormat, @Nullable EglBase.Context sharedContext) { if (!isSupportedColorFormat(colorFormat)) { throw new IllegalArgumentException("Unsupported color format: " + colorFormat); } + Logging.d(TAG, + "ctor name: " + codecName + " type: " + codecType + " color format: " + colorFormat + + " context: " + sharedContext); this.mediaCodecWrapperFactory = mediaCodecWrapperFactory; this.codecName = codecName; this.codecType = codecType; @@ -155,7 +162,9 @@ class HardwareVideoDecoder implements VideoDecoder, VideoSink { // Internal variant is used when restarting the codec due to reconfiguration. private VideoCodecStatus initDecodeInternal(int width, int height) { decoderThreadChecker.checkIsOnValidThread(); - Logging.d(TAG, "initDecodeInternal"); + Logging.d(TAG, + "initDecodeInternal name: " + codecName + " type: " + codecType + " width: " + width + + " height: " + height); if (outputThread != null) { Logging.e(TAG, "initDecodeInternal called while the codec is already running"); return VideoCodecStatus.FALLBACK_SOFTWARE; @@ -295,7 +304,7 @@ class HardwareVideoDecoder implements VideoDecoder, VideoSink { @Override public String getImplementationName() { - return "HWDecoder"; + return codecName; } @Override @@ -358,7 +367,7 @@ class HardwareVideoDecoder implements VideoDecoder, VideoSink { } private Thread createOutputThread() { - return new Thread("HardwareVideoDecoder.outputThread") { + return new Thread("AndroidVideoDecoder.outputThread") { @Override public void run() { outputThreadChecker = new ThreadChecker(); diff --git a/sdk/android/src/java/org/webrtc/MediaCodecUtils.java b/sdk/android/src/java/org/webrtc/MediaCodecUtils.java index 6221a104dc..8eb1b20ac1 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecUtils.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecUtils.java @@ -30,6 +30,7 @@ class MediaCodecUtils { static final String INTEL_PREFIX = "OMX.Intel."; static final String NVIDIA_PREFIX = "OMX.Nvidia."; static final String QCOM_PREFIX = "OMX.qcom."; + static final String[] SOFTWARE_IMPLEMENTATION_PREFIXES = {"OMX.google.", "OMX.SEC."}; // NV12 color format supported by QCOM codec, but not declared in MediaCodec - // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h diff --git a/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java new file mode 100644 index 0000000000..aa272930f4 --- /dev/null +++ b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java @@ -0,0 +1,159 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import static org.webrtc.MediaCodecUtils.EXYNOS_PREFIX; +import static org.webrtc.MediaCodecUtils.QCOM_PREFIX; + +import android.media.MediaCodecInfo; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.media.MediaCodecList; +import android.os.Build; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nullable; + +/** Factory for decoders backed by Android MediaCodec API. */ +@SuppressWarnings("deprecation") // API level 16 requires use of deprecated methods. +class MediaCodecVideoDecoderFactory implements VideoDecoderFactory { + private static final String TAG = "MediaCodecVideoDecoderFactory"; + + private final @Nullable EglBase.Context sharedContext; + private final String[] prefixWhitelist; + private final String[] prefixBlacklist; + + /** + * MediaCodecVideoDecoderFactory will support codecs whitelisted excluding those blacklisted. + * + * @param sharedContext The textures generated will be accessible from this context. May be null, + * this disables texture support. + * @param prefixWhitelist List of codec prefixes to be whitelisted. + * @param prefixBlacklist List of codec prefixes to be blacklisted. + */ + public MediaCodecVideoDecoderFactory( + @Nullable EglBase.Context sharedContext, String[] prefixWhitelist, String[] prefixBlacklist) { + this.sharedContext = sharedContext; + this.prefixWhitelist = Arrays.copyOf(prefixWhitelist, prefixWhitelist.length); + this.prefixBlacklist = Arrays.copyOf(prefixBlacklist, prefixBlacklist.length); + } + + @Nullable + @Override + public VideoDecoder createDecoder(VideoCodecInfo codecType) { + VideoCodecType type = VideoCodecType.valueOf(codecType.getName()); + MediaCodecInfo info = findCodecForType(type); + + if (info == null) { + return null; + } + + CodecCapabilities capabilities = info.getCapabilitiesForType(type.mimeType()); + return new AndroidVideoDecoder(new MediaCodecWrapperFactoryImpl(), info.getName(), type, + MediaCodecUtils.selectColorFormat(MediaCodecUtils.DECODER_COLOR_FORMATS, capabilities), + sharedContext); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + List supportedCodecInfos = new ArrayList(); + // Generate a list of supported codecs in order of preference: + // VP8, VP9, H264 (high profile), and H264 (baseline profile). + for (VideoCodecType type : + new VideoCodecType[] {VideoCodecType.VP8, VideoCodecType.VP9, VideoCodecType.H264}) { + MediaCodecInfo codec = findCodecForType(type); + if (codec != null) { + String name = type.name(); + if (type == VideoCodecType.H264 && isH264HighProfileSupported(codec)) { + supportedCodecInfos.add(new VideoCodecInfo( + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); + } + + supportedCodecInfos.add(new VideoCodecInfo( + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); + } + } + + return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); + } + + private @Nullable MediaCodecInfo findCodecForType(VideoCodecType type) { + // HW decoding is not supported on builds before KITKAT. + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { + return null; + } + + for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { + MediaCodecInfo info = null; + try { + info = MediaCodecList.getCodecInfoAt(i); + } catch (IllegalArgumentException e) { + Logging.e(TAG, "Cannot retrieve decoder codec info", e); + } + + if (info == null || info.isEncoder()) { + continue; + } + + if (isSupportedCodec(info, type)) { + return info; + } + } + + return null; // No support for this type. + } + + // Returns true if the given MediaCodecInfo indicates a supported encoder for the given type. + private boolean isSupportedCodec(MediaCodecInfo info, VideoCodecType type) { + String name = info.getName(); + if (!MediaCodecUtils.codecSupportsType(info, type)) { + return false; + } + // Check for a supported color format. + if (MediaCodecUtils.selectColorFormat( + MediaCodecUtils.DECODER_COLOR_FORMATS, info.getCapabilitiesForType(type.mimeType())) + == null) { + return false; + } + return isWhitelisted(name) && !isBlacklisted(name); + } + + private boolean isWhitelisted(String name) { + for (String prefix : prefixWhitelist) { + if (name.startsWith(prefix)) { + return true; + } + } + return false; + } + + private boolean isBlacklisted(String name) { + for (String prefix : prefixBlacklist) { + if (name.startsWith(prefix)) { + return true; + } + } + return false; + } + + private boolean isH264HighProfileSupported(MediaCodecInfo info) { + String name = info.getName(); + // Support H.264 HP decoding on QCOM chips for Android L and above. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP && name.startsWith(QCOM_PREFIX)) { + return true; + } + // Support H.264 HP decoding on Exynos chips for Android M and above. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && name.startsWith(EXYNOS_PREFIX)) { + return true; + } + return false; + } +} diff --git a/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java b/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java index 7e376b8e2d..bb67d1f4b9 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java @@ -19,7 +19,7 @@ import java.nio.ByteBuffer; /** * Subset of methods defined in {@link android.media.MediaCodec} needed by - * {@link HardwareVideoEncoder} and {@link HardwareVideoDecoder}. This interface + * {@link HardwareVideoEncoder} and {@link AndroidVideoDecoder}. This interface * exists to allow mocking and using a fake implementation in tests. */ interface MediaCodecWrapper { diff --git a/sdk/android/tests/src/org/webrtc/HardwareVideoDecoderTest.java b/sdk/android/tests/src/org/webrtc/AndroidVideoDecoderTest.java similarity index 93% rename from sdk/android/tests/src/org/webrtc/HardwareVideoDecoderTest.java rename to sdk/android/tests/src/org/webrtc/AndroidVideoDecoderTest.java index f47d06902a..8f96214ad1 100644 --- a/sdk/android/tests/src/org/webrtc/HardwareVideoDecoderTest.java +++ b/sdk/android/tests/src/org/webrtc/AndroidVideoDecoderTest.java @@ -23,16 +23,12 @@ import static org.mockito.Mockito.verify; import static org.robolectric.Shadows.shadowOf; import android.graphics.Matrix; -import android.media.MediaCodec.BufferInfo; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaFormat; import android.os.Handler; import java.nio.ByteBuffer; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; import org.chromium.testing.local.LocalRobolectricTestRunner; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -40,25 +36,15 @@ import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.robolectric.annotation.Config; -import org.robolectric.shadows.ShadowSystemClock; -import org.webrtc.EglBase; -import org.webrtc.EncodedImage; import org.webrtc.EncodedImage.FrameType; import org.webrtc.FakeMediaCodecWrapper.State; -import org.webrtc.SurfaceTextureHelper; -import org.webrtc.TextureBufferImpl; -import org.webrtc.VideoCodecStatus; -import org.webrtc.VideoDecoder; import org.webrtc.VideoDecoder.DecodeInfo; -import org.webrtc.VideoFrame; import org.webrtc.VideoFrame.I420Buffer; import org.webrtc.VideoFrame.TextureBuffer.Type; -import org.webrtc.VideoSink; -import org.webrtc.YuvConverter; @RunWith(LocalRobolectricTestRunner.class) @Config(manifest = Config.NONE) -public class HardwareVideoDecoderTest { +public class AndroidVideoDecoderTest { private static final VideoDecoder.Settings TEST_DECODER_SETTINGS = new VideoDecoder.Settings(/* numberOfCores= */ 1, /* width= */ 640, /* height= */ 480); private static final int COLOR_FORMAT = CodecCapabilities.COLOR_FormatYUV420Planar; @@ -67,7 +53,7 @@ public class HardwareVideoDecoderTest { private static final byte[] ENCODED_TEST_DATA = new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - private class TestDecoder extends HardwareVideoDecoder { + private class TestDecoder extends AndroidVideoDecoder { private final Object deliverDecodedFrameLock = new Object(); private boolean deliverDecodedFrameDone = true; @@ -198,8 +184,7 @@ public class HardwareVideoDecoderTest { @Test public void testInit() { // Set-up. - HardwareVideoDecoder decoder = - new TestDecoderBuilder().setCodecType(VideoCodecType.VP8).build(); + AndroidVideoDecoder decoder = new TestDecoderBuilder().setCodecType(VideoCodecType.VP8).build(); // Test. assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback)) @@ -221,7 +206,7 @@ public class HardwareVideoDecoderTest { @Test public void testRelease() { // Set-up. - HardwareVideoDecoder decoder = new TestDecoderBuilder().build(); + AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); // Test. @@ -234,7 +219,7 @@ public class HardwareVideoDecoderTest { @Test public void testReleaseMultipleTimes() { // Set-up. - HardwareVideoDecoder decoder = new TestDecoderBuilder().build(); + AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); // Test. @@ -248,7 +233,7 @@ public class HardwareVideoDecoderTest { @Test public void testDecodeQueuesData() { // Set-up. - HardwareVideoDecoder decoder = new TestDecoderBuilder().build(); + AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); // Test. @@ -401,7 +386,7 @@ public class HardwareVideoDecoderTest { .when(fakeMediaCodecWrapper) .configure(any(), any(), any(), anyInt()); - HardwareVideoDecoder decoder = new TestDecoderBuilder().build(); + AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); // Test. assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback)) @@ -413,7 +398,7 @@ public class HardwareVideoDecoderTest { // Set-up. doThrow(new IllegalStateException("Fake error")).when(fakeMediaCodecWrapper).start(); - HardwareVideoDecoder decoder = new TestDecoderBuilder().build(); + AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); // Test. assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback)) diff --git a/sdk/android/tests/src/org/webrtc/CodecTestHelper.java b/sdk/android/tests/src/org/webrtc/CodecTestHelper.java index 8067408c5c..08a10707f8 100644 --- a/sdk/android/tests/src/org/webrtc/CodecTestHelper.java +++ b/sdk/android/tests/src/org/webrtc/CodecTestHelper.java @@ -15,11 +15,9 @@ import static com.google.common.truth.Truth.assertWithMessage; import java.nio.ByteBuffer; import java.util.Random; -import org.webrtc.JavaI420Buffer; -import org.webrtc.VideoFrame; /** - * Helper methods for {@link HardwareVideoEncoderTest} and {@link HardwareVideoDecoderTest}. + * Helper methods for {@link HardwareVideoEncoderTest} and {@link AndroidVideoDecoderTest}. */ class CodecTestHelper { static void assertEqualContents(byte[] expected, ByteBuffer actual, int offset, int size) {