From cc02cb595f313cfb1e592c7d43a93fe065122ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sami=20Kalliom=C3=A4ki?= Date: Mon, 21 May 2018 13:45:51 +0200 Subject: [PATCH] Add getSupportedCodecs to VideoDecoderFactory interface. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default implementation of the method is to return an empty list. Clients should update their implementations before WebRTC starts calling this method. Also updates internal WebRTC implentations of this interface to implement the method. Bug: webrtc:7925 Change-Id: I258de2f09f6d4cc5dd9f4657e5d54e8411f8f5d8 Reviewed-on: https://webrtc-review.googlesource.com/77641 Reviewed-by: Anders Carlsson Commit-Queue: Sami Kalliomäki Cr-Commit-Position: refs/heads/master@{#23325} --- .../webrtc/DefaultVideoDecoderFactory.java | 13 +++++ .../webrtc/HardwareVideoDecoderFactory.java | 48 +++++++++++++++++++ .../webrtc/HardwareVideoEncoderFactory.java | 34 ++++--------- .../webrtc/SoftwareVideoDecoderFactory.java | 19 ++++++++ .../api/org/webrtc/VideoDecoderFactory.java | 8 ++++ .../src/java/org/webrtc/MediaCodecUtils.java | 20 ++++++++ .../src/jni/videodecoderfactorywrapper.cc | 7 ++- 7 files changed, 123 insertions(+), 26 deletions(-) diff --git a/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java b/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java index b9a26d3b50..55b29a4288 100644 --- a/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java +++ b/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java @@ -10,6 +10,9 @@ package org.webrtc; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; import javax.annotation.Nullable; public class DefaultVideoDecoderFactory implements VideoDecoderFactory { @@ -30,4 +33,14 @@ public class DefaultVideoDecoderFactory implements VideoDecoderFactory { } return softwareVideoDecoderFactory.createDecoder(codecType); } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + LinkedHashSet supportedCodecInfos = new LinkedHashSet(); + + supportedCodecInfos.addAll(Arrays.asList(softwareVideoDecoderFactory.getSupportedCodecs())); + supportedCodecInfos.addAll(Arrays.asList(hardwareVideoDecoderFactory.getSupportedCodecs())); + + return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); + } } diff --git a/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java index 2163a04e78..b40bd05c8c 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java @@ -19,6 +19,8 @@ import android.media.MediaCodecInfo; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecList; import android.os.Build; +import java.util.ArrayList; +import java.util.List; import javax.annotation.Nullable; /** Factory for Android hardware VideoDecoders. */ @@ -72,6 +74,39 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory { sharedContext); } + @Override + public VideoCodecInfo[] getSupportedCodecs() { + List supportedCodecInfos = new ArrayList(); + // Generate a list of supported codecs in order of preference: + // VP8, VP9, H264 (high profile), and H264 (baseline profile). + for (VideoCodecType type : + new VideoCodecType[] {VideoCodecType.VP8, VideoCodecType.VP9, VideoCodecType.H264}) { + MediaCodecInfo codec = findCodecForType(type); + if (codec != null) { + String name = type.name(); + if (type == VideoCodecType.H264 && isH264HighProfileSupported(codec)) { + supportedCodecInfos.add(new VideoCodecInfo( + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); + } + + supportedCodecInfos.add(new VideoCodecInfo( + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); + } + } + + // TODO(andersc): This is for backwards compatibility. Remove when clients have migrated to + // new DefaultVideoEncoderFactory. + if (fallbackToSoftware) { + for (VideoCodecInfo info : SoftwareVideoDecoderFactory.supportedCodecs()) { + if (!supportedCodecInfos.contains(info)) { + supportedCodecInfos.add(info); + } + } + } + + return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); + } + private @Nullable MediaCodecInfo findCodecForType(VideoCodecType type) { // HW decoding is not supported on builds before KITKAT. if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { @@ -129,4 +164,17 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory { return false; } } + + private boolean isH264HighProfileSupported(MediaCodecInfo info) { + String name = info.getName(); + // Support H.264 HP decoding on QCOM chips for Android L and above. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP && name.startsWith(QCOM_PREFIX)) { + return true; + } + // Support H.264 HP decoding on Exynos chips for Android M and above. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && name.startsWith(EXYNOS_PREFIX)) { + return true; + } + return false; + } } diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java index 19c641b89a..9c16298da1 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java @@ -19,7 +19,6 @@ import android.media.MediaCodecList; import android.os.Build; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -96,10 +95,11 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory { MediaCodecUtils.ENCODER_COLOR_FORMATS, info.getCapabilitiesForType(mime)); if (type == VideoCodecType.H264) { - boolean isHighProfile = nativeIsSameH264Profile(input.params, getCodecProperties(type, true)) + boolean isHighProfile = nativeIsSameH264Profile(input.params, + MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true)) && isH264HighProfileSupported(info); - boolean isBaselineProfile = - nativeIsSameH264Profile(input.params, getCodecProperties(type, false)); + boolean isBaselineProfile = nativeIsSameH264Profile( + input.params, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false)); if (!isHighProfile && !isBaselineProfile) { return null; @@ -121,11 +121,15 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory { MediaCodecInfo codec = findCodecForType(type); if (codec != null) { String name = type.name(); + // TODO(sakal): Always add H264 HP once WebRTC correctly removes codecs that are not + // supported by the decoder. if (type == VideoCodecType.H264 && isH264HighProfileSupported(codec)) { - supportedCodecInfos.add(new VideoCodecInfo(name, getCodecProperties(type, true))); + supportedCodecInfos.add(new VideoCodecInfo( + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); } - supportedCodecInfos.add(new VideoCodecInfo(name, getCodecProperties(type, false))); + supportedCodecInfos.add(new VideoCodecInfo( + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); } } @@ -266,24 +270,6 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory { && info.getName().startsWith(EXYNOS_PREFIX); } - private Map getCodecProperties(VideoCodecType type, boolean highProfile) { - switch (type) { - case VP8: - case VP9: - return new HashMap(); - case H264: - Map properties = new HashMap<>(); - properties.put(VideoCodecInfo.H264_FMTP_LEVEL_ASYMMETRY_ALLOWED, "1"); - properties.put(VideoCodecInfo.H264_FMTP_PACKETIZATION_MODE, "1"); - properties.put(VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID, - highProfile ? VideoCodecInfo.H264_CONSTRAINED_HIGH_3_1 - : VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1); - return properties; - default: - throw new IllegalArgumentException("Unsupported codec: " + type); - } - } - private static native boolean nativeIsSameH264Profile( Map params1, Map params2); } diff --git a/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java b/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java index d3158ff281..095e01a1bd 100644 --- a/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java +++ b/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java @@ -10,6 +10,9 @@ package org.webrtc; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import javax.annotation.Nullable; public class SoftwareVideoDecoderFactory implements VideoDecoderFactory { @@ -25,4 +28,20 @@ public class SoftwareVideoDecoderFactory implements VideoDecoderFactory { return null; } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return supportedCodecs(); + } + + static VideoCodecInfo[] supportedCodecs() { + List codecs = new ArrayList(); + + codecs.add(new VideoCodecInfo("VP8", new HashMap<>())); + if (VP9Decoder.nativeIsSupported()) { + codecs.add(new VideoCodecInfo("VP9", new HashMap<>())); + } + + return codecs.toArray(new VideoCodecInfo[codecs.size()]); + } } diff --git a/sdk/android/api/org/webrtc/VideoDecoderFactory.java b/sdk/android/api/org/webrtc/VideoDecoderFactory.java index ca903f89be..77abd79c81 100644 --- a/sdk/android/api/org/webrtc/VideoDecoderFactory.java +++ b/sdk/android/api/org/webrtc/VideoDecoderFactory.java @@ -19,4 +19,12 @@ public interface VideoDecoderFactory { * VideoEncoderFactory. */ @Nullable @CalledByNative VideoDecoder createDecoder(String codecType); + + /** + * Enumerates the list of supported video codecs. + */ + @CalledByNative + default VideoCodecInfo[] getSupportedCodecs() { + return new VideoCodecInfo[0]; + } } diff --git a/sdk/android/src/java/org/webrtc/MediaCodecUtils.java b/sdk/android/src/java/org/webrtc/MediaCodecUtils.java index 2fee1c3ca3..5b3ff602c8 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecUtils.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecUtils.java @@ -14,6 +14,8 @@ import android.annotation.TargetApi; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecInfo.CodecCapabilities; +import java.util.HashMap; +import java.util.Map; import javax.annotation.Nullable; /** Container class for static constants and helpers used with MediaCodec. */ @@ -76,6 +78,24 @@ class MediaCodecUtils { return false; } + static Map getCodecProperties(VideoCodecType type, boolean highProfile) { + switch (type) { + case VP8: + case VP9: + return new HashMap(); + case H264: + Map properties = new HashMap<>(); + properties.put(VideoCodecInfo.H264_FMTP_LEVEL_ASYMMETRY_ALLOWED, "1"); + properties.put(VideoCodecInfo.H264_FMTP_PACKETIZATION_MODE, "1"); + properties.put(VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID, + highProfile ? VideoCodecInfo.H264_CONSTRAINED_HIGH_3_1 + : VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1); + return properties; + default: + throw new IllegalArgumentException("Unsupported codec: " + type); + } + } + private MediaCodecUtils() { // This class should not be instantiated. } diff --git a/sdk/android/src/jni/videodecoderfactorywrapper.cc b/sdk/android/src/jni/videodecoderfactorywrapper.cc index be717f930b..22bd17fae4 100644 --- a/sdk/android/src/jni/videodecoderfactorywrapper.cc +++ b/sdk/android/src/jni/videodecoderfactorywrapper.cc @@ -16,6 +16,7 @@ #include "rtc_base/logging.h" #include "sdk/android/generated_video_jni/jni/VideoDecoderFactory_jni.h" #include "sdk/android/native_api/jni/java_types.h" +#include "sdk/android/src/jni/videocodecinfo.h" #include "sdk/android/src/jni/wrappednativecodec.h" namespace webrtc { @@ -39,8 +40,10 @@ std::unique_ptr VideoDecoderFactoryWrapper::CreateVideoDecoder( std::vector VideoDecoderFactoryWrapper::GetSupportedFormats() const { - // TODO(andersc): VideoDecoderFactory.java does not have this method. - return std::vector(); + JNIEnv* env = AttachCurrentThreadIfNeeded(); + return JavaToNativeVector( + env, Java_VideoDecoderFactory_getSupportedCodecs(env, decoder_factory_), + &VideoCodecInfoToSdpVideoFormat); } } // namespace jni