diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 013cbbe35b..9ecf0afb34 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -116,7 +116,6 @@ if (is_android) { ":builtin_audio_codecs_jni", ":default_video_codec_factory_jni", ":java_audio_device_module_jni", - ":legacy_hwcodecs_jni", ":peerconnection_jni", ":video_jni", "../../api:create_peerconnection_factory", @@ -269,8 +268,6 @@ if (is_android) { "api/org/webrtc/FrameDecryptor.java", "api/org/webrtc/FrameEncryptor.java", "api/org/webrtc/IceCandidate.java", - "api/org/webrtc/MediaCodecVideoDecoder.java", - "api/org/webrtc/MediaCodecVideoEncoder.java", "api/org/webrtc/MediaConstraints.java", "api/org/webrtc/MediaSource.java", "api/org/webrtc/MediaStream.java", @@ -598,41 +595,6 @@ if (current_os == "linux" || is_android) { ] } - # Corresponds to MediaCodecVideoEncoder/Decoder in Java. - rtc_library("legacy_hwcodecs_jni") { - visibility = [ "*" ] - allow_poison = [ "software_video_codecs" ] - sources = [ - "src/jni/android_media_codec_common.h", - "src/jni/android_media_decoder.cc", - "src/jni/android_media_encoder.cc", - ] - deps = [ - ":base_jni", - ":default_video_codec_factory_jni", - ":generated_video_jni", - ":native_api_jni", - ":video_jni", - ":videoframe_jni", - "../../api:scoped_refptr", - "../../api/task_queue", - "../../api/video_codecs:video_codecs_api", - "../../common_video", - "../../media:rtc_internal_video_codecs", - "../../media:rtc_media_base", - "../../modules/video_coding:video_codec_interface", - "../../modules/video_coding:video_coding_utility", - "../../rtc_base", - "../../rtc_base:checks", - "../../rtc_base:rtc_task_queue", - "../../rtc_base:weak_ptr", - "../../rtc_base/synchronization:sequence_checker", - "../../system_wrappers:field_trial", - "//third_party/abseil-cpp/absl/memory", - "//third_party/libyuv", - ] - } - rtc_library("video_jni") { visibility = [ "*" ] sources = [ @@ -1204,8 +1166,6 @@ if (current_os == "linux" || is_android) { sources = [ "api/org/webrtc/EncodedImage.java", "api/org/webrtc/JavaI420Buffer.java", - "api/org/webrtc/MediaCodecVideoDecoder.java", - "api/org/webrtc/MediaCodecVideoEncoder.java", "api/org/webrtc/TimestampAligner.java", "api/org/webrtc/VideoCodecInfo.java", "api/org/webrtc/VideoCodecStatus.java", @@ -1360,7 +1320,6 @@ if (is_android) { "instrumentationtests/src/org/webrtc/GlRectDrawerTest.java", "instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java", "instrumentationtests/src/org/webrtc/LoggableTest.java", - "instrumentationtests/src/org/webrtc/MediaCodecVideoEncoderTest.java", "instrumentationtests/src/org/webrtc/NetworkMonitorTest.java", "instrumentationtests/src/org/webrtc/PeerConnectionEndToEndTest.java", "instrumentationtests/src/org/webrtc/PeerConnectionFactoryTest.java", diff --git a/sdk/android/api/org/webrtc/MediaCodecVideoDecoder.java b/sdk/android/api/org/webrtc/MediaCodecVideoDecoder.java deleted file mode 100644 index 5b51df26cb..0000000000 --- a/sdk/android/api/org/webrtc/MediaCodecVideoDecoder.java +++ /dev/null @@ -1,1009 +0,0 @@ -/* - * Copyright 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -package org.webrtc; - -import android.media.MediaCodec; -import android.media.MediaCodecInfo; -import android.media.MediaCodecInfo.CodecCapabilities; -import android.media.MediaCodecList; -import android.media.MediaFormat; -import android.os.Build; -import android.os.SystemClock; -import android.support.annotation.Nullable; -import android.view.Surface; -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import org.webrtc.EglBase; -import org.webrtc.VideoFrame; - -// Java-side of peerconnection.cc:MediaCodecVideoDecoder. -// This class is an implementation detail of the Java PeerConnection API. -@SuppressWarnings("deprecation") -@Deprecated -public class MediaCodecVideoDecoder { - // This class is constructed, operated, and destroyed by its C++ incarnation, - // so the class and its methods have non-public visibility. The API this - // class exposes aims to mimic the webrtc::VideoDecoder API as closely as - // possibly to minimize the amount of translation work necessary. - - private static final String TAG = "MediaCodecVideoDecoder"; - - /** - * Create a VideoDecoderFactory that can be injected in the PeerConnectionFactory and replicate - * the old behavior. - */ - public static VideoDecoderFactory createFactory() { - return new DefaultVideoDecoderFactory(new HwDecoderFactory()); - } - - // Factory for creating HW MediaCodecVideoDecoder instances. - static class HwDecoderFactory implements VideoDecoderFactory { - private static boolean isSameCodec(VideoCodecInfo codecA, VideoCodecInfo codecB) { - if (!codecA.name.equalsIgnoreCase(codecB.name)) { - return false; - } - return codecA.name.equalsIgnoreCase("H264") - ? H264Utils.isSameH264Profile(codecA.params, codecB.params) - : true; - } - - private static boolean isCodecSupported( - VideoCodecInfo[] supportedCodecs, VideoCodecInfo codec) { - for (VideoCodecInfo supportedCodec : supportedCodecs) { - if (isSameCodec(supportedCodec, codec)) { - return true; - } - } - return false; - } - - private static VideoCodecInfo[] getSupportedHardwareCodecs() { - final List codecs = new ArrayList(); - - if (isVp8HwSupported()) { - Logging.d(TAG, "VP8 HW Decoder supported."); - codecs.add(new VideoCodecInfo("VP8", new HashMap<>())); - } - - if (isVp9HwSupported()) { - Logging.d(TAG, "VP9 HW Decoder supported."); - codecs.add(new VideoCodecInfo("VP9", new HashMap<>())); - } - - if (isH264HighProfileHwSupported()) { - Logging.d(TAG, "H.264 High Profile HW Decoder supported."); - codecs.add(H264Utils.DEFAULT_H264_HIGH_PROFILE_CODEC); - } - - if (isH264HwSupported()) { - Logging.d(TAG, "H.264 HW Decoder supported."); - codecs.add(H264Utils.DEFAULT_H264_BASELINE_PROFILE_CODEC); - } - - return codecs.toArray(new VideoCodecInfo[codecs.size()]); - } - - private final VideoCodecInfo[] supportedHardwareCodecs = getSupportedHardwareCodecs(); - - @Override - public VideoCodecInfo[] getSupportedCodecs() { - return supportedHardwareCodecs; - } - - @Nullable - @Override - public VideoDecoder createDecoder(VideoCodecInfo codec) { - if (!isCodecSupported(supportedHardwareCodecs, codec)) { - Logging.d(TAG, "No HW video decoder for codec " + codec.name); - return null; - } - Logging.d(TAG, "Create HW video decoder for " + codec.name); - return new WrappedNativeVideoDecoder() { - @Override - public long createNativeVideoDecoder() { - return nativeCreateDecoder(codec.name, useSurface()); - } - }; - } - } - - private static final long MAX_DECODE_TIME_MS = 200; - - // TODO(magjed): Use MediaFormat constants when part of the public API. - private static final String FORMAT_KEY_STRIDE = "stride"; - private static final String FORMAT_KEY_SLICE_HEIGHT = "slice-height"; - private static final String FORMAT_KEY_CROP_LEFT = "crop-left"; - private static final String FORMAT_KEY_CROP_RIGHT = "crop-right"; - private static final String FORMAT_KEY_CROP_TOP = "crop-top"; - private static final String FORMAT_KEY_CROP_BOTTOM = "crop-bottom"; - - // Timeout for input buffer dequeue. - private static final int DEQUEUE_INPUT_TIMEOUT = 500000; - // Timeout for codec releasing. - private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; - // Max number of output buffers queued before starting to drop decoded frames. - private static final int MAX_QUEUED_OUTPUTBUFFERS = 3; - // Active running decoder instance. Set in initDecode() (called from native code) - // and reset to null in release() call. - @Nullable private static MediaCodecVideoDecoder runningInstance; - @Nullable private static MediaCodecVideoDecoderErrorCallback errorCallback; - private static int codecErrors; - // List of disabled codec types - can be set from application. - private static Set hwDecoderDisabledTypes = new HashSet(); - @Nullable private static EglBase eglBase; - - @Nullable private Thread mediaCodecThread; - @Nullable private MediaCodec mediaCodec; - private ByteBuffer[] inputBuffers; - private ByteBuffer[] outputBuffers; - private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8"; - private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9"; - private static final String H264_MIME_TYPE = "video/avc"; - // List of supported HW VP8 decoders. - private static final String[] supportedVp8HwCodecPrefixes() { - ArrayList supportedPrefixes = new ArrayList(); - supportedPrefixes.add("OMX.qcom."); - supportedPrefixes.add("OMX.Nvidia."); - supportedPrefixes.add("OMX.Exynos."); - supportedPrefixes.add("OMX.Intel."); - if (PeerConnectionFactory.fieldTrialsFindFullName("WebRTC-MediaTekVP8").equals("Enabled") - && Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - supportedPrefixes.add("OMX.MTK."); - } - return supportedPrefixes.toArray(new String[supportedPrefixes.size()]); - } - // List of supported HW VP9 decoders. - private static final String[] supportedVp9HwCodecPrefixes = {"OMX.qcom.", "OMX.Exynos."}; - // List of supported HW H.264 decoders. - private static final String[] supportedH264HwCodecPrefixes() { - ArrayList supportedPrefixes = new ArrayList(); - supportedPrefixes.add("OMX.qcom."); - supportedPrefixes.add("OMX.Intel."); - supportedPrefixes.add("OMX.Exynos."); - if (PeerConnectionFactory.fieldTrialsFindFullName("WebRTC-MediaTekH264").equals("Enabled") - && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { - supportedPrefixes.add("OMX.MTK."); - } - return supportedPrefixes.toArray(new String[supportedPrefixes.size()]); - } - - // List of supported HW H.264 high profile decoders. - private static final String supportedQcomH264HighProfileHwCodecPrefix = "OMX.qcom."; - private static final String supportedExynosH264HighProfileHwCodecPrefix = "OMX.Exynos."; - private static final String supportedMediaTekH264HighProfileHwCodecPrefix = "OMX.MTK."; - - // NV12 color format supported by QCOM codec, but not declared in MediaCodec - - // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h - private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka = 0x7FA30C01; - private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka = 0x7FA30C02; - private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka = 0x7FA30C03; - private static final int COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04; - // Allowable color formats supported by codec - in order of preference. - private static final List supportedColorList = Arrays.asList( - CodecCapabilities.COLOR_FormatYUV420Planar, CodecCapabilities.COLOR_FormatYUV420SemiPlanar, - CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, - COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka, COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka, - COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka, - COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); - - private int colorFormat; - private int width; - private int height; - private int stride; - private int sliceHeight; - private boolean hasDecodedFirstFrame; - private final Queue decodeStartTimeMs = new ArrayDeque(); - - // The below variables are only used when decoding to a Surface. - @Nullable private TextureListener textureListener; - private int droppedFrames; - @Nullable private Surface surface; - private final Queue dequeuedSurfaceOutputBuffers = - new ArrayDeque(); - - // MediaCodec error handler - invoked when critical error happens which may prevent - // further use of media codec API. Now it means that one of media codec instances - // is hanging and can no longer be used in the next call. - public static interface MediaCodecVideoDecoderErrorCallback { - void onMediaCodecVideoDecoderCriticalError(int codecErrors); - } - - /** Set EGL context used by HW decoding. The EGL context must be shared with the remote render. */ - public static void setEglContext(EglBase.Context eglContext) { - if (eglBase != null) { - Logging.w(TAG, "Egl context already set."); - eglBase.release(); - } - eglBase = EglBase.create(eglContext); - } - - /** Dispose the EGL context used by HW decoding. */ - public static void disposeEglContext() { - if (eglBase != null) { - eglBase.release(); - eglBase = null; - } - } - - static boolean useSurface() { - return eglBase != null; - } - - public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorCallback) { - Logging.d(TAG, "Set error callback"); - MediaCodecVideoDecoder.errorCallback = errorCallback; - } - - // Functions to disable HW decoding - can be called from applications for platforms - // which have known HW decoding problems. - public static void disableVp8HwCodec() { - Logging.w(TAG, "VP8 decoding is disabled by application."); - hwDecoderDisabledTypes.add(VP8_MIME_TYPE); - } - - public static void disableVp9HwCodec() { - Logging.w(TAG, "VP9 decoding is disabled by application."); - hwDecoderDisabledTypes.add(VP9_MIME_TYPE); - } - - public static void disableH264HwCodec() { - Logging.w(TAG, "H.264 decoding is disabled by application."); - hwDecoderDisabledTypes.add(H264_MIME_TYPE); - } - - // Functions to query if HW decoding is supported. - public static boolean isVp8HwSupported() { - return !hwDecoderDisabledTypes.contains(VP8_MIME_TYPE) - && (findDecoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes()) != null); - } - - public static boolean isVp9HwSupported() { - return !hwDecoderDisabledTypes.contains(VP9_MIME_TYPE) - && (findDecoder(VP9_MIME_TYPE, supportedVp9HwCodecPrefixes) != null); - } - - public static boolean isH264HwSupported() { - return !hwDecoderDisabledTypes.contains(H264_MIME_TYPE) - && (findDecoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes()) != null); - } - - public static boolean isH264HighProfileHwSupported() { - if (hwDecoderDisabledTypes.contains(H264_MIME_TYPE)) { - return false; - } - // Support H.264 HP decoding on QCOM chips for Android L and above. - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP - && findDecoder(H264_MIME_TYPE, new String[] {supportedQcomH264HighProfileHwCodecPrefix}) - != null) { - return true; - } - // Support H.264 HP decoding on Exynos chips for Android M and above. - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M - && findDecoder(H264_MIME_TYPE, new String[] {supportedExynosH264HighProfileHwCodecPrefix}) - != null) { - return true; - } - // Support H.264 HP decoding on MediaTek chips for Android O_MR1 and above - if (PeerConnectionFactory.fieldTrialsFindFullName("WebRTC-MediaTekH264").equals("Enabled") - && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1 - && findDecoder(H264_MIME_TYPE, new String[] {supportedMediaTekH264HighProfileHwCodecPrefix}) - != null) { - return true; - } - return false; - } - - public static void printStackTrace() { - if (runningInstance != null && runningInstance.mediaCodecThread != null) { - StackTraceElement[] mediaCodecStackTraces = runningInstance.mediaCodecThread.getStackTrace(); - if (mediaCodecStackTraces.length > 0) { - Logging.d(TAG, "MediaCodecVideoDecoder stacks trace:"); - for (StackTraceElement stackTrace : mediaCodecStackTraces) { - Logging.d(TAG, stackTrace.toString()); - } - } - } - } - - // Helper struct for findDecoder() below. - private static class DecoderProperties { - public DecoderProperties(String codecName, int colorFormat) { - this.codecName = codecName; - this.colorFormat = colorFormat; - } - public final String codecName; // OpenMax component name for VP8 codec. - public final int colorFormat; // Color format supported by codec. - } - - private static @Nullable DecoderProperties findDecoder( - String mime, String[] supportedCodecPrefixes) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { - return null; // MediaCodec.setParameters is missing. - } - Logging.d(TAG, "Trying to find HW decoder for mime " + mime); - for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { - MediaCodecInfo info = null; - try { - info = MediaCodecList.getCodecInfoAt(i); - } catch (IllegalArgumentException e) { - Logging.e(TAG, "Cannot retrieve decoder codec info", e); - } - if (info == null || info.isEncoder()) { - continue; - } - String name = null; - for (String mimeType : info.getSupportedTypes()) { - if (mimeType.equals(mime)) { - name = info.getName(); - break; - } - } - if (name == null) { - continue; // No HW support in this codec; try the next one. - } - Logging.d(TAG, "Found candidate decoder " + name); - - // Check if this is supported decoder. - boolean supportedCodec = false; - for (String codecPrefix : supportedCodecPrefixes) { - if (name.startsWith(codecPrefix)) { - supportedCodec = true; - break; - } - } - if (!supportedCodec) { - continue; - } - - // Check if codec supports either yuv420 or nv12. - CodecCapabilities capabilities; - try { - capabilities = info.getCapabilitiesForType(mime); - } catch (IllegalArgumentException e) { - Logging.e(TAG, "Cannot retrieve decoder capabilities", e); - continue; - } - for (int colorFormat : capabilities.colorFormats) { - Logging.v(TAG, " Color: 0x" + Integer.toHexString(colorFormat)); - } - for (int supportedColorFormat : supportedColorList) { - for (int codecColorFormat : capabilities.colorFormats) { - if (codecColorFormat == supportedColorFormat) { - // Found supported HW decoder. - Logging.d(TAG, "Found target decoder " + name + ". Color: 0x" - + Integer.toHexString(codecColorFormat)); - return new DecoderProperties(name, codecColorFormat); - } - } - } - } - Logging.d(TAG, "No HW decoder found for mime " + mime); - return null; // No HW decoder. - } - - @CalledByNative - MediaCodecVideoDecoder() {} - - private void checkOnMediaCodecThread() throws IllegalStateException { - if (mediaCodecThread.getId() != Thread.currentThread().getId()) { - throw new IllegalStateException("MediaCodecVideoDecoder previously operated on " - + mediaCodecThread + " but is now called on " + Thread.currentThread()); - } - } - - @CalledByNativeUnchecked - private boolean initDecode(@VideoCodecType int type, int width, int height) { - if (mediaCodecThread != null) { - throw new RuntimeException("initDecode: Forgot to release()?"); - } - - String mime = null; - String[] supportedCodecPrefixes = null; - if (type == VideoCodecType.VIDEO_CODEC_VP8) { - mime = VP8_MIME_TYPE; - supportedCodecPrefixes = supportedVp8HwCodecPrefixes(); - } else if (type == VideoCodecType.VIDEO_CODEC_VP9) { - mime = VP9_MIME_TYPE; - supportedCodecPrefixes = supportedVp9HwCodecPrefixes; - } else if (type == VideoCodecType.VIDEO_CODEC_H264) { - mime = H264_MIME_TYPE; - supportedCodecPrefixes = supportedH264HwCodecPrefixes(); - } else { - throw new RuntimeException("initDecode: Non-supported codec " + type); - } - DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); - if (properties == null) { - throw new RuntimeException("Cannot find HW decoder for " + type); - } - - Logging.d(TAG, - "Java initDecode: " + type + " : " + width + " x " + height + ". Color: 0x" - + Integer.toHexString(properties.colorFormat) + ". Use Surface: " + useSurface()); - - runningInstance = this; // Decoder is now running and can be queried for stack traces. - mediaCodecThread = Thread.currentThread(); - try { - this.width = width; - this.height = height; - stride = width; - sliceHeight = height; - - if (useSurface()) { - @Nullable - final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create( - "Decoder SurfaceTextureHelper", eglBase.getEglBaseContext()); - if (surfaceTextureHelper != null) { - textureListener = new TextureListener(surfaceTextureHelper); - textureListener.setSize(width, height); - surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); - } - } - - MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); - if (!useSurface()) { - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); - } - Logging.d(TAG, " Format: " + format); - mediaCodec = MediaCodecVideoEncoder.createByCodecName(properties.codecName); - if (mediaCodec == null) { - Logging.e(TAG, "Can not create media decoder"); - return false; - } - mediaCodec.configure(format, surface, null, 0); - mediaCodec.start(); - - colorFormat = properties.colorFormat; - outputBuffers = mediaCodec.getOutputBuffers(); - inputBuffers = mediaCodec.getInputBuffers(); - decodeStartTimeMs.clear(); - hasDecodedFirstFrame = false; - dequeuedSurfaceOutputBuffers.clear(); - droppedFrames = 0; - Logging.d(TAG, - "Input buffers: " + inputBuffers.length + ". Output buffers: " + outputBuffers.length); - return true; - } catch (IllegalStateException e) { - Logging.e(TAG, "initDecode failed", e); - return false; - } - } - - // Resets the decoder so it can start decoding frames with new resolution. - // Flushes MediaCodec and clears decoder output buffers. - @CalledByNativeUnchecked - private void reset(int width, int height) { - if (mediaCodecThread == null || mediaCodec == null) { - throw new RuntimeException("Incorrect reset call for non-initialized decoder."); - } - Logging.d(TAG, "Java reset: " + width + " x " + height); - - mediaCodec.flush(); - - this.width = width; - this.height = height; - if (textureListener != null) { - textureListener.setSize(width, height); - } - decodeStartTimeMs.clear(); - dequeuedSurfaceOutputBuffers.clear(); - hasDecodedFirstFrame = false; - droppedFrames = 0; - } - - @CalledByNativeUnchecked - private void release() { - Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + droppedFrames); - checkOnMediaCodecThread(); - - // Run Mediacodec stop() and release() on separate thread since sometime - // Mediacodec.stop() may hang. - final CountDownLatch releaseDone = new CountDownLatch(1); - - Runnable runMediaCodecRelease = new Runnable() { - @Override - public void run() { - try { - Logging.d(TAG, "Java releaseDecoder on release thread"); - mediaCodec.stop(); - mediaCodec.release(); - Logging.d(TAG, "Java releaseDecoder on release thread done"); - } catch (Exception e) { - Logging.e(TAG, "Media decoder release failed", e); - } - releaseDone.countDown(); - } - }; - new Thread(runMediaCodecRelease).start(); - - if (!ThreadUtils.awaitUninterruptibly(releaseDone, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { - Logging.e(TAG, "Media decoder release timeout"); - codecErrors++; - if (errorCallback != null) { - Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors); - errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); - } - } - - mediaCodec = null; - mediaCodecThread = null; - runningInstance = null; - if (useSurface()) { - surface.release(); - surface = null; - textureListener.release(); - } - Logging.d(TAG, "Java releaseDecoder done"); - } - - // Dequeue an input buffer and return its index, -1 if no input buffer is - // available, or -2 if the codec is no longer operative. - @CalledByNativeUnchecked - private int dequeueInputBuffer() { - checkOnMediaCodecThread(); - try { - return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); - } catch (IllegalStateException e) { - Logging.e(TAG, "dequeueIntputBuffer failed", e); - return -2; - } - } - - @CalledByNativeUnchecked - private boolean queueInputBuffer(int inputBufferIndex, int size, long presentationTimeStamUs, - long timeStampMs, long ntpTimeStamp) { - checkOnMediaCodecThread(); - try { - inputBuffers[inputBufferIndex].position(0); - inputBuffers[inputBufferIndex].limit(size); - decodeStartTimeMs.add( - new TimeStamps(SystemClock.elapsedRealtime(), timeStampMs, ntpTimeStamp)); - mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, presentationTimeStamUs, 0); - return true; - } catch (IllegalStateException e) { - Logging.e(TAG, "decode failed", e); - return false; - } - } - - private static class TimeStamps { - public TimeStamps(long decodeStartTimeMs, long timeStampMs, long ntpTimeStampMs) { - this.decodeStartTimeMs = decodeStartTimeMs; - this.timeStampMs = timeStampMs; - this.ntpTimeStampMs = ntpTimeStampMs; - } - // Time when this frame was queued for decoding. - private final long decodeStartTimeMs; - // Only used for bookkeeping in Java. Stores C++ inputImage._timeStamp value for input frame. - private final long timeStampMs; - // Only used for bookkeeping in Java. Stores C++ inputImage.ntp_time_ms_ value for input frame. - private final long ntpTimeStampMs; - } - - // Helper struct for dequeueOutputBuffer() below. - private static class DecodedOutputBuffer { - public DecodedOutputBuffer(int index, int offset, int size, long presentationTimeStampMs, - long timeStampMs, long ntpTimeStampMs, long decodeTime, long endDecodeTime) { - this.index = index; - this.offset = offset; - this.size = size; - this.presentationTimeStampMs = presentationTimeStampMs; - this.timeStampMs = timeStampMs; - this.ntpTimeStampMs = ntpTimeStampMs; - this.decodeTimeMs = decodeTime; - this.endDecodeTimeMs = endDecodeTime; - } - - private final int index; - private final int offset; - private final int size; - // Presentation timestamp returned in dequeueOutputBuffer call. - private final long presentationTimeStampMs; - // C++ inputImage._timeStamp value for output frame. - private final long timeStampMs; - // C++ inputImage.ntp_time_ms_ value for output frame. - private final long ntpTimeStampMs; - // Number of ms it took to decode this frame. - private final long decodeTimeMs; - // System time when this frame decoding finished. - private final long endDecodeTimeMs; - - @CalledByNative("DecodedOutputBuffer") - int getIndex() { - return index; - } - - @CalledByNative("DecodedOutputBuffer") - int getOffset() { - return offset; - } - - @CalledByNative("DecodedOutputBuffer") - int getSize() { - return size; - } - - @CalledByNative("DecodedOutputBuffer") - long getPresentationTimestampMs() { - return presentationTimeStampMs; - } - - @CalledByNative("DecodedOutputBuffer") - long getTimestampMs() { - return timeStampMs; - } - - @CalledByNative("DecodedOutputBuffer") - long getNtpTimestampMs() { - return ntpTimeStampMs; - } - - @CalledByNative("DecodedOutputBuffer") - long getDecodeTimeMs() { - return decodeTimeMs; - } - } - - // Helper struct for dequeueTextureBuffer() below. - private static class DecodedTextureBuffer { - private final VideoFrame.Buffer videoFrameBuffer; - // Presentation timestamp returned in dequeueOutputBuffer call. - private final long presentationTimeStampMs; - // C++ inputImage._timeStamp value for output frame. - private final long timeStampMs; - // C++ inputImage.ntp_time_ms_ value for output frame. - private final long ntpTimeStampMs; - // Number of ms it took to decode this frame. - private final long decodeTimeMs; - // Interval from when the frame finished decoding until this buffer has been created. - // Since there is only one texture, this interval depend on the time from when - // a frame is decoded and provided to C++ and until that frame is returned to the MediaCodec - // so that the texture can be updated with the next decoded frame. - private final long frameDelayMs; - - // A DecodedTextureBuffer with zero |textureID| has special meaning and represents a frame - // that was dropped. - public DecodedTextureBuffer(VideoFrame.Buffer videoFrameBuffer, long presentationTimeStampMs, - long timeStampMs, long ntpTimeStampMs, long decodeTimeMs, long frameDelay) { - this.videoFrameBuffer = videoFrameBuffer; - this.presentationTimeStampMs = presentationTimeStampMs; - this.timeStampMs = timeStampMs; - this.ntpTimeStampMs = ntpTimeStampMs; - this.decodeTimeMs = decodeTimeMs; - this.frameDelayMs = frameDelay; - } - - @CalledByNative("DecodedTextureBuffer") - VideoFrame.Buffer getVideoFrameBuffer() { - return videoFrameBuffer; - } - - @CalledByNative("DecodedTextureBuffer") - long getPresentationTimestampMs() { - return presentationTimeStampMs; - } - - @CalledByNative("DecodedTextureBuffer") - long getTimeStampMs() { - return timeStampMs; - } - - @CalledByNative("DecodedTextureBuffer") - long getNtpTimestampMs() { - return ntpTimeStampMs; - } - - @CalledByNative("DecodedTextureBuffer") - long getDecodeTimeMs() { - return decodeTimeMs; - } - - @CalledByNative("DecodedTextureBuffer") - long getFrameDelayMs() { - return frameDelayMs; - } - } - - // Poll based texture listener. - private class TextureListener implements VideoSink { - private final SurfaceTextureHelper surfaceTextureHelper; - // |newFrameLock| is used to synchronize arrival of new frames with wait()/notifyAll(). - private final Object newFrameLock = new Object(); - // |bufferToRender| is non-null when waiting for transition between addBufferToRender() to - // onFrame(). - @Nullable private DecodedOutputBuffer bufferToRender; - @Nullable private DecodedTextureBuffer renderedBuffer; - - public TextureListener(SurfaceTextureHelper surfaceTextureHelper) { - this.surfaceTextureHelper = surfaceTextureHelper; - surfaceTextureHelper.startListening(this); - } - - public void addBufferToRender(DecodedOutputBuffer buffer) { - if (bufferToRender != null) { - Logging.e(TAG, "Unexpected addBufferToRender() called while waiting for a texture."); - throw new IllegalStateException("Waiting for a texture."); - } - bufferToRender = buffer; - } - - public boolean isWaitingForTexture() { - synchronized (newFrameLock) { - return bufferToRender != null; - } - } - - public void setSize(int width, int height) { - surfaceTextureHelper.setTextureSize(width, height); - } - - // Callback from |surfaceTextureHelper|. May be called on an arbitrary thread. - @Override - public void onFrame(VideoFrame frame) { - synchronized (newFrameLock) { - if (renderedBuffer != null) { - Logging.e(TAG, "Unexpected onFrame() called while already holding a texture."); - throw new IllegalStateException("Already holding a texture."); - } - // |timestampNs| is always zero on some Android versions. - final VideoFrame.Buffer buffer = frame.getBuffer(); - buffer.retain(); - renderedBuffer = new DecodedTextureBuffer(buffer, bufferToRender.presentationTimeStampMs, - bufferToRender.timeStampMs, bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs, - SystemClock.elapsedRealtime() - bufferToRender.endDecodeTimeMs); - bufferToRender = null; - newFrameLock.notifyAll(); - } - } - - // Dequeues and returns a DecodedTextureBuffer if available, or null otherwise. - @Nullable - @SuppressWarnings("WaitNotInLoop") - public DecodedTextureBuffer dequeueTextureBuffer(int timeoutMs) { - synchronized (newFrameLock) { - if (renderedBuffer == null && timeoutMs > 0 && isWaitingForTexture()) { - try { - newFrameLock.wait(timeoutMs); - } catch (InterruptedException e) { - // Restore the interrupted status by reinterrupting the thread. - Thread.currentThread().interrupt(); - } - } - DecodedTextureBuffer returnedBuffer = renderedBuffer; - renderedBuffer = null; - return returnedBuffer; - } - } - - public void release() { - // SurfaceTextureHelper.stopListening() will block until any onFrame() in progress is done. - // Therefore, the call must be outside any synchronized statement that is also used in the - // onFrame() above to avoid deadlocks. - surfaceTextureHelper.stopListening(); - synchronized (newFrameLock) { - if (renderedBuffer != null) { - renderedBuffer.getVideoFrameBuffer().release(); - renderedBuffer = null; - } - } - surfaceTextureHelper.dispose(); - } - } - - // Returns null if no decoded buffer is available, and otherwise a DecodedByteBuffer. - // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an - // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException - // upon codec error. - @CalledByNativeUnchecked - private @Nullable DecodedOutputBuffer dequeueOutputBuffer(int dequeueTimeoutMs) { - checkOnMediaCodecThread(); - if (decodeStartTimeMs.isEmpty()) { - return null; - } - // Drain the decoder until receiving a decoded buffer or hitting - // MediaCodec.INFO_TRY_AGAIN_LATER. - final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); - while (true) { - final int result = - mediaCodec.dequeueOutputBuffer(info, TimeUnit.MILLISECONDS.toMicros(dequeueTimeoutMs)); - switch (result) { - case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: - outputBuffers = mediaCodec.getOutputBuffers(); - Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.length); - if (hasDecodedFirstFrame) { - throw new RuntimeException("Unexpected output buffer change event."); - } - break; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - MediaFormat format = mediaCodec.getOutputFormat(); - Logging.d(TAG, "Decoder format changed: " + format.toString()); - final int newWidth; - final int newHeight; - if (format.containsKey(FORMAT_KEY_CROP_LEFT) && format.containsKey(FORMAT_KEY_CROP_RIGHT) - && format.containsKey(FORMAT_KEY_CROP_BOTTOM) - && format.containsKey(FORMAT_KEY_CROP_TOP)) { - newWidth = 1 + format.getInteger(FORMAT_KEY_CROP_RIGHT) - - format.getInteger(FORMAT_KEY_CROP_LEFT); - newHeight = 1 + format.getInteger(FORMAT_KEY_CROP_BOTTOM) - - format.getInteger(FORMAT_KEY_CROP_TOP); - } else { - newWidth = format.getInteger(MediaFormat.KEY_WIDTH); - newHeight = format.getInteger(MediaFormat.KEY_HEIGHT); - } - if (hasDecodedFirstFrame && (newWidth != width || newHeight != height)) { - throw new RuntimeException("Unexpected size change. Configured " + width + "*" + height - + ". New " + newWidth + "*" + newHeight); - } - width = newWidth; - height = newHeight; - if (textureListener != null) { - textureListener.setSize(width, height); - } - - if (!useSurface() && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { - colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); - Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); - if (!supportedColorList.contains(colorFormat)) { - throw new IllegalStateException("Non supported color format: " + colorFormat); - } - } - if (format.containsKey(FORMAT_KEY_STRIDE)) { - stride = format.getInteger(FORMAT_KEY_STRIDE); - } - if (format.containsKey(FORMAT_KEY_SLICE_HEIGHT)) { - sliceHeight = format.getInteger(FORMAT_KEY_SLICE_HEIGHT); - } - Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sliceHeight); - stride = Math.max(width, stride); - sliceHeight = Math.max(height, sliceHeight); - break; - case MediaCodec.INFO_TRY_AGAIN_LATER: - return null; - default: - hasDecodedFirstFrame = true; - TimeStamps timeStamps = decodeStartTimeMs.remove(); - long decodeTimeMs = SystemClock.elapsedRealtime() - timeStamps.decodeStartTimeMs; - if (decodeTimeMs > MAX_DECODE_TIME_MS) { - Logging.e(TAG, "Very high decode time: " + decodeTimeMs + "ms" - + ". Q size: " + decodeStartTimeMs.size() - + ". Might be caused by resuming H264 decoding after a pause."); - decodeTimeMs = MAX_DECODE_TIME_MS; - } - return new DecodedOutputBuffer(result, info.offset, info.size, - TimeUnit.MICROSECONDS.toMillis(info.presentationTimeUs), timeStamps.timeStampMs, - timeStamps.ntpTimeStampMs, decodeTimeMs, SystemClock.elapsedRealtime()); - } - } - } - - // Returns null if no decoded buffer is available, and otherwise a DecodedTextureBuffer. - // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an - // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException - // upon codec error. If |dequeueTimeoutMs| > 0, the oldest decoded frame will be dropped if - // a frame can't be returned. - @CalledByNativeUnchecked - private @Nullable DecodedTextureBuffer dequeueTextureBuffer(int dequeueTimeoutMs) { - checkOnMediaCodecThread(); - if (!useSurface()) { - throw new IllegalStateException("dequeueTexture() called for byte buffer decoding."); - } - DecodedOutputBuffer outputBuffer = dequeueOutputBuffer(dequeueTimeoutMs); - if (outputBuffer != null) { - dequeuedSurfaceOutputBuffers.add(outputBuffer); - } - - MaybeRenderDecodedTextureBuffer(); - // Check if there is texture ready now by waiting max |dequeueTimeoutMs|. - DecodedTextureBuffer renderedBuffer = textureListener.dequeueTextureBuffer(dequeueTimeoutMs); - if (renderedBuffer != null) { - MaybeRenderDecodedTextureBuffer(); - return renderedBuffer; - } - - if ((dequeuedSurfaceOutputBuffers.size() - >= Math.min(MAX_QUEUED_OUTPUTBUFFERS, outputBuffers.length) - || (dequeueTimeoutMs > 0 && !dequeuedSurfaceOutputBuffers.isEmpty()))) { - ++droppedFrames; - // Drop the oldest frame still in dequeuedSurfaceOutputBuffers. - // The oldest frame is owned by |textureListener| and can't be dropped since - // mediaCodec.releaseOutputBuffer has already been called. - final DecodedOutputBuffer droppedFrame = dequeuedSurfaceOutputBuffers.remove(); - if (dequeueTimeoutMs > 0) { - // TODO(perkj): Re-add the below log when VideoRenderGUI has been removed or fixed to - // return the one and only texture even if it does not render. - Logging.w(TAG, "Draining decoder. Dropping frame with TS: " - + droppedFrame.presentationTimeStampMs + ". Total number of dropped frames: " - + droppedFrames); - } else { - Logging.w(TAG, "Too many output buffers " + dequeuedSurfaceOutputBuffers.size() - + ". Dropping frame with TS: " + droppedFrame.presentationTimeStampMs - + ". Total number of dropped frames: " + droppedFrames); - } - - mediaCodec.releaseOutputBuffer(droppedFrame.index, false /* render */); - return new DecodedTextureBuffer(null /* videoFrameBuffer */, - droppedFrame.presentationTimeStampMs, droppedFrame.timeStampMs, - droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs, - SystemClock.elapsedRealtime() - droppedFrame.endDecodeTimeMs); - } - return null; - } - - private void MaybeRenderDecodedTextureBuffer() { - if (dequeuedSurfaceOutputBuffers.isEmpty() || textureListener.isWaitingForTexture()) { - return; - } - // Get the first frame in the queue and render to the decoder output surface. - final DecodedOutputBuffer buffer = dequeuedSurfaceOutputBuffers.remove(); - textureListener.addBufferToRender(buffer); - mediaCodec.releaseOutputBuffer(buffer.index, true /* render */); - } - - // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for - // non-surface decoding. - // Throws IllegalStateException if the call is made on the wrong thread, if codec is configured - // for surface decoding, or if |mediaCodec| is not in the Executing state. Throws - // MediaCodec.CodecException upon codec error. - @CalledByNativeUnchecked - private void returnDecodedOutputBuffer(int index) - throws IllegalStateException, MediaCodec.CodecException { - checkOnMediaCodecThread(); - if (useSurface()) { - throw new IllegalStateException("returnDecodedOutputBuffer() called for surface decoding."); - } - mediaCodec.releaseOutputBuffer(index, false /* render */); - } - - @CalledByNative - ByteBuffer[] getInputBuffers() { - return inputBuffers; - } - - @CalledByNative - ByteBuffer[] getOutputBuffers() { - return outputBuffers; - } - - @CalledByNative - int getColorFormat() { - return colorFormat; - } - - @CalledByNative - int getWidth() { - return width; - } - - @CalledByNative - int getHeight() { - return height; - } - - @CalledByNative - int getStride() { - return stride; - } - - @CalledByNative - int getSliceHeight() { - return sliceHeight; - } - - private static native long nativeCreateDecoder(String codec, boolean useSurface); -} diff --git a/sdk/android/api/org/webrtc/MediaCodecVideoEncoder.java b/sdk/android/api/org/webrtc/MediaCodecVideoEncoder.java deleted file mode 100644 index 1c9bc42ba5..0000000000 --- a/sdk/android/api/org/webrtc/MediaCodecVideoEncoder.java +++ /dev/null @@ -1,1094 +0,0 @@ -/* - * Copyright 2013 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -package org.webrtc; - -import android.annotation.TargetApi; -import android.graphics.Matrix; -import android.media.MediaCodec; -import android.media.MediaCodecInfo; -import android.media.MediaCodecInfo.CodecCapabilities; -import android.media.MediaCodecList; -import android.media.MediaFormat; -import android.opengl.GLES20; -import android.os.Build; -import android.os.Bundle; -import android.support.annotation.Nullable; -import android.view.Surface; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import org.webrtc.EglBase; -import org.webrtc.EglBase14; -import org.webrtc.VideoFrame; - -// Java-side of peerconnection.cc:MediaCodecVideoEncoder. -// This class is an implementation detail of the Java PeerConnection API. -@TargetApi(19) -@SuppressWarnings("deprecation") -@Deprecated -public class MediaCodecVideoEncoder { - // This class is constructed, operated, and destroyed by its C++ incarnation, - // so the class and its methods have non-public visibility. The API this - // class exposes aims to mimic the webrtc::VideoEncoder API as closely as - // possibly to minimize the amount of translation work necessary. - - private static final String TAG = "MediaCodecVideoEncoder"; - - /** - * Create a VideoEncoderFactory that can be injected in the PeerConnectionFactory and replicate - * the old behavior. - */ - public static VideoEncoderFactory createFactory() { - return new DefaultVideoEncoderFactory(new HwEncoderFactory()); - } - - // Factory for creating HW MediaCodecVideoEncoder instances. - static class HwEncoderFactory implements VideoEncoderFactory { - private static boolean isSameCodec(VideoCodecInfo codecA, VideoCodecInfo codecB) { - if (!codecA.name.equalsIgnoreCase(codecB.name)) { - return false; - } - return codecA.name.equalsIgnoreCase("H264") - ? H264Utils.isSameH264Profile(codecA.params, codecB.params) - : true; - } - - private static boolean isCodecSupported( - VideoCodecInfo[] supportedCodecs, VideoCodecInfo codec) { - for (VideoCodecInfo supportedCodec : supportedCodecs) { - if (isSameCodec(supportedCodec, codec)) { - return true; - } - } - return false; - } - - private static VideoCodecInfo[] getSupportedHardwareCodecs() { - final List codecs = new ArrayList(); - - if (isVp8HwSupported()) { - Logging.d(TAG, "VP8 HW Encoder supported."); - codecs.add(new VideoCodecInfo("VP8", new HashMap<>())); - } - - if (isVp9HwSupported()) { - Logging.d(TAG, "VP9 HW Encoder supported."); - codecs.add(new VideoCodecInfo("VP9", new HashMap<>())); - } - - // Check if high profile is supported by decoder. If yes, encoder can always - // fall back to baseline profile as a subset as high profile. - if (MediaCodecVideoDecoder.isH264HighProfileHwSupported()) { - Logging.d(TAG, "H.264 High Profile HW Encoder supported."); - codecs.add(H264Utils.DEFAULT_H264_HIGH_PROFILE_CODEC); - } - - if (isH264HwSupported()) { - Logging.d(TAG, "H.264 HW Encoder supported."); - codecs.add(H264Utils.DEFAULT_H264_BASELINE_PROFILE_CODEC); - } - - return codecs.toArray(new VideoCodecInfo[codecs.size()]); - } - - private final VideoCodecInfo[] supportedHardwareCodecs = getSupportedHardwareCodecs(); - - @Override - public VideoCodecInfo[] getSupportedCodecs() { - return supportedHardwareCodecs; - } - - @Nullable - @Override - public VideoEncoder createEncoder(VideoCodecInfo info) { - if (!isCodecSupported(supportedHardwareCodecs, info)) { - Logging.d(TAG, "No HW video encoder for codec " + info.name); - return null; - } - Logging.d(TAG, "Create HW video encoder for " + info.name); - return new WrappedNativeVideoEncoder() { - @Override - public long createNativeVideoEncoder() { - return nativeCreateEncoder( - info, /* hasEgl14Context= */ staticEglBase instanceof EglBase14); - } - - @Override - public boolean isHardwareEncoder() { - return true; - } - }; - } - } - - private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; // Timeout for codec releasing. - private static final int DEQUEUE_TIMEOUT = 0; // Non-blocking, no wait. - private static final int BITRATE_ADJUSTMENT_FPS = 30; - private static final int MAXIMUM_INITIAL_FPS = 30; - private static final double BITRATE_CORRECTION_SEC = 3.0; - // Maximum bitrate correction scale - no more than 4 times. - private static final double BITRATE_CORRECTION_MAX_SCALE = 4; - // Amount of correction steps to reach correction maximum scale. - private static final int BITRATE_CORRECTION_STEPS = 20; - // Forced key frame interval - used to reduce color distortions on Qualcomm platform. - private static final long QCOM_VP8_KEY_FRAME_INTERVAL_ANDROID_L_MS = 15000; - private static final long QCOM_VP8_KEY_FRAME_INTERVAL_ANDROID_M_MS = 20000; - private static final long QCOM_VP8_KEY_FRAME_INTERVAL_ANDROID_N_MS = 15000; - - // Active running encoder instance. Set in initEncode() (called from native code) - // and reset to null in release() call. - @Nullable private static MediaCodecVideoEncoder runningInstance; - @Nullable private static MediaCodecVideoEncoderErrorCallback errorCallback; - private static int codecErrors; - // List of disabled codec types - can be set from application. - private static Set hwEncoderDisabledTypes = new HashSet(); - @Nullable private static EglBase staticEglBase; - - @Nullable private Thread mediaCodecThread; - @Nullable private MediaCodec mediaCodec; - private ByteBuffer[] outputBuffers; - @Nullable private EglBase14 eglBase; - private int profile; - private int width; - private int height; - @Nullable private Surface inputSurface; - @Nullable private GlRectDrawer drawer; - - private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8"; - private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9"; - private static final String H264_MIME_TYPE = "video/avc"; - - private static final int VIDEO_AVCProfileHigh = 8; - private static final int VIDEO_AVCLevel3 = 0x100; - - // Type of bitrate adjustment for video encoder. - public enum BitrateAdjustmentType { - // No adjustment - video encoder has no known bitrate problem. - NO_ADJUSTMENT, - // Framerate based bitrate adjustment is required - HW encoder does not use frame - // timestamps to calculate frame bitrate budget and instead is relying on initial - // fps configuration assuming that all frames are coming at fixed initial frame rate. - FRAMERATE_ADJUSTMENT, - // Dynamic bitrate adjustment is required - HW encoder used frame timestamps, but actual - // bitrate deviates too much from the target value. - DYNAMIC_ADJUSTMENT - } - - // Should be in sync with webrtc::H264::Profile. - public static enum H264Profile { - CONSTRAINED_BASELINE(0), - BASELINE(1), - MAIN(2), - CONSTRAINED_HIGH(3), - HIGH(4); - - private final int value; - - H264Profile(int value) { - this.value = value; - } - - public int getValue() { - return value; - } - } - - // Class describing supported media codec properties. - private static class MediaCodecProperties { - public final String codecPrefix; - // Minimum Android SDK required for this codec to be used. - public final int minSdk; - // Flag if encoder implementation does not use frame timestamps to calculate frame bitrate - // budget and instead is relying on initial fps configuration assuming that all frames are - // coming at fixed initial frame rate. Bitrate adjustment is required for this case. - public final BitrateAdjustmentType bitrateAdjustmentType; - - MediaCodecProperties( - String codecPrefix, int minSdk, BitrateAdjustmentType bitrateAdjustmentType) { - this.codecPrefix = codecPrefix; - this.minSdk = minSdk; - this.bitrateAdjustmentType = bitrateAdjustmentType; - } - } - - /** - * Set EGL context used by HW encoding. The EGL context must be shared with the video capturer - * and any local render. - */ - public static void setEglContext(EglBase.Context eglContext) { - if (staticEglBase != null) { - Logging.w(TAG, "Egl context already set."); - staticEglBase.release(); - } - staticEglBase = EglBase.create(eglContext); - } - - /** Dispose the EGL context used by HW encoding. */ - public static void disposeEglContext() { - if (staticEglBase != null) { - staticEglBase.release(); - staticEglBase = null; - } - } - - @Nullable - static EglBase.Context getEglContext() { - return staticEglBase == null ? null : staticEglBase.getEglBaseContext(); - } - - // List of supported HW VP8 encoders. - private static final MediaCodecProperties qcomVp8HwProperties = new MediaCodecProperties( - "OMX.qcom.", Build.VERSION_CODES.KITKAT, BitrateAdjustmentType.NO_ADJUSTMENT); - private static final MediaCodecProperties exynosVp8HwProperties = new MediaCodecProperties( - "OMX.Exynos.", Build.VERSION_CODES.M, BitrateAdjustmentType.DYNAMIC_ADJUSTMENT); - private static final MediaCodecProperties intelVp8HwProperties = new MediaCodecProperties( - "OMX.Intel.", Build.VERSION_CODES.LOLLIPOP, BitrateAdjustmentType.NO_ADJUSTMENT); - private static MediaCodecProperties[] vp8HwList() { - final ArrayList supported_codecs = new ArrayList(); - supported_codecs.add(qcomVp8HwProperties); - supported_codecs.add(exynosVp8HwProperties); - if (PeerConnectionFactory.fieldTrialsFindFullName("WebRTC-IntelVP8").equals("Enabled")) { - supported_codecs.add(intelVp8HwProperties); - } - return supported_codecs.toArray(new MediaCodecProperties[supported_codecs.size()]); - } - - // List of supported HW VP9 encoders. - private static final MediaCodecProperties qcomVp9HwProperties = new MediaCodecProperties( - "OMX.qcom.", Build.VERSION_CODES.N, BitrateAdjustmentType.NO_ADJUSTMENT); - private static final MediaCodecProperties exynosVp9HwProperties = new MediaCodecProperties( - "OMX.Exynos.", Build.VERSION_CODES.N, BitrateAdjustmentType.FRAMERATE_ADJUSTMENT); - private static final MediaCodecProperties[] vp9HwList = - new MediaCodecProperties[] {qcomVp9HwProperties, exynosVp9HwProperties}; - - // List of supported HW H.264 encoders. - private static final MediaCodecProperties qcomH264HwProperties = new MediaCodecProperties( - "OMX.qcom.", Build.VERSION_CODES.KITKAT, BitrateAdjustmentType.NO_ADJUSTMENT); - private static final MediaCodecProperties exynosH264HwProperties = new MediaCodecProperties( - "OMX.Exynos.", Build.VERSION_CODES.LOLLIPOP, BitrateAdjustmentType.FRAMERATE_ADJUSTMENT); - private static final MediaCodecProperties mediatekH264HwProperties = new MediaCodecProperties( - "OMX.MTK.", Build.VERSION_CODES.O_MR1, BitrateAdjustmentType.FRAMERATE_ADJUSTMENT); - private static final MediaCodecProperties[] h264HwList() { - final ArrayList supported_codecs = new ArrayList(); - supported_codecs.add(qcomH264HwProperties); - supported_codecs.add(exynosH264HwProperties); - if (PeerConnectionFactory.fieldTrialsFindFullName("WebRTC-MediaTekH264").equals("Enabled")) { - supported_codecs.add(mediatekH264HwProperties); - } - return supported_codecs.toArray(new MediaCodecProperties[supported_codecs.size()]); - } - - // List of supported HW H.264 high profile encoders. - private static final MediaCodecProperties exynosH264HighProfileHwProperties = - new MediaCodecProperties( - "OMX.Exynos.", Build.VERSION_CODES.M, BitrateAdjustmentType.FRAMERATE_ADJUSTMENT); - private static final MediaCodecProperties[] h264HighProfileHwList = - new MediaCodecProperties[] {exynosH264HighProfileHwProperties}; - - // List of devices with poor H.264 encoder quality. - // HW H.264 encoder on below devices has poor bitrate control - actual - // bitrates deviates a lot from the target value. - private static final String[] H264_HW_EXCEPTION_MODELS = - new String[] {"SAMSUNG-SGH-I337", "Nexus 7", "Nexus 4"}; - - // Bitrate modes - should be in sync with OMX_VIDEO_CONTROLRATETYPE defined - // in OMX_Video.h - private static final int VIDEO_ControlRateConstant = 2; - // NV12 color format supported by QCOM codec, but not declared in MediaCodec - - // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h - private static final int COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04; - // Allowable color formats supported by codec - in order of preference. - private static final int[] supportedColorList = {CodecCapabilities.COLOR_FormatYUV420Planar, - CodecCapabilities.COLOR_FormatYUV420SemiPlanar, - CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, - COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m}; - private static final int[] supportedSurfaceColorList = {CodecCapabilities.COLOR_FormatSurface}; - @VideoCodecType private int type; - private int colorFormat; - - // Variables used for dynamic bitrate adjustment. - private BitrateAdjustmentType bitrateAdjustmentType = BitrateAdjustmentType.NO_ADJUSTMENT; - private double bitrateAccumulator; - private double bitrateAccumulatorMax; - private double bitrateObservationTimeMs; - private int bitrateAdjustmentScaleExp; - private int targetBitrateBps; - private int targetFps; - - // Interval in ms to force key frame generation. Used to reduce the time of color distortions - // happened sometime when using Qualcomm video encoder. - private long forcedKeyFrameMs; - private long lastKeyFrameMs; - - // SPS and PPS NALs (Config frame) for H.264. - @Nullable private ByteBuffer configData; - - // MediaCodec error handler - invoked when critical error happens which may prevent - // further use of media codec API. Now it means that one of media codec instances - // is hanging and can no longer be used in the next call. - public static interface MediaCodecVideoEncoderErrorCallback { - void onMediaCodecVideoEncoderCriticalError(int codecErrors); - } - - public static void setErrorCallback(MediaCodecVideoEncoderErrorCallback errorCallback) { - Logging.d(TAG, "Set error callback"); - MediaCodecVideoEncoder.errorCallback = errorCallback; - } - - // Functions to disable HW encoding - can be called from applications for platforms - // which have known HW decoding problems. - public static void disableVp8HwCodec() { - Logging.w(TAG, "VP8 encoding is disabled by application."); - hwEncoderDisabledTypes.add(VP8_MIME_TYPE); - } - - public static void disableVp9HwCodec() { - Logging.w(TAG, "VP9 encoding is disabled by application."); - hwEncoderDisabledTypes.add(VP9_MIME_TYPE); - } - - public static void disableH264HwCodec() { - Logging.w(TAG, "H.264 encoding is disabled by application."); - hwEncoderDisabledTypes.add(H264_MIME_TYPE); - } - - // Functions to query if HW encoding is supported. - public static boolean isVp8HwSupported() { - return !hwEncoderDisabledTypes.contains(VP8_MIME_TYPE) - && (findHwEncoder(VP8_MIME_TYPE, vp8HwList(), supportedColorList) != null); - } - - public static @Nullable EncoderProperties vp8HwEncoderProperties() { - if (hwEncoderDisabledTypes.contains(VP8_MIME_TYPE)) { - return null; - } else { - return findHwEncoder(VP8_MIME_TYPE, vp8HwList(), supportedColorList); - } - } - - public static boolean isVp9HwSupported() { - return !hwEncoderDisabledTypes.contains(VP9_MIME_TYPE) - && (findHwEncoder(VP9_MIME_TYPE, vp9HwList, supportedColorList) != null); - } - - public static boolean isH264HwSupported() { - return !hwEncoderDisabledTypes.contains(H264_MIME_TYPE) - && (findHwEncoder(H264_MIME_TYPE, h264HwList(), supportedColorList) != null); - } - - public static boolean isH264HighProfileHwSupported() { - return !hwEncoderDisabledTypes.contains(H264_MIME_TYPE) - && (findHwEncoder(H264_MIME_TYPE, h264HighProfileHwList, supportedColorList) != null); - } - - public static boolean isVp8HwSupportedUsingTextures() { - return !hwEncoderDisabledTypes.contains(VP8_MIME_TYPE) - && (findHwEncoder(VP8_MIME_TYPE, vp8HwList(), supportedSurfaceColorList) != null); - } - - public static boolean isVp9HwSupportedUsingTextures() { - return !hwEncoderDisabledTypes.contains(VP9_MIME_TYPE) - && (findHwEncoder(VP9_MIME_TYPE, vp9HwList, supportedSurfaceColorList) != null); - } - - public static boolean isH264HwSupportedUsingTextures() { - return !hwEncoderDisabledTypes.contains(H264_MIME_TYPE) - && (findHwEncoder(H264_MIME_TYPE, h264HwList(), supportedSurfaceColorList) != null); - } - - // Helper struct for findHwEncoder() below. - public static class EncoderProperties { - public EncoderProperties( - String codecName, int colorFormat, BitrateAdjustmentType bitrateAdjustmentType) { - this.codecName = codecName; - this.colorFormat = colorFormat; - this.bitrateAdjustmentType = bitrateAdjustmentType; - } - public final String codecName; // OpenMax component name for HW codec. - public final int colorFormat; // Color format supported by codec. - public final BitrateAdjustmentType bitrateAdjustmentType; // Bitrate adjustment type - } - - private static @Nullable EncoderProperties findHwEncoder( - String mime, MediaCodecProperties[] supportedHwCodecProperties, int[] colorList) { - // MediaCodec.setParameters is missing for JB and below, so bitrate - // can not be adjusted dynamically. - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { - return null; - } - - // Check if device is in H.264 exception list. - if (mime.equals(H264_MIME_TYPE)) { - List exceptionModels = Arrays.asList(H264_HW_EXCEPTION_MODELS); - if (exceptionModels.contains(Build.MODEL)) { - Logging.w(TAG, "Model: " + Build.MODEL + " has black listed H.264 encoder."); - return null; - } - } - - for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { - MediaCodecInfo info = null; - try { - info = MediaCodecList.getCodecInfoAt(i); - } catch (IllegalArgumentException e) { - Logging.e(TAG, "Cannot retrieve encoder codec info", e); - } - if (info == null || !info.isEncoder()) { - continue; - } - String name = null; - for (String mimeType : info.getSupportedTypes()) { - if (mimeType.equals(mime)) { - name = info.getName(); - break; - } - } - if (name == null) { - continue; // No HW support in this codec; try the next one. - } - Logging.v(TAG, "Found candidate encoder " + name); - - // Check if this is supported HW encoder. - boolean supportedCodec = false; - BitrateAdjustmentType bitrateAdjustmentType = BitrateAdjustmentType.NO_ADJUSTMENT; - for (MediaCodecProperties codecProperties : supportedHwCodecProperties) { - if (name.startsWith(codecProperties.codecPrefix)) { - if (Build.VERSION.SDK_INT < codecProperties.minSdk) { - Logging.w( - TAG, "Codec " + name + " is disabled due to SDK version " + Build.VERSION.SDK_INT); - continue; - } - if (codecProperties.bitrateAdjustmentType != BitrateAdjustmentType.NO_ADJUSTMENT) { - bitrateAdjustmentType = codecProperties.bitrateAdjustmentType; - Logging.w( - TAG, "Codec " + name + " requires bitrate adjustment: " + bitrateAdjustmentType); - } - supportedCodec = true; - break; - } - } - if (!supportedCodec) { - continue; - } - - // Check if HW codec supports known color format. - CodecCapabilities capabilities; - try { - capabilities = info.getCapabilitiesForType(mime); - } catch (IllegalArgumentException e) { - Logging.e(TAG, "Cannot retrieve encoder capabilities", e); - continue; - } - for (int colorFormat : capabilities.colorFormats) { - Logging.v(TAG, " Color: 0x" + Integer.toHexString(colorFormat)); - } - - for (int supportedColorFormat : colorList) { - for (int codecColorFormat : capabilities.colorFormats) { - if (codecColorFormat == supportedColorFormat) { - // Found supported HW encoder. - Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name + ". Color: 0x" - + Integer.toHexString(codecColorFormat) + ". Bitrate adjustment: " - + bitrateAdjustmentType); - return new EncoderProperties(name, codecColorFormat, bitrateAdjustmentType); - } - } - } - } - return null; // No HW encoder. - } - - @CalledByNative - MediaCodecVideoEncoder() {} - - private void checkOnMediaCodecThread() { - if (mediaCodecThread.getId() != Thread.currentThread().getId()) { - throw new RuntimeException("MediaCodecVideoEncoder previously operated on " + mediaCodecThread - + " but is now called on " + Thread.currentThread()); - } - } - - public static void printStackTrace() { - if (runningInstance != null && runningInstance.mediaCodecThread != null) { - StackTraceElement[] mediaCodecStackTraces = runningInstance.mediaCodecThread.getStackTrace(); - if (mediaCodecStackTraces.length > 0) { - Logging.d(TAG, "MediaCodecVideoEncoder stacks trace:"); - for (StackTraceElement stackTrace : mediaCodecStackTraces) { - Logging.d(TAG, stackTrace.toString()); - } - } - } - } - - static @Nullable MediaCodec createByCodecName(String codecName) { - try { - // In the L-SDK this call can throw IOException so in order to work in - // both cases catch an exception. - return MediaCodec.createByCodecName(codecName); - } catch (Exception e) { - return null; - } - } - - @CalledByNativeUnchecked - boolean initEncode(@VideoCodecType int type, int profile, int width, int height, int kbps, - int fps, boolean useSurface) { - Logging.d(TAG, - "Java initEncode: " + type + ". Profile: " + profile + " : " + width + " x " + height - + ". @ " + kbps + " kbps. Fps: " + fps + ". Encode from texture : " + useSurface); - - this.profile = profile; - this.width = width; - this.height = height; - if (mediaCodecThread != null) { - throw new RuntimeException("Forgot to release()?"); - } - EncoderProperties properties = null; - String mime = null; - int keyFrameIntervalSec = 0; - boolean configureH264HighProfile = false; - if (type == VideoCodecType.VIDEO_CODEC_VP8) { - mime = VP8_MIME_TYPE; - properties = findHwEncoder( - VP8_MIME_TYPE, vp8HwList(), useSurface ? supportedSurfaceColorList : supportedColorList); - keyFrameIntervalSec = 100; - } else if (type == VideoCodecType.VIDEO_CODEC_VP9) { - mime = VP9_MIME_TYPE; - properties = findHwEncoder( - VP9_MIME_TYPE, vp9HwList, useSurface ? supportedSurfaceColorList : supportedColorList); - keyFrameIntervalSec = 100; - } else if (type == VideoCodecType.VIDEO_CODEC_H264) { - mime = H264_MIME_TYPE; - properties = findHwEncoder(H264_MIME_TYPE, h264HwList(), - useSurface ? supportedSurfaceColorList : supportedColorList); - if (profile == H264Profile.CONSTRAINED_HIGH.getValue()) { - EncoderProperties h264HighProfileProperties = findHwEncoder(H264_MIME_TYPE, - h264HighProfileHwList, useSurface ? supportedSurfaceColorList : supportedColorList); - if (h264HighProfileProperties != null) { - Logging.d(TAG, "High profile H.264 encoder supported."); - configureH264HighProfile = true; - } else { - Logging.d(TAG, "High profile H.264 encoder requested, but not supported. Use baseline."); - } - } - keyFrameIntervalSec = 20; - } else { - throw new RuntimeException("initEncode: Non-supported codec " + type); - } - if (properties == null) { - throw new RuntimeException("Can not find HW encoder for " + type); - } - runningInstance = this; // Encoder is now running and can be queried for stack traces. - colorFormat = properties.colorFormat; - bitrateAdjustmentType = properties.bitrateAdjustmentType; - if (bitrateAdjustmentType == BitrateAdjustmentType.FRAMERATE_ADJUSTMENT) { - fps = BITRATE_ADJUSTMENT_FPS; - } else { - fps = Math.min(fps, MAXIMUM_INITIAL_FPS); - } - - forcedKeyFrameMs = 0; - lastKeyFrameMs = -1; - if (type == VideoCodecType.VIDEO_CODEC_VP8 - && properties.codecName.startsWith(qcomVp8HwProperties.codecPrefix)) { - if (Build.VERSION.SDK_INT == Build.VERSION_CODES.LOLLIPOP - || Build.VERSION.SDK_INT == Build.VERSION_CODES.LOLLIPOP_MR1) { - forcedKeyFrameMs = QCOM_VP8_KEY_FRAME_INTERVAL_ANDROID_L_MS; - } else if (Build.VERSION.SDK_INT == Build.VERSION_CODES.M) { - forcedKeyFrameMs = QCOM_VP8_KEY_FRAME_INTERVAL_ANDROID_M_MS; - } else if (Build.VERSION.SDK_INT > Build.VERSION_CODES.M) { - forcedKeyFrameMs = QCOM_VP8_KEY_FRAME_INTERVAL_ANDROID_N_MS; - } - } - - Logging.d(TAG, "Color format: " + colorFormat + ". Bitrate adjustment: " + bitrateAdjustmentType - + ". Key frame interval: " + forcedKeyFrameMs + " . Initial fps: " + fps); - targetBitrateBps = 1000 * kbps; - targetFps = fps; - bitrateAccumulatorMax = targetBitrateBps / 8.0; - bitrateAccumulator = 0; - bitrateObservationTimeMs = 0; - bitrateAdjustmentScaleExp = 0; - - mediaCodecThread = Thread.currentThread(); - try { - MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); - format.setInteger(MediaFormat.KEY_BIT_RATE, targetBitrateBps); - format.setInteger("bitrate-mode", VIDEO_ControlRateConstant); - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); - format.setInteger(MediaFormat.KEY_FRAME_RATE, targetFps); - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec); - if (configureH264HighProfile) { - format.setInteger("profile", VIDEO_AVCProfileHigh); - format.setInteger("level", VIDEO_AVCLevel3); - } - Logging.d(TAG, " Format: " + format); - mediaCodec = createByCodecName(properties.codecName); - this.type = type; - if (mediaCodec == null) { - Logging.e(TAG, "Can not create media encoder"); - release(); - return false; - } - mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - - if (useSurface) { - eglBase = - EglBase.createEgl14((EglBase14.Context) getEglContext(), EglBase.CONFIG_RECORDABLE); - // Create an input surface and keep a reference since we must release the surface when done. - inputSurface = mediaCodec.createInputSurface(); - eglBase.createSurface(inputSurface); - drawer = new GlRectDrawer(); - } - mediaCodec.start(); - outputBuffers = mediaCodec.getOutputBuffers(); - Logging.d(TAG, "Output buffers: " + outputBuffers.length); - - } catch (IllegalStateException e) { - Logging.e(TAG, "initEncode failed", e); - release(); - return false; - } - return true; - } - - @CalledByNativeUnchecked - ByteBuffer[] getInputBuffers() { - ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers(); - Logging.d(TAG, "Input buffers: " + inputBuffers.length); - return inputBuffers; - } - - void checkKeyFrameRequired(boolean requestedKeyFrame, long presentationTimestampUs) { - long presentationTimestampMs = (presentationTimestampUs + 500) / 1000; - if (lastKeyFrameMs < 0) { - lastKeyFrameMs = presentationTimestampMs; - } - boolean forcedKeyFrame = false; - if (!requestedKeyFrame && forcedKeyFrameMs > 0 - && presentationTimestampMs > lastKeyFrameMs + forcedKeyFrameMs) { - forcedKeyFrame = true; - } - if (requestedKeyFrame || forcedKeyFrame) { - // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could - // indicate this in queueInputBuffer() below and guarantee _this_ frame - // be encoded as a key frame, but sadly that flag is ignored. Instead, - // we request a key frame "soon". - if (requestedKeyFrame) { - Logging.d(TAG, "Sync frame request"); - } else { - Logging.d(TAG, "Sync frame forced"); - } - Bundle b = new Bundle(); - b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); - mediaCodec.setParameters(b); - lastKeyFrameMs = presentationTimestampMs; - } - } - - @CalledByNativeUnchecked - boolean encodeBuffer( - boolean isKeyframe, int inputBuffer, int size, long presentationTimestampUs) { - checkOnMediaCodecThread(); - try { - checkKeyFrameRequired(isKeyframe, presentationTimestampUs); - mediaCodec.queueInputBuffer(inputBuffer, 0, size, presentationTimestampUs, 0); - return true; - } catch (IllegalStateException e) { - Logging.e(TAG, "encodeBuffer failed", e); - return false; - } - } - - /** - * Encodes a new style VideoFrame. |bufferIndex| is -1 if we are not encoding in surface mode. - */ - @CalledByNativeUnchecked - boolean encodeFrame(long nativeEncoder, boolean isKeyframe, VideoFrame frame, int bufferIndex, - long presentationTimestampUs) { - checkOnMediaCodecThread(); - try { - checkKeyFrameRequired(isKeyframe, presentationTimestampUs); - - VideoFrame.Buffer buffer = frame.getBuffer(); - if (buffer instanceof VideoFrame.TextureBuffer) { - VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) buffer; - eglBase.makeCurrent(); - // TODO(perkj): glClear() shouldn't be necessary since every pixel is covered anyway, - // but it's a workaround for bug webrtc:5147. - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - VideoFrameDrawer.drawTexture(drawer, textureBuffer, new Matrix() /* renderMatrix */, width, - height, 0 /* viewportX */, 0 /* viewportY */, width, height); - eglBase.swapBuffers(TimeUnit.MICROSECONDS.toNanos(presentationTimestampUs)); - } else { - VideoFrame.I420Buffer i420Buffer = buffer.toI420(); - final int chromaHeight = (height + 1) / 2; - final ByteBuffer dataY = i420Buffer.getDataY(); - final ByteBuffer dataU = i420Buffer.getDataU(); - final ByteBuffer dataV = i420Buffer.getDataV(); - final int strideY = i420Buffer.getStrideY(); - final int strideU = i420Buffer.getStrideU(); - final int strideV = i420Buffer.getStrideV(); - if (dataY.capacity() < strideY * height) { - throw new RuntimeException("Y-plane buffer size too small."); - } - if (dataU.capacity() < strideU * chromaHeight) { - throw new RuntimeException("U-plane buffer size too small."); - } - if (dataV.capacity() < strideV * chromaHeight) { - throw new RuntimeException("V-plane buffer size too small."); - } - nativeFillInputBuffer( - nativeEncoder, bufferIndex, dataY, strideY, dataU, strideU, dataV, strideV); - i420Buffer.release(); - // I420 consists of one full-resolution and two half-resolution planes. - // 1 + 1 / 4 + 1 / 4 = 3 / 2 - int yuvSize = width * height * 3 / 2; - mediaCodec.queueInputBuffer(bufferIndex, 0, yuvSize, presentationTimestampUs, 0); - } - return true; - } catch (RuntimeException e) { - Logging.e(TAG, "encodeFrame failed", e); - return false; - } - } - - @CalledByNativeUnchecked - void release() { - Logging.d(TAG, "Java releaseEncoder"); - checkOnMediaCodecThread(); - - class CaughtException { - Exception e; - } - final CaughtException caughtException = new CaughtException(); - boolean stopHung = false; - - if (mediaCodec != null) { - // Run Mediacodec stop() and release() on separate thread since sometime - // Mediacodec.stop() may hang. - final CountDownLatch releaseDone = new CountDownLatch(1); - - Runnable runMediaCodecRelease = new Runnable() { - @Override - public void run() { - Logging.d(TAG, "Java releaseEncoder on release thread"); - try { - mediaCodec.stop(); - } catch (Exception e) { - Logging.e(TAG, "Media encoder stop failed", e); - } - try { - mediaCodec.release(); - } catch (Exception e) { - Logging.e(TAG, "Media encoder release failed", e); - caughtException.e = e; - } - Logging.d(TAG, "Java releaseEncoder on release thread done"); - - releaseDone.countDown(); - } - }; - new Thread(runMediaCodecRelease).start(); - - if (!ThreadUtils.awaitUninterruptibly(releaseDone, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { - Logging.e(TAG, "Media encoder release timeout"); - stopHung = true; - } - - mediaCodec = null; - } - - mediaCodecThread = null; - if (drawer != null) { - drawer.release(); - drawer = null; - } - if (eglBase != null) { - eglBase.release(); - eglBase = null; - } - if (inputSurface != null) { - inputSurface.release(); - inputSurface = null; - } - runningInstance = null; - - if (stopHung) { - codecErrors++; - if (errorCallback != null) { - Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors); - errorCallback.onMediaCodecVideoEncoderCriticalError(codecErrors); - } - throw new RuntimeException("Media encoder release timeout."); - } - - // Re-throw any runtime exception caught inside the other thread. Since this is an invoke, add - // stack trace for the waiting thread as well. - if (caughtException.e != null) { - final RuntimeException runtimeException = new RuntimeException(caughtException.e); - runtimeException.setStackTrace(ThreadUtils.concatStackTraces( - caughtException.e.getStackTrace(), runtimeException.getStackTrace())); - throw runtimeException; - } - - Logging.d(TAG, "Java releaseEncoder done"); - } - - @CalledByNativeUnchecked - private boolean setRates(int kbps, int frameRate) { - checkOnMediaCodecThread(); - - int codecBitrateBps = 1000 * kbps; - if (bitrateAdjustmentType == BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) { - bitrateAccumulatorMax = codecBitrateBps / 8.0; - if (targetBitrateBps > 0 && codecBitrateBps < targetBitrateBps) { - // Rescale the accumulator level if the accumulator max decreases - bitrateAccumulator = bitrateAccumulator * codecBitrateBps / targetBitrateBps; - } - } - targetBitrateBps = codecBitrateBps; - targetFps = frameRate; - - // Adjust actual encoder bitrate based on bitrate adjustment type. - if (bitrateAdjustmentType == BitrateAdjustmentType.FRAMERATE_ADJUSTMENT && targetFps > 0) { - codecBitrateBps = BITRATE_ADJUSTMENT_FPS * targetBitrateBps / targetFps; - Logging.v(TAG, - "setRates: " + kbps + " -> " + (codecBitrateBps / 1000) + " kbps. Fps: " + targetFps); - } else if (bitrateAdjustmentType == BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) { - Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + targetFps + ". ExpScale: " - + bitrateAdjustmentScaleExp); - if (bitrateAdjustmentScaleExp != 0) { - codecBitrateBps = (int) (codecBitrateBps * getBitrateScale(bitrateAdjustmentScaleExp)); - } - } else { - Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + targetFps); - } - - try { - Bundle params = new Bundle(); - params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, codecBitrateBps); - mediaCodec.setParameters(params); - return true; - } catch (IllegalStateException e) { - Logging.e(TAG, "setRates failed", e); - return false; - } - } - - // Dequeue an input buffer and return its index, -1 if no input buffer is - // available, or -2 if the codec is no longer operative. - @CalledByNativeUnchecked - int dequeueInputBuffer() { - checkOnMediaCodecThread(); - try { - return mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT); - } catch (IllegalStateException e) { - Logging.e(TAG, "dequeueIntputBuffer failed", e); - return -2; - } - } - - // Helper struct for dequeueOutputBuffer() below. - static class OutputBufferInfo { - public OutputBufferInfo( - int index, ByteBuffer buffer, boolean isKeyFrame, long presentationTimestampUs) { - this.index = index; - this.buffer = buffer; - this.isKeyFrame = isKeyFrame; - this.presentationTimestampUs = presentationTimestampUs; - } - - public final int index; - public final ByteBuffer buffer; - public final boolean isKeyFrame; - public final long presentationTimestampUs; - - @CalledByNative("OutputBufferInfo") - int getIndex() { - return index; - } - - @CalledByNative("OutputBufferInfo") - ByteBuffer getBuffer() { - return buffer; - } - - @CalledByNative("OutputBufferInfo") - boolean isKeyFrame() { - return isKeyFrame; - } - - @CalledByNative("OutputBufferInfo") - long getPresentationTimestampUs() { - return presentationTimestampUs; - } - } - - // Dequeue and return an output buffer, or null if no output is ready. Return - // a fake OutputBufferInfo with index -1 if the codec is no longer operable. - @Nullable - @CalledByNativeUnchecked - OutputBufferInfo dequeueOutputBuffer() { - checkOnMediaCodecThread(); - try { - MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); - int result = mediaCodec.dequeueOutputBuffer(info, DEQUEUE_TIMEOUT); - // Check if this is config frame and save configuration data. - if (result >= 0) { - boolean isConfigFrame = (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; - if (isConfigFrame) { - Logging.d(TAG, "Config frame generated. Offset: " + info.offset + ". Size: " + info.size); - configData = ByteBuffer.allocateDirect(info.size); - outputBuffers[result].position(info.offset); - outputBuffers[result].limit(info.offset + info.size); - configData.put(outputBuffers[result]); - // Log few SPS header bytes to check profile and level. - String spsData = ""; - for (int i = 0; i < (info.size < 8 ? info.size : 8); i++) { - spsData += Integer.toHexString(configData.get(i) & 0xff) + " "; - } - Logging.d(TAG, spsData); - // Release buffer back. - mediaCodec.releaseOutputBuffer(result, false); - // Query next output. - result = mediaCodec.dequeueOutputBuffer(info, DEQUEUE_TIMEOUT); - } - } - if (result >= 0) { - // MediaCodec doesn't care about Buffer position/remaining/etc so we can - // mess with them to get a slice and avoid having to pass extra - // (BufferInfo-related) parameters back to C++. - ByteBuffer outputBuffer = outputBuffers[result].duplicate(); - outputBuffer.position(info.offset); - outputBuffer.limit(info.offset + info.size); - reportEncodedFrame(info.size); - - // Check key frame flag. - boolean isKeyFrame = (info.flags & MediaCodec.BUFFER_FLAG_SYNC_FRAME) != 0; - if (isKeyFrame) { - Logging.d(TAG, "Sync frame generated"); - } - if (isKeyFrame && type == VideoCodecType.VIDEO_CODEC_H264) { - Logging.d(TAG, "Appending config frame of size " + configData.capacity() - + " to output buffer with offset " + info.offset + ", size " + info.size); - // For H.264 key frame append SPS and PPS NALs at the start - ByteBuffer keyFrameBuffer = ByteBuffer.allocateDirect(configData.capacity() + info.size); - configData.rewind(); - keyFrameBuffer.put(configData); - keyFrameBuffer.put(outputBuffer); - keyFrameBuffer.position(0); - return new OutputBufferInfo(result, keyFrameBuffer, isKeyFrame, info.presentationTimeUs); - } else { - return new OutputBufferInfo( - result, outputBuffer.slice(), isKeyFrame, info.presentationTimeUs); - } - } else if (result == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { - outputBuffers = mediaCodec.getOutputBuffers(); - return dequeueOutputBuffer(); - } else if (result == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - return dequeueOutputBuffer(); - } else if (result == MediaCodec.INFO_TRY_AGAIN_LATER) { - return null; - } - throw new RuntimeException("dequeueOutputBuffer: " + result); - } catch (IllegalStateException e) { - Logging.e(TAG, "dequeueOutputBuffer failed", e); - return new OutputBufferInfo(-1, null, false, -1); - } - } - - private double getBitrateScale(int bitrateAdjustmentScaleExp) { - return Math.pow(BITRATE_CORRECTION_MAX_SCALE, - (double) bitrateAdjustmentScaleExp / BITRATE_CORRECTION_STEPS); - } - - private void reportEncodedFrame(int size) { - if (targetFps == 0 || bitrateAdjustmentType != BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) { - return; - } - - // Accumulate the difference between actial and expected frame sizes. - double expectedBytesPerFrame = targetBitrateBps / (8.0 * targetFps); - bitrateAccumulator += (size - expectedBytesPerFrame); - bitrateObservationTimeMs += 1000.0 / targetFps; - - // Put a cap on the accumulator, i.e., don't let it grow beyond some level to avoid - // using too old data for bitrate adjustment. - double bitrateAccumulatorCap = BITRATE_CORRECTION_SEC * bitrateAccumulatorMax; - bitrateAccumulator = Math.min(bitrateAccumulator, bitrateAccumulatorCap); - bitrateAccumulator = Math.max(bitrateAccumulator, -bitrateAccumulatorCap); - - // Do bitrate adjustment every 3 seconds if actual encoder bitrate deviates too much - // form the target value. - if (bitrateObservationTimeMs > 1000 * BITRATE_CORRECTION_SEC) { - Logging.d(TAG, "Acc: " + (int) bitrateAccumulator + ". Max: " + (int) bitrateAccumulatorMax - + ". ExpScale: " + bitrateAdjustmentScaleExp); - boolean bitrateAdjustmentScaleChanged = false; - if (bitrateAccumulator > bitrateAccumulatorMax) { - // Encoder generates too high bitrate - need to reduce the scale. - int bitrateAdjustmentInc = (int) (bitrateAccumulator / bitrateAccumulatorMax + 0.5); - bitrateAdjustmentScaleExp -= bitrateAdjustmentInc; - bitrateAccumulator = bitrateAccumulatorMax; - bitrateAdjustmentScaleChanged = true; - } else if (bitrateAccumulator < -bitrateAccumulatorMax) { - // Encoder generates too low bitrate - need to increase the scale. - int bitrateAdjustmentInc = (int) (-bitrateAccumulator / bitrateAccumulatorMax + 0.5); - bitrateAdjustmentScaleExp += bitrateAdjustmentInc; - bitrateAccumulator = -bitrateAccumulatorMax; - bitrateAdjustmentScaleChanged = true; - } - if (bitrateAdjustmentScaleChanged) { - bitrateAdjustmentScaleExp = Math.min(bitrateAdjustmentScaleExp, BITRATE_CORRECTION_STEPS); - bitrateAdjustmentScaleExp = Math.max(bitrateAdjustmentScaleExp, -BITRATE_CORRECTION_STEPS); - Logging.d(TAG, "Adjusting bitrate scale to " + bitrateAdjustmentScaleExp + ". Value: " - + getBitrateScale(bitrateAdjustmentScaleExp)); - setRates(targetBitrateBps / 1000, targetFps); - } - bitrateObservationTimeMs = 0; - } - } - - // Release a dequeued output buffer back to the codec for re-use. Return - // false if the codec is no longer operable. - @CalledByNativeUnchecked - boolean releaseOutputBuffer(int index) { - checkOnMediaCodecThread(); - try { - mediaCodec.releaseOutputBuffer(index, false); - return true; - } catch (IllegalStateException e) { - Logging.e(TAG, "releaseOutputBuffer failed", e); - return false; - } - } - - @CalledByNative - int getColorFormat() { - return colorFormat; - } - - @CalledByNative - static boolean isTextureBuffer(VideoFrame.Buffer buffer) { - return buffer instanceof VideoFrame.TextureBuffer; - } - - /** Fills an inputBuffer with the given index with data from the byte buffers. */ - private static native void nativeFillInputBuffer(long encoder, int inputBuffer, ByteBuffer dataY, - int strideY, ByteBuffer dataU, int strideU, ByteBuffer dataV, int strideV); - private static native long nativeCreateEncoder(VideoCodecInfo info, boolean hasEgl14Context); -} diff --git a/sdk/android/api/org/webrtc/PeerConnectionFactory.java b/sdk/android/api/org/webrtc/PeerConnectionFactory.java index 683ac88364..decdc0cc42 100644 --- a/sdk/android/api/org/webrtc/PeerConnectionFactory.java +++ b/sdk/android/api/org/webrtc/PeerConnectionFactory.java @@ -501,8 +501,6 @@ public class PeerConnectionFactory { networkThread = null; workerThread = null; signalingThread = null; - MediaCodecVideoEncoder.disposeEglContext(); - MediaCodecVideoDecoder.disposeEglContext(); nativeFactory = 0; } diff --git a/sdk/android/instrumentationtests/src/org/webrtc/MediaCodecVideoEncoderTest.java b/sdk/android/instrumentationtests/src/org/webrtc/MediaCodecVideoEncoderTest.java deleted file mode 100644 index 7182f481fb..0000000000 --- a/sdk/android/instrumentationtests/src/org/webrtc/MediaCodecVideoEncoderTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -package org.webrtc; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import android.annotation.TargetApi; -import android.os.Build; -import android.support.test.filters.SmallTest; -import android.util.Log; -import java.nio.ByteBuffer; -import org.chromium.base.test.BaseJUnit4ClassRunner; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.webrtc.MediaCodecVideoEncoder.OutputBufferInfo; - -@TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH_MR1) -@RunWith(BaseJUnit4ClassRunner.class) -public class MediaCodecVideoEncoderTest { - final static String TAG = "MCVideoEncoderTest"; - final static int profile = MediaCodecVideoEncoder.H264Profile.CONSTRAINED_BASELINE.getValue(); - - @Test - @SmallTest - public void testInitializeUsingByteBuffer() { - if (!MediaCodecVideoEncoder.isVp8HwSupported()) { - Log.i(TAG, "Hardware does not support VP8 encoding, skipping testInitReleaseUsingByteBuffer"); - return; - } - MediaCodecVideoEncoder encoder = new MediaCodecVideoEncoder(); - assertTrue(encoder.initEncode( - VideoCodecType.VIDEO_CODEC_VP8, profile, 640, 480, 300, 30, /* useSurface= */ false)); - encoder.release(); - } - - @Test - @SmallTest - public void testInitilizeUsingTextures() { - if (!MediaCodecVideoEncoder.isVp8HwSupportedUsingTextures()) { - Log.i(TAG, "hardware does not support VP8 encoding, skipping testEncoderUsingTextures"); - return; - } - EglBase14 eglBase = EglBase.createEgl14(EglBase.CONFIG_PLAIN); - MediaCodecVideoEncoder.setEglContext(eglBase.getEglBaseContext()); - MediaCodecVideoEncoder encoder = new MediaCodecVideoEncoder(); - assertTrue(encoder.initEncode( - VideoCodecType.VIDEO_CODEC_VP8, profile, 640, 480, 300, 30, /* useSurface= */ true)); - encoder.release(); - MediaCodecVideoEncoder.disposeEglContext(); - eglBase.release(); - } - - @Test - @SmallTest - public void testInitializeUsingByteBufferReInitilizeUsingTextures() { - if (!MediaCodecVideoEncoder.isVp8HwSupportedUsingTextures()) { - Log.i(TAG, "hardware does not support VP8 encoding, skipping testEncoderUsingTextures"); - return; - } - MediaCodecVideoEncoder encoder = new MediaCodecVideoEncoder(); - assertTrue(encoder.initEncode( - VideoCodecType.VIDEO_CODEC_VP8, profile, 640, 480, 300, 30, /* useSurface= */ false)); - encoder.release(); - EglBase14 eglBase = EglBase.createEgl14(EglBase.CONFIG_PLAIN); - MediaCodecVideoEncoder.setEglContext(eglBase.getEglBaseContext()); - assertTrue(encoder.initEncode( - VideoCodecType.VIDEO_CODEC_VP8, profile, 640, 480, 300, 30, /* useSurface= */ true)); - encoder.release(); - MediaCodecVideoEncoder.disposeEglContext(); - eglBase.release(); - } - - @Test - @SmallTest - public void testEncoderUsingByteBuffer() throws InterruptedException { - if (!MediaCodecVideoEncoder.isVp8HwSupported()) { - Log.i(TAG, "Hardware does not support VP8 encoding, skipping testEncoderUsingByteBuffer"); - return; - } - - final int width = 640; - final int height = 480; - final int min_size = width * height * 3 / 2; - final long presentationTimestampUs = 2; - - MediaCodecVideoEncoder encoder = new MediaCodecVideoEncoder(); - - assertTrue(encoder.initEncode( - VideoCodecType.VIDEO_CODEC_VP8, profile, width, height, 300, 30, /* useSurface= */ false)); - ByteBuffer[] inputBuffers = encoder.getInputBuffers(); - assertNotNull(inputBuffers); - assertTrue(min_size <= inputBuffers[0].capacity()); - - int bufferIndex; - do { - Thread.sleep(10); - bufferIndex = encoder.dequeueInputBuffer(); - } while (bufferIndex == -1); // |-1| is returned when there is no buffer available yet. - - assertTrue(bufferIndex >= 0); - assertTrue(bufferIndex < inputBuffers.length); - assertTrue(encoder.encodeBuffer(true, bufferIndex, min_size, presentationTimestampUs)); - - OutputBufferInfo info; - do { - info = encoder.dequeueOutputBuffer(); - Thread.sleep(10); - } while (info == null); - assertTrue(info.index >= 0); - assertEquals(presentationTimestampUs, info.presentationTimestampUs); - assertTrue(info.buffer.capacity() > 0); - encoder.releaseOutputBuffer(info.index); - - encoder.release(); - } -} diff --git a/sdk/android/src/jni/android_media_decoder.cc b/sdk/android/src/jni/android_media_decoder.cc deleted file mode 100644 index 7c731ce7a4..0000000000 --- a/sdk/android/src/jni/android_media_decoder.cc +++ /dev/null @@ -1,786 +0,0 @@ -/* - * Copyright 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include -#include -#include -#include - -#include "api/scoped_refptr.h" -#include "api/video_codecs/sdp_video_format.h" -#include "common_video/h264/h264_bitstream_parser.h" -#include "common_video/include/i420_buffer_pool.h" -#include "media/base/media_constants.h" -#include "modules/video_coding/include/video_codec_interface.h" -#include "modules/video_coding/utility/vp8_header_parser.h" -#include "rtc_base/bind.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "rtc_base/numerics/safe_conversions.h" -#include "rtc_base/thread.h" -#include "rtc_base/time_utils.h" -#include "sdk/android/generated_video_jni/MediaCodecVideoDecoder_jni.h" -#include "sdk/android/native_api/jni/java_types.h" -#include "sdk/android/src/jni/android_media_codec_common.h" -#include "sdk/android/src/jni/video_frame.h" -#include "third_party/libyuv/include/libyuv/convert.h" -#include "third_party/libyuv/include/libyuv/planar_functions.h" -#include "third_party/libyuv/include/libyuv/video_common.h" - -using rtc::Bind; -using rtc::ThreadManager; -namespace webrtc { -namespace jni { - -// Logging macros. -#define TAG_DECODER "MediaCodecVideoDecoder" -#ifdef TRACK_BUFFER_TIMING -#define ALOGV(...) \ - __android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__) -#else -#define ALOGV(...) -#endif -#define ALOGD RTC_LOG_TAG(rtc::LS_INFO, TAG_DECODER) -#define ALOGW RTC_LOG_TAG(rtc::LS_WARNING, TAG_DECODER) -#define ALOGE RTC_LOG_TAG(rtc::LS_ERROR, TAG_DECODER) - -enum { kMaxWarningLogFrames = 2 }; - -class MediaCodecVideoDecoder : public VideoDecoder, public rtc::MessageHandler { - public: - explicit MediaCodecVideoDecoder(JNIEnv* jni, - VideoCodecType codecType, - bool use_surface); - ~MediaCodecVideoDecoder() override; - - int32_t InitDecode(const VideoCodec* codecSettings, - int32_t numberOfCores) override; - - int32_t Decode(const EncodedImage& inputImage, - bool missingFrames, - int64_t renderTimeMs = -1) override; - - int32_t RegisterDecodeCompleteCallback( - DecodedImageCallback* callback) override; - - int32_t Release() override; - - bool PrefersLateDecoding() const override { return true; } - - // rtc::MessageHandler implementation. - void OnMessage(rtc::Message* msg) override; - - const char* ImplementationName() const override; - - private: - // CHECK-fail if not running on |codec_thread_|. - void CheckOnCodecThread(); - - int32_t InitDecodeOnCodecThread(); - int32_t ResetDecodeOnCodecThread(); - int32_t ReleaseOnCodecThread(); - int32_t DecodeOnCodecThread(const EncodedImage& inputImage); - // Deliver any outputs pending in the MediaCodec to our |callback_| and return - // true on success. - bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us); - int32_t ProcessHWErrorOnCodecThread(); - void EnableFrameLogOnWarning(); - void ResetVariables(); - - // Type of video codec. - VideoCodecType codecType_; - - bool key_frame_required_; - bool inited_; - bool sw_fallback_required_; - const bool use_surface_; - VideoCodec codec_; - I420BufferPool decoded_frame_pool_; - DecodedImageCallback* callback_; - int frames_received_; // Number of frames received by decoder. - int frames_decoded_; // Number of frames decoded by decoder. - // Number of decoded frames for which log information is displayed. - int frames_decoded_logged_; - int64_t start_time_ms_; // Start time for statistics. - int current_frames_; // Number of frames in the current statistics interval. - int current_bytes_; // Encoded bytes in the current statistics interval. - int current_decoding_time_ms_; // Overall decoding time in the current second - int current_delay_time_ms_; // Overall delay time in the current second. - int32_t max_pending_frames_; // Maximum number of pending input frames. - H264BitstreamParser h264_bitstream_parser_; - std::deque> pending_frame_qps_; - - // State that is constant for the lifetime of this object once the ctor - // returns. - std::unique_ptr - codec_thread_; // Thread on which to operate MediaCodec. - ScopedJavaGlobalRef j_media_codec_video_decoder_; - - // Global references; must be deleted in Release(). - std::vector> input_buffers_; -}; - -MediaCodecVideoDecoder::MediaCodecVideoDecoder(JNIEnv* jni, - VideoCodecType codecType, - bool use_surface) - : codecType_(codecType), - key_frame_required_(true), - inited_(false), - sw_fallback_required_(false), - use_surface_(use_surface), - codec_thread_(rtc::Thread::Create()), - j_media_codec_video_decoder_( - jni, - Java_MediaCodecVideoDecoder_Constructor(jni)) { - codec_thread_->SetName("MediaCodecVideoDecoder", NULL); - RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder"; - - ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_; - memset(&codec_, 0, sizeof(codec_)); - AllowBlockingCalls(); -} - -MediaCodecVideoDecoder::~MediaCodecVideoDecoder() { - // Call Release() to ensure no more callbacks to us after we are deleted. - Release(); -} - -int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst, - int32_t numberOfCores) { - ALOGD << "InitDecode."; - if (inst == NULL) { - ALOGE << "NULL VideoCodec instance"; - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - // Factory should guard against other codecs being used with us. - RTC_CHECK(inst->codecType == codecType_) - << "Unsupported codec " << inst->codecType << " for " << codecType_; - - if (sw_fallback_required_) { - ALOGE << "InitDecode() - fallback to SW decoder"; - return WEBRTC_VIDEO_CODEC_OK; - } - // Save VideoCodec instance for later. - if (&codec_ != inst) { - codec_ = *inst; - } - // If maxFramerate is not set then assume 30 fps. - codec_.maxFramerate = (codec_.maxFramerate >= 1) ? codec_.maxFramerate : 30; - - // Call Java init. - return codec_thread_->Invoke( - RTC_FROM_HERE, - Bind(&MediaCodecVideoDecoder::InitDecodeOnCodecThread, this)); -} - -void MediaCodecVideoDecoder::ResetVariables() { - CheckOnCodecThread(); - - key_frame_required_ = true; - frames_received_ = 0; - frames_decoded_ = 0; - frames_decoded_logged_ = kMaxDecodedLogFrames; - start_time_ms_ = rtc::TimeMillis(); - current_frames_ = 0; - current_bytes_ = 0; - current_decoding_time_ms_ = 0; - current_delay_time_ms_ = 0; - pending_frame_qps_.clear(); -} - -int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { - CheckOnCodecThread(); - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - ALOGD << "InitDecodeOnCodecThread Type: " << static_cast(codecType_) - << ". " << codec_.width << " x " << codec_.height - << ". Fps: " << static_cast(codec_.maxFramerate); - - // Release previous codec first if it was allocated before. - int ret_val = ReleaseOnCodecThread(); - if (ret_val < 0) { - ALOGE << "Release failure: " << ret_val << " - fallback to SW codec"; - sw_fallback_required_ = true; - return WEBRTC_VIDEO_CODEC_ERROR; - } - - ResetVariables(); - - bool success = Java_MediaCodecVideoDecoder_initDecode( - jni, j_media_codec_video_decoder_, codecType_, codec_.width, - codec_.height); - - if (CheckException(jni) || !success) { - ALOGE << "Codec initialization error - fallback to SW codec."; - sw_fallback_required_ = true; - return WEBRTC_VIDEO_CODEC_ERROR; - } - inited_ = true; - - switch (codecType_) { - case kVideoCodecVP8: - max_pending_frames_ = kMaxPendingFramesVp8; - break; - case kVideoCodecVP9: - max_pending_frames_ = kMaxPendingFramesVp9; - break; - case kVideoCodecH264: - max_pending_frames_ = kMaxPendingFramesH264; - break; - default: - max_pending_frames_ = 0; - } - ALOGD << "Maximum amount of pending frames: " << max_pending_frames_; - - ScopedJavaLocalRef input_buffers = - Java_MediaCodecVideoDecoder_getInputBuffers(jni, - j_media_codec_video_decoder_); - input_buffers_ = JavaToNativeVector>( - jni, input_buffers, [](JNIEnv* env, const JavaRef& o) { - return ScopedJavaGlobalRef(env, o); - }); - - codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this); - - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() { - CheckOnCodecThread(); - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - ALOGD << "ResetDecodeOnCodecThread Type: " << static_cast(codecType_) - << ". " << codec_.width << " x " << codec_.height; - ALOGD << " Frames received: " << frames_received_ - << ". Frames decoded: " << frames_decoded_; - - inited_ = false; - rtc::ThreadManager::Clear(this); - ResetVariables(); - - Java_MediaCodecVideoDecoder_reset(jni, j_media_codec_video_decoder_, - codec_.width, codec_.height); - - if (CheckException(jni)) { - ALOGE << "Soft reset error - fallback to SW codec."; - sw_fallback_required_ = true; - return WEBRTC_VIDEO_CODEC_ERROR; - } - inited_ = true; - - codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this); - - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t MediaCodecVideoDecoder::Release() { - ALOGD << "DecoderRelease request"; - return codec_thread_->Invoke( - RTC_FROM_HERE, Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this)); -} - -int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() { - if (!inited_) { - return WEBRTC_VIDEO_CODEC_OK; - } - CheckOnCodecThread(); - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ALOGD << "DecoderReleaseOnCodecThread: Frames received: " << frames_received_ - << ". Frames decoded: " << frames_decoded_; - ScopedLocalRefFrame local_ref_frame(jni); - input_buffers_.clear(); - Java_MediaCodecVideoDecoder_release(jni, j_media_codec_video_decoder_); - inited_ = false; - rtc::ThreadManager::Clear(this); - if (CheckException(jni)) { - ALOGE << "Decoder release exception"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - ALOGD << "DecoderReleaseOnCodecThread done"; - return WEBRTC_VIDEO_CODEC_OK; -} - -void MediaCodecVideoDecoder::CheckOnCodecThread() { - RTC_CHECK(codec_thread_.get() == ThreadManager::Instance()->CurrentThread()) - << "Running on wrong thread!"; -} - -void MediaCodecVideoDecoder::EnableFrameLogOnWarning() { - // Log next 2 output frames. - frames_decoded_logged_ = - std::max(frames_decoded_logged_, frames_decoded_ + kMaxWarningLogFrames); -} - -int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() { - CheckOnCodecThread(); - int ret_val = ReleaseOnCodecThread(); - if (ret_val < 0) { - ALOGE << "ProcessHWError: Release failure"; - } - if (codecType_ == kVideoCodecH264) { - // For now there is no SW H.264 which can be used as fallback codec. - // So try to restart hw codec for now. - ret_val = InitDecodeOnCodecThread(); - ALOGE << "Reset H.264 codec done. Status: " << ret_val; - if (ret_val == WEBRTC_VIDEO_CODEC_OK) { - // H.264 codec was succesfully reset - return regular error code. - return WEBRTC_VIDEO_CODEC_ERROR; - } else { - // Fail to restart H.264 codec - return error code which should stop the - // call. - return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; - } - } else { - sw_fallback_required_ = true; - ALOGE << "Return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE"; - return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; - } -} - -int32_t MediaCodecVideoDecoder::Decode( - const EncodedImage& inputImage, - bool missingFrames, - int64_t renderTimeMs) { - if (sw_fallback_required_) { - ALOGE << "Decode() - fallback to SW codec"; - return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; - } - if (callback_ == NULL) { - ALOGE << "Decode() - callback_ is NULL"; - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - if (inputImage.data() == NULL && inputImage.size() > 0) { - ALOGE << "Decode() - inputImage is incorrect"; - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - if (!inited_) { - ALOGE << "Decode() - decoder is not initialized"; - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - - // Check if encoded frame dimension has changed. - if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) && - (inputImage._encodedWidth != codec_.width || - inputImage._encodedHeight != codec_.height)) { - ALOGW << "Input resolution changed from " << codec_.width << " x " - << codec_.height << " to " << inputImage._encodedWidth << " x " - << inputImage._encodedHeight; - codec_.width = inputImage._encodedWidth; - codec_.height = inputImage._encodedHeight; - int32_t ret; - if (use_surface_ && - (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecH264)) { - // Soft codec reset - only for surface decoding. - ret = codec_thread_->Invoke( - RTC_FROM_HERE, - Bind(&MediaCodecVideoDecoder::ResetDecodeOnCodecThread, this)); - } else { - // Hard codec reset. - ret = InitDecode(&codec_, 1); - } - if (ret < 0) { - ALOGE << "InitDecode failure: " << ret << " - fallback to SW codec"; - sw_fallback_required_ = true; - return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; - } - } - - // Always start with a complete key frame. - if (key_frame_required_) { - if (inputImage._frameType != VideoFrameType::kVideoFrameKey) { - ALOGE << "Decode() - key frame is required"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - if (!inputImage._completeFrame) { - ALOGE << "Decode() - complete frame is required"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - key_frame_required_ = false; - } - if (inputImage.size() == 0) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - - return codec_thread_->Invoke( - RTC_FROM_HERE, - Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage)); -} - -int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( - const EncodedImage& inputImage) { - CheckOnCodecThread(); - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - - // Try to drain the decoder and wait until output is not too - // much behind the input. - if (codecType_ == kVideoCodecH264 && - frames_received_ > frames_decoded_ + max_pending_frames_) { - // Print warning for H.264 only - for VP8/VP9 one frame delay is ok. - ALOGW << "Decoder is too far behind. Try to drain. Received: " - << frames_received_ << ". Decoded: " << frames_decoded_; - EnableFrameLogOnWarning(); - } - const int64_t drain_start = rtc::TimeMillis(); - while ((frames_received_ > frames_decoded_ + max_pending_frames_) && - (rtc::TimeMillis() - drain_start) < kMediaCodecTimeoutMs) { - if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) { - ALOGE << "DeliverPendingOutputs error. Frames received: " - << frames_received_ << ". Frames decoded: " << frames_decoded_; - return ProcessHWErrorOnCodecThread(); - } - } - if (frames_received_ > frames_decoded_ + max_pending_frames_) { - ALOGE << "Output buffer dequeue timeout. Frames received: " - << frames_received_ << ". Frames decoded: " << frames_decoded_; - return ProcessHWErrorOnCodecThread(); - } - - // Get input buffer. - int j_input_buffer_index = Java_MediaCodecVideoDecoder_dequeueInputBuffer( - jni, j_media_codec_video_decoder_); - if (CheckException(jni) || j_input_buffer_index < 0) { - ALOGE << "dequeueInputBuffer error: " << j_input_buffer_index - << ". Retry DeliverPendingOutputs."; - EnableFrameLogOnWarning(); - // Try to drain the decoder. - if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) { - ALOGE << "DeliverPendingOutputs error. Frames received: " - << frames_received_ << ". Frames decoded: " << frames_decoded_; - return ProcessHWErrorOnCodecThread(); - } - // Try dequeue input buffer one last time. - j_input_buffer_index = Java_MediaCodecVideoDecoder_dequeueInputBuffer( - jni, j_media_codec_video_decoder_); - if (CheckException(jni) || j_input_buffer_index < 0) { - ALOGE << "dequeueInputBuffer critical error: " << j_input_buffer_index; - return ProcessHWErrorOnCodecThread(); - } - } - - // Copy encoded data to Java ByteBuffer. - jobject j_input_buffer = input_buffers_[j_input_buffer_index].obj(); - uint8_t* buffer = - reinterpret_cast(jni->GetDirectBufferAddress(j_input_buffer)); - RTC_CHECK(buffer) << "Indirect buffer??"; - size_t buffer_capacity = - rtc::dchecked_cast(jni->GetDirectBufferCapacity(j_input_buffer)); - if (CheckException(jni) || buffer_capacity < inputImage.size()) { - ALOGE << "Input frame size " << inputImage.size() - << " is bigger than buffer size " << buffer_capacity; - return ProcessHWErrorOnCodecThread(); - } - jlong presentation_timestamp_us = static_cast( - static_cast(frames_received_) * 1000000 / codec_.maxFramerate); - memcpy(buffer, inputImage.data(), inputImage.size()); - - if (frames_decoded_ < frames_decoded_logged_) { - ALOGD << "Decoder frame in # " << frames_received_ - << ". Type: " << static_cast(inputImage._frameType) - << ". Buffer # " << j_input_buffer_index - << ". TS: " << presentation_timestamp_us / 1000 - << ". Size: " << inputImage.size(); - } - - // Save input image timestamps for later output. - frames_received_++; - current_bytes_ += inputImage.size(); - absl::optional qp; - if (codecType_ == kVideoCodecVP8) { - int qp_int; - if (vp8::GetQp(inputImage.data(), inputImage.size(), &qp_int)) { - qp = qp_int; - } - } else if (codecType_ == kVideoCodecH264) { - h264_bitstream_parser_.ParseBitstream(inputImage.data(), inputImage.size()); - int qp_int; - if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) { - qp = qp_int; - } - } - pending_frame_qps_.push_back(qp); - - // Feed input to decoder. - bool success = Java_MediaCodecVideoDecoder_queueInputBuffer( - jni, j_media_codec_video_decoder_, j_input_buffer_index, - static_cast(inputImage.size()), presentation_timestamp_us, - static_cast(inputImage.Timestamp()), inputImage.ntp_time_ms_); - if (CheckException(jni) || !success) { - ALOGE << "queueInputBuffer error"; - return ProcessHWErrorOnCodecThread(); - } - - // Try to drain the decoder - if (!DeliverPendingOutputs(jni, 0)) { - ALOGE << "DeliverPendingOutputs error"; - return ProcessHWErrorOnCodecThread(); - } - - return WEBRTC_VIDEO_CODEC_OK; -} - -bool MediaCodecVideoDecoder::DeliverPendingOutputs(JNIEnv* jni, - int dequeue_timeout_ms) { - CheckOnCodecThread(); - if (frames_received_ <= frames_decoded_) { - // No need to query for output buffers - decoder is drained. - return true; - } - // Get decoder output. - ScopedJavaLocalRef j_decoder_output_buffer = - (use_surface_ ? &Java_MediaCodecVideoDecoder_dequeueTextureBuffer - : &Java_MediaCodecVideoDecoder_dequeueOutputBuffer)( - jni, j_media_codec_video_decoder_, dequeue_timeout_ms); - if (CheckException(jni)) { - ALOGE << "dequeueOutputBuffer() error"; - return false; - } - if (IsNull(jni, j_decoder_output_buffer)) { - // No decoded frame ready. - return true; - } - - // Get decoded video frame properties. - int color_format = Java_MediaCodecVideoDecoder_getColorFormat( - jni, j_media_codec_video_decoder_); - int width = - Java_MediaCodecVideoDecoder_getWidth(jni, j_media_codec_video_decoder_); - int height = - Java_MediaCodecVideoDecoder_getHeight(jni, j_media_codec_video_decoder_); - - rtc::scoped_refptr frame_buffer; - int64_t presentation_timestamps_ms = 0; - int64_t output_timestamps_ms = 0; - int64_t output_ntp_timestamps_ms = 0; - int decode_time_ms = 0; - int64_t frame_delayed_ms = 0; - if (use_surface_) { - // Extract data from Java DecodedTextureBuffer. - presentation_timestamps_ms = - Java_DecodedTextureBuffer_getPresentationTimestampMs( - jni, j_decoder_output_buffer); - output_timestamps_ms = - Java_DecodedTextureBuffer_getTimeStampMs(jni, j_decoder_output_buffer); - output_ntp_timestamps_ms = Java_DecodedTextureBuffer_getNtpTimestampMs( - jni, j_decoder_output_buffer); - decode_time_ms = - Java_DecodedTextureBuffer_getDecodeTimeMs(jni, j_decoder_output_buffer); - - ScopedJavaLocalRef j_video_frame_buffer = - Java_DecodedTextureBuffer_getVideoFrameBuffer(jni, - j_decoder_output_buffer); - // |video_frame_buffer| == null represents a dropped frame. - if (!j_video_frame_buffer.is_null()) { - frame_delayed_ms = Java_DecodedTextureBuffer_getFrameDelayMs( - jni, j_decoder_output_buffer); - frame_buffer = AndroidVideoBuffer::Adopt(jni, j_video_frame_buffer); - } else { - EnableFrameLogOnWarning(); - } - } else { - // Extract data from Java ByteBuffer and create output yuv420 frame - - // for non surface decoding only. - int stride = Java_MediaCodecVideoDecoder_getStride( - jni, j_media_codec_video_decoder_); - const int slice_height = Java_MediaCodecVideoDecoder_getSliceHeight( - jni, j_media_codec_video_decoder_); - const int output_buffer_index = - Java_DecodedOutputBuffer_getIndex(jni, j_decoder_output_buffer); - const int output_buffer_offset = - Java_DecodedOutputBuffer_getOffset(jni, j_decoder_output_buffer); - const int output_buffer_size = - Java_DecodedOutputBuffer_getSize(jni, j_decoder_output_buffer); - presentation_timestamps_ms = - Java_DecodedOutputBuffer_getPresentationTimestampMs( - jni, j_decoder_output_buffer); - output_timestamps_ms = - Java_DecodedOutputBuffer_getTimestampMs(jni, j_decoder_output_buffer); - output_ntp_timestamps_ms = Java_DecodedOutputBuffer_getNtpTimestampMs( - jni, j_decoder_output_buffer); - - decode_time_ms = - Java_DecodedOutputBuffer_getDecodeTimeMs(jni, j_decoder_output_buffer); - RTC_CHECK_GE(slice_height, height); - - if (output_buffer_size < width * height * 3 / 2) { - ALOGE << "Insufficient output buffer size: " << output_buffer_size; - return false; - } - if (output_buffer_size < stride * height * 3 / 2 && - slice_height == height && stride > width) { - // Some codecs (Exynos) incorrectly report stride information for - // output byte buffer, so actual stride value need to be corrected. - stride = output_buffer_size * 2 / (height * 3); - } - ScopedJavaLocalRef output_buffers = - Java_MediaCodecVideoDecoder_getOutputBuffers( - jni, j_media_codec_video_decoder_); - jobject output_buffer = - jni->GetObjectArrayElement(output_buffers.obj(), output_buffer_index); - uint8_t* payload = - reinterpret_cast(jni->GetDirectBufferAddress(output_buffer)); - if (CheckException(jni)) { - return false; - } - payload += output_buffer_offset; - - // Create yuv420 frame. - rtc::scoped_refptr i420_buffer = - decoded_frame_pool_.CreateBuffer(width, height); - if (color_format == COLOR_FormatYUV420Planar) { - RTC_CHECK_EQ(0, stride % 2); - const int uv_stride = stride / 2; - const uint8_t* y_ptr = payload; - const uint8_t* u_ptr = y_ptr + stride * slice_height; - - // Note that the case with odd |slice_height| is handled in a special way. - // The chroma height contained in the payload is rounded down instead of - // up, making it one row less than what we expect in WebRTC. Therefore, we - // have to duplicate the last chroma rows for this case. Also, the offset - // between the Y plane and the U plane is unintuitive for this case. See - // http://bugs.webrtc.org/6651 for more info. - const int chroma_width = (width + 1) / 2; - const int chroma_height = - (slice_height % 2 == 0) ? (height + 1) / 2 : height / 2; - const int u_offset = uv_stride * slice_height / 2; - const uint8_t* v_ptr = u_ptr + u_offset; - libyuv::CopyPlane(y_ptr, stride, i420_buffer->MutableDataY(), - i420_buffer->StrideY(), width, height); - libyuv::CopyPlane(u_ptr, uv_stride, i420_buffer->MutableDataU(), - i420_buffer->StrideU(), chroma_width, chroma_height); - libyuv::CopyPlane(v_ptr, uv_stride, i420_buffer->MutableDataV(), - i420_buffer->StrideV(), chroma_width, chroma_height); - if (slice_height % 2 == 1) { - RTC_CHECK_EQ(height, slice_height); - // Duplicate the last chroma rows. - uint8_t* u_last_row_ptr = i420_buffer->MutableDataU() + - chroma_height * i420_buffer->StrideU(); - memcpy(u_last_row_ptr, u_last_row_ptr - i420_buffer->StrideU(), - i420_buffer->StrideU()); - uint8_t* v_last_row_ptr = i420_buffer->MutableDataV() + - chroma_height * i420_buffer->StrideV(); - memcpy(v_last_row_ptr, v_last_row_ptr - i420_buffer->StrideV(), - i420_buffer->StrideV()); - } - } else { - // All other supported formats are nv12. - const uint8_t* y_ptr = payload; - const uint8_t* uv_ptr = y_ptr + stride * slice_height; - libyuv::NV12ToI420(y_ptr, stride, uv_ptr, stride, - i420_buffer->MutableDataY(), i420_buffer->StrideY(), - i420_buffer->MutableDataU(), i420_buffer->StrideU(), - i420_buffer->MutableDataV(), i420_buffer->StrideV(), - width, height); - } - frame_buffer = i420_buffer; - - // Return output byte buffer back to codec. - Java_MediaCodecVideoDecoder_returnDecodedOutputBuffer( - jni, j_media_codec_video_decoder_, output_buffer_index); - if (CheckException(jni)) { - ALOGE << "returnDecodedOutputBuffer error"; - return false; - } - } - if (frames_decoded_ < frames_decoded_logged_) { - ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width << " x " - << height << ". Color: " << color_format - << ". TS: " << presentation_timestamps_ms - << ". DecTime: " << static_cast(decode_time_ms) - << ". DelayTime: " << static_cast(frame_delayed_ms); - } - - // Calculate and print decoding statistics - every 3 seconds. - frames_decoded_++; - current_frames_++; - current_decoding_time_ms_ += decode_time_ms; - current_delay_time_ms_ += frame_delayed_ms; - int statistic_time_ms = rtc::TimeMillis() - start_time_ms_; - if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs && - current_frames_ > 0) { - int current_bitrate = current_bytes_ * 8 / statistic_time_ms; - int current_fps = - (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms; - ALOGD << "Frames decoded: " << frames_decoded_ - << ". Received: " << frames_received_ - << ". Bitrate: " << current_bitrate - << " kbps" - ". Fps: " - << current_fps - << ". DecTime: " << (current_decoding_time_ms_ / current_frames_) - << ". DelayTime: " << (current_delay_time_ms_ / current_frames_) - << " for last " << statistic_time_ms << " ms."; - start_time_ms_ = rtc::TimeMillis(); - current_frames_ = 0; - current_bytes_ = 0; - current_decoding_time_ms_ = 0; - current_delay_time_ms_ = 0; - } - - // If the frame was dropped, frame_buffer is left as nullptr. - if (frame_buffer) { - VideoFrame decoded_frame = VideoFrame::Builder() - .set_video_frame_buffer(frame_buffer) - .set_timestamp_rtp(0) - .set_timestamp_ms(0) - .set_rotation(kVideoRotation_0) - .build(); - decoded_frame.set_timestamp(output_timestamps_ms); - decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms); - - absl::optional qp = pending_frame_qps_.front(); - pending_frame_qps_.pop_front(); - callback_->Decoded(decoded_frame, decode_time_ms, qp); - } - return true; -} - -int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback( - DecodedImageCallback* callback) { - callback_ = callback; - return WEBRTC_VIDEO_CODEC_OK; -} - -void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) { - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - if (!inited_) { - return; - } - // We only ever send one message to |this| directly (not through a Bind()'d - // functor), so expect no ID/data. - RTC_CHECK(!msg->message_id) << "Unexpected message!"; - RTC_CHECK(!msg->pdata) << "Unexpected message!"; - CheckOnCodecThread(); - - if (!DeliverPendingOutputs(jni, 0)) { - ALOGE << "OnMessage: DeliverPendingOutputs error"; - ProcessHWErrorOnCodecThread(); - return; - } - codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this); -} - -const char* MediaCodecVideoDecoder::ImplementationName() const { - return "MediaCodec"; -} - -static jlong JNI_MediaCodecVideoDecoder_CreateDecoder( - JNIEnv* env, - const JavaParamRef& codec, - jboolean use_surface) { - ScopedLocalRefFrame local_ref_frame(env); - return jlongFromPointer(new MediaCodecVideoDecoder( - env, PayloadStringToCodecType(JavaToNativeString(env, codec)), - use_surface)); -} - -} // namespace jni -} // namespace webrtc diff --git a/sdk/android/src/jni/android_media_encoder.cc b/sdk/android/src/jni/android_media_encoder.cc deleted file mode 100644 index 0d0e83a49f..0000000000 --- a/sdk/android/src/jni/android_media_encoder.cc +++ /dev/null @@ -1,1247 +0,0 @@ -/* - * Copyright 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include -#include -#include -#include -#include - -#include "absl/memory/memory.h" -#include "api/task_queue/queued_task.h" -#include "api/task_queue/task_queue_base.h" -#include "api/video_codecs/sdp_video_format.h" -#include "api/video_codecs/video_encoder.h" -#include "common_video/h264/h264_bitstream_parser.h" -#include "common_video/h264/h264_common.h" -#include "common_video/h264/profile_level_id.h" -#include "media/base/codec.h" -#include "media/base/media_constants.h" -#include "media/engine/internal_encoder_factory.h" -#include "modules/video_coding/include/video_codec_interface.h" -#include "modules/video_coding/utility/quality_scaler.h" -#include "modules/video_coding/utility/vp8_header_parser.h" -#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" -#include "rtc_base/bind.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "rtc_base/synchronization/sequence_checker.h" -#include "rtc_base/thread.h" -#include "rtc_base/time_utils.h" -#include "rtc_base/weak_ptr.h" -#include "sdk/android/generated_video_jni/MediaCodecVideoEncoder_jni.h" -#include "sdk/android/native_api/jni/java_types.h" -#include "sdk/android/src/jni/android_media_codec_common.h" -#include "sdk/android/src/jni/jni_helpers.h" -#include "sdk/android/src/jni/video_codec_info.h" -#include "sdk/android/src/jni/video_frame.h" -#include "system_wrappers/include/field_trial.h" -#include "third_party/libyuv/include/libyuv/convert.h" -#include "third_party/libyuv/include/libyuv/convert_from.h" -#include "third_party/libyuv/include/libyuv/video_common.h" - -using rtc::Bind; -using rtc::ThreadManager; - -namespace webrtc { -namespace jni { - -// Maximum supported HW video encoder fps. -#define MAX_VIDEO_FPS 30 -// Maximum allowed fps value in SetRates() call. -#define MAX_ALLOWED_VIDEO_FPS 60 -// Maximum allowed frames in encoder input queue. -#define MAX_ENCODER_Q_SIZE 2 -// Maximum amount of dropped frames caused by full encoder queue - exceeding -// this threshold means that encoder probably got stuck and need to be reset. -#define ENCODER_STALL_FRAMEDROP_THRESHOLD 60 - -// Logging macros. -#define TAG_ENCODER "MediaCodecVideoEncoder" -#ifdef TRACK_BUFFER_TIMING -#define ALOGV(...) -__android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__) -#else -#define ALOGV(...) -#endif -#define ALOGD RTC_LOG_TAG(rtc::LS_INFO, TAG_ENCODER) -#define ALOGW RTC_LOG_TAG(rtc::LS_WARNING, TAG_ENCODER) -#define ALOGE RTC_LOG_TAG(rtc::LS_ERROR, TAG_ENCODER) - - namespace { - // Maximum time limit between incoming frames before requesting a key frame. - const int64_t kFrameDiffThresholdMs = 350; - const int kMinKeyFrameInterval = 6; - const char kCustomQPThresholdsFieldTrial[] = "WebRTC-CustomQPThresholds"; -} // namespace - -// MediaCodecVideoEncoder is a VideoEncoder implementation that uses -// Android's MediaCodec SDK API behind the scenes to implement (hopefully) -// HW-backed video encode. This C++ class is implemented as a very thin shim, -// delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder. -// MediaCodecVideoEncoder must be operated on a single task queue, currently -// this is the encoder queue from ViE encoder. -class MediaCodecVideoEncoder : public VideoEncoder { - public: - ~MediaCodecVideoEncoder() override; - MediaCodecVideoEncoder(JNIEnv* jni, - const SdpVideoFormat& format, - bool has_egl_context); - - // VideoEncoder implementation. - int32_t InitEncode(const VideoCodec* codec_settings, - const Settings& settings) override; - int32_t Encode(const VideoFrame& input_image, - const std::vector* frame_types) override; - int32_t RegisterEncodeCompleteCallback( - EncodedImageCallback* callback) override; - int32_t Release() override; - void SetRates(const RateControlParameters& parameters) override; - EncoderInfo GetEncoderInfo() const override; - - // Fills the input buffer with data from the buffers passed as parameters. - bool FillInputBuffer(JNIEnv* jni, - int input_buffer_index, - uint8_t const* buffer_y, - int stride_y, - uint8_t const* buffer_u, - int stride_u, - uint8_t const* buffer_v, - int stride_v); - - private: - class EncodeTask : public QueuedTask { - public: - explicit EncodeTask(rtc::WeakPtr encoder); - bool Run() override; - - private: - rtc::WeakPtr encoder_; - }; - - // ResetCodec() calls Release() and InitEncodeInternal() in an attempt to - // restore the codec to an operable state. Necessary after all manner of - // OMX-layer errors. Returns true if the codec was reset successfully. - bool ResetCodec(); - - // Fallback to a software encoder if one is supported else try to reset the - // encoder. Called with |reset_if_fallback_unavailable| equal to false from - // init/release encoder so that we don't go into infinite recursion. - // Returns true if the codec was reset successfully. - bool ProcessHWError(bool reset_if_fallback_unavailable); - - // Calls ProcessHWError(true). Returns WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE if - // sw_fallback_required_ was set or WEBRTC_VIDEO_CODEC_ERROR otherwise. - int32_t ProcessHWErrorOnEncode(); - - // If width==0 then this is assumed to be a re-initialization and the - // previously-current values are reused instead of the passed parameters - // (makes it easier to reason about thread-safety). - int32_t InitEncodeInternal(int width, - int height, - int kbps, - int fps, - bool use_surface); - // Reconfigure to match |frame| in width, height. Also reconfigures the - // encoder if |frame| is a texture/byte buffer and the encoder is initialized - // for byte buffer/texture. Returns false if reconfiguring fails. - bool MaybeReconfigureEncoder(JNIEnv* jni, const VideoFrame& frame); - - // Returns true if the frame is a texture frame and we should use surface - // based encoding. - bool IsTextureFrame(JNIEnv* jni, const VideoFrame& frame); - - bool EncodeByteBuffer(JNIEnv* jni, - bool key_frame, - const VideoFrame& frame, - int input_buffer_index); - // Encodes a new style org.webrtc.VideoFrame. Might be a I420 or a texture - // frame. - bool EncodeJavaFrame(JNIEnv* jni, - bool key_frame, - const JavaRef& frame, - int input_buffer_index); - - // Deliver any outputs pending in the MediaCodec to our |callback_| and return - // true on success. - bool DeliverPendingOutputs(JNIEnv* jni); - - VideoEncoder::ScalingSettings GetScalingSettingsInternal() const; - - // Displays encoder statistics. - void LogStatistics(bool force_log); - - VideoCodecType GetCodecType() const; - -#if RTC_DCHECK_IS_ON - // Mutex for protecting inited_. It is only used for correctness checking on - // debug build. It is used for checking that encoder has been released in the - // destructor. Because this might happen on a different thread, we need a - // mutex. - rtc::CriticalSection inited_crit_; -#endif - - // Type of video codec. - const SdpVideoFormat format_; - - EncodedImageCallback* callback_; - - // State that is constant for the lifetime of this object once the ctor - // returns. - SequenceChecker encoder_queue_checker_; - ScopedJavaGlobalRef j_media_codec_video_encoder_; - - // State that is valid only between InitEncode() and the next Release(). - int width_; // Frame width in pixels. - int height_; // Frame height in pixels. - bool inited_; - bool use_surface_; - enum libyuv::FourCC encoder_fourcc_; // Encoder color space format. - uint32_t last_set_bitrate_kbps_; // Last-requested bitrate in kbps. - uint32_t last_set_fps_; // Last-requested frame rate. - int64_t current_timestamp_us_; // Current frame timestamps in us. - int frames_received_; // Number of frames received by encoder. - int frames_encoded_; // Number of frames encoded by encoder. - int frames_dropped_media_encoder_; // Number of frames dropped by encoder. - // Number of dropped frames caused by full queue. - int consecutive_full_queue_frame_drops_; - int64_t stat_start_time_ms_; // Start time for statistics. - int current_frames_; // Number of frames in the current statistics interval. - int current_bytes_; // Encoded bytes in the current statistics interval. - int current_acc_qp_; // Accumulated QP in the current statistics interval. - int current_encoding_time_ms_; // Overall encoding time in the current second - int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame. - int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame. - // Holds the task while the polling loop is paused. - std::unique_ptr encode_task_; - - struct InputFrameInfo { - InputFrameInfo(int64_t encode_start_time, - int32_t frame_timestamp, - int64_t frame_render_time_ms, - VideoRotation rotation) - : encode_start_time(encode_start_time), - frame_timestamp(frame_timestamp), - frame_render_time_ms(frame_render_time_ms), - rotation(rotation) {} - // Time when video frame is sent to encoder input. - const int64_t encode_start_time; - - // Input frame information. - const int32_t frame_timestamp; - const int64_t frame_render_time_ms; - const VideoRotation rotation; - }; - std::list input_frame_infos_; - int32_t output_timestamp_; // Last output frame timestamp from - // |input_frame_infos_|. - int64_t output_render_time_ms_; // Last output frame render time from - // |input_frame_infos_|. - VideoRotation output_rotation_; // Last output frame rotation from - // |input_frame_infos_|. - - // Frame size in bytes fed to MediaCodec. - int yuv_size_; - // True only when between a callback_->OnEncodedImage() call return a positive - // value and the next Encode() call being ignored. - bool drop_next_input_frame_; - bool scale_; - H264::Profile profile_; - // Global references; must be deleted in Release(). - std::vector> input_buffers_; - H264BitstreamParser h264_bitstream_parser_; - - // VP9 variables to populate codec specific structure. - GofInfoVP9 gof_; // Contains each frame's temporal information for - // non-flexible VP9 mode. - size_t gof_idx_; - - const bool has_egl_context_; - EncoderInfo encoder_info_; - - // Temporary fix for VP8. - // Sends a key frame if frames are largely spaced apart (possibly - // corresponding to a large image change). - int64_t last_frame_received_ms_; - int frames_received_since_last_key_; - VideoCodecMode codec_mode_; - - bool sw_fallback_required_; - - // All other member variables should be before WeakPtrFactory. Valid only from - // InitEncode to Release. - std::unique_ptr> weak_factory_; -}; - -MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { -#if RTC_DCHECK_IS_ON - rtc::CritScope lock(&inited_crit_); - RTC_DCHECK(!inited_); -#endif -} - -MediaCodecVideoEncoder::MediaCodecVideoEncoder(JNIEnv* jni, - const SdpVideoFormat& format, - bool has_egl_context) - : format_(format), - callback_(NULL), - j_media_codec_video_encoder_( - jni, - Java_MediaCodecVideoEncoder_Constructor(jni)), - inited_(false), - use_surface_(false), - has_egl_context_(has_egl_context), - sw_fallback_required_(false) { - encoder_queue_checker_.Detach(); -} - -int32_t MediaCodecVideoEncoder::InitEncode(const VideoCodec* codec_settings, - const Settings& settings) { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - if (codec_settings == NULL) { - ALOGE << "NULL VideoCodec instance"; - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - // Factory should guard against other codecs being used with us. - const VideoCodecType codec_type = GetCodecType(); - RTC_CHECK(codec_settings->codecType == codec_type) - << "Unsupported codec " << codec_settings->codecType << " for " - << codec_type; - if (sw_fallback_required_) { - return WEBRTC_VIDEO_CODEC_OK; - } - codec_mode_ = codec_settings->mode; - int init_width = codec_settings->width; - int init_height = codec_settings->height; - // Scaling is optionally enabled for VP8 and VP9. - // TODO(pbos): Extract automaticResizeOn out of VP8 settings. - scale_ = false; - if (codec_type == kVideoCodecVP8) { - scale_ = codec_settings->VP8().automaticResizeOn; - } else if (codec_type == kVideoCodecVP9) { - scale_ = codec_settings->VP9().automaticResizeOn; - } else { - scale_ = true; - } - - ALOGD << "InitEncode request: " << init_width << " x " << init_height; - ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); - - if (codec_settings->numberOfSimulcastStreams > 1) { - ALOGD << "Number of simulcast layers requested: " - << codec_settings->numberOfSimulcastStreams - << ". Requesting software fallback."; - return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; - } - - // Check allowed H.264 profile - profile_ = H264::Profile::kProfileBaseline; - if (codec_type == kVideoCodecH264) { - const absl::optional profile_level_id = - H264::ParseSdpProfileLevelId(format_.parameters); - RTC_DCHECK(profile_level_id); - profile_ = profile_level_id->profile; - ALOGD << "H.264 profile: " << profile_; - } - - encoder_info_.supports_native_handle = has_egl_context_; - encoder_info_.implementation_name = "MediaCodec"; - encoder_info_.scaling_settings = GetScalingSettingsInternal(); - encoder_info_.is_hardware_accelerated = true; - encoder_info_.has_internal_source = false; - - return InitEncodeInternal( - init_width, init_height, codec_settings->startBitrate, - codec_settings->maxFramerate, - codec_settings->expect_encode_from_texture && has_egl_context_); -} - -bool MediaCodecVideoEncoder::ResetCodec() { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - ALOGE << "Reset"; - if (Release() != WEBRTC_VIDEO_CODEC_OK) { - ALOGE << "Releasing codec failed during reset."; - return false; - } - if (InitEncodeInternal(width_, height_, 0, 0, false) != - WEBRTC_VIDEO_CODEC_OK) { - ALOGE << "Initializing encoder failed during reset."; - return false; - } - return true; -} - -MediaCodecVideoEncoder::EncodeTask::EncodeTask( - rtc::WeakPtr encoder) - : encoder_(encoder) {} - -bool MediaCodecVideoEncoder::EncodeTask::Run() { - if (!encoder_) { - // Encoder was destroyed. - return true; - } - - RTC_DCHECK_RUN_ON(&encoder_->encoder_queue_checker_); - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - - if (!encoder_->inited_) { - encoder_->encode_task_ = absl::WrapUnique(this); - return false; - } - - // It would be nice to recover from a failure here if one happened, but it's - // unclear how to signal such a failure to the app, so instead we stay silent - // about it and let the next app-called API method reveal the borkedness. - encoder_->DeliverPendingOutputs(jni); - - if (!encoder_) { - // Encoder can be destroyed in DeliverPendingOutputs. - return true; - } - - // Call log statistics here so it's called even if no frames are being - // delivered. - encoder_->LogStatistics(false); - - // If there aren't more frames to deliver, we can start polling at lower rate. - if (encoder_->input_frame_infos_.empty()) { - TaskQueueBase::Current()->PostDelayedTask(absl::WrapUnique(this), - kMediaCodecPollNoFramesMs); - } else { - TaskQueueBase::Current()->PostDelayedTask(absl::WrapUnique(this), - kMediaCodecPollMs); - } - - return false; -} - -bool IsFormatSupported(const std::vector& supported_formats, - const SdpVideoFormat& format) { - for (const SdpVideoFormat& supported_format : supported_formats) { - if (cricket::IsSameCodec(format.name, format.parameters, - supported_format.name, - supported_format.parameters)) { - return true; - } - } - return false; -} - -bool MediaCodecVideoEncoder::ProcessHWError( - bool reset_if_fallback_unavailable) { - ALOGE << "ProcessHWError"; - if (IsFormatSupported(InternalEncoderFactory::SupportedFormats(), format_)) { - ALOGE << "Fallback to SW encoder."; - sw_fallback_required_ = true; - return false; - } else if (reset_if_fallback_unavailable) { - ALOGE << "Reset encoder."; - return ResetCodec(); - } - return false; -} - -int32_t MediaCodecVideoEncoder::ProcessHWErrorOnEncode() { - ProcessHWError(true /* reset_if_fallback_unavailable */); - return sw_fallback_required_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE - : WEBRTC_VIDEO_CODEC_ERROR; -} - -VideoCodecType MediaCodecVideoEncoder::GetCodecType() const { - return PayloadStringToCodecType(format_.name); -} - -int32_t MediaCodecVideoEncoder::InitEncodeInternal(int width, - int height, - int kbps, - int fps, - bool use_surface) { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - if (sw_fallback_required_) { - return WEBRTC_VIDEO_CODEC_OK; - } - RTC_CHECK(!use_surface || has_egl_context_) << "EGL context not set."; - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - - const VideoCodecType codec_type = GetCodecType(); - ALOGD << "InitEncodeInternal Type: " << static_cast(codec_type) << ", " - << width << " x " << height << ". Bitrate: " << kbps - << " kbps. Fps: " << fps << ". Profile: " << profile_ << "."; - if (kbps == 0) { - kbps = last_set_bitrate_kbps_; - } - if (fps == 0) { - fps = MAX_VIDEO_FPS; - } - - width_ = width; - height_ = height; - last_set_bitrate_kbps_ = kbps; - last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS; - yuv_size_ = width_ * height_ * 3 / 2; - frames_received_ = 0; - frames_encoded_ = 0; - frames_dropped_media_encoder_ = 0; - consecutive_full_queue_frame_drops_ = 0; - current_timestamp_us_ = 0; - stat_start_time_ms_ = rtc::TimeMillis(); - current_frames_ = 0; - current_bytes_ = 0; - current_acc_qp_ = 0; - current_encoding_time_ms_ = 0; - last_input_timestamp_ms_ = -1; - last_output_timestamp_ms_ = -1; - output_timestamp_ = 0; - output_render_time_ms_ = 0; - input_frame_infos_.clear(); - drop_next_input_frame_ = false; - use_surface_ = use_surface; - gof_.SetGofInfoVP9(TemporalStructureMode::kTemporalStructureMode1); - gof_idx_ = 0; - last_frame_received_ms_ = -1; - frames_received_since_last_key_ = kMinKeyFrameInterval; - - // We enforce no extra stride/padding in the format creation step. - const bool encode_status = Java_MediaCodecVideoEncoder_initEncode( - jni, j_media_codec_video_encoder_, codec_type, profile_, width, height, - kbps, fps, use_surface); - - if (!encode_status) { - ALOGE << "Failed to configure encoder."; - ProcessHWError(false /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - if (CheckException(jni)) { - ALOGE << "Exception in init encode."; - ProcessHWError(false /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - if (!use_surface) { - ScopedJavaLocalRef input_buffers = - Java_MediaCodecVideoEncoder_getInputBuffers( - jni, j_media_codec_video_encoder_); - if (CheckException(jni)) { - ALOGE << "Exception in get input buffers."; - ProcessHWError(false /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - if (IsNull(jni, input_buffers)) { - ProcessHWError(false /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - switch (Java_MediaCodecVideoEncoder_getColorFormat( - jni, j_media_codec_video_encoder_)) { - case COLOR_FormatYUV420Planar: - encoder_fourcc_ = libyuv::FOURCC_YU12; - break; - case COLOR_FormatYUV420SemiPlanar: - case COLOR_QCOM_FormatYUV420SemiPlanar: - case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: - encoder_fourcc_ = libyuv::FOURCC_NV12; - break; - default: - RTC_LOG(LS_ERROR) << "Wrong color format."; - ProcessHWError(false /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - RTC_CHECK(input_buffers_.empty()) - << "Unexpected double InitEncode without Release"; - input_buffers_ = JavaToNativeVector>( - jni, input_buffers, [](JNIEnv* env, const JavaRef& o) { - return ScopedJavaGlobalRef(env, o); - }); - for (const ScopedJavaGlobalRef& buffer : input_buffers_) { - int64_t yuv_buffer_capacity = jni->GetDirectBufferCapacity(buffer.obj()); - if (CheckException(jni)) { - ALOGE << "Exception in get direct buffer capacity."; - ProcessHWError(false /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; - } - } - - { -#if RTC_DCHECK_IS_ON - rtc::CritScope lock(&inited_crit_); -#endif - inited_ = true; - } - weak_factory_.reset(new rtc::WeakPtrFactory(this)); - encode_task_.reset(new EncodeTask(weak_factory_->GetWeakPtr())); - - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t MediaCodecVideoEncoder::Encode( - const VideoFrame& frame, - const std::vector* frame_types) { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - if (sw_fallback_required_) - return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - const int64_t frame_input_time_ms = rtc::TimeMillis(); - - if (!inited_) { - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - - bool send_key_frame = false; - if (codec_mode_ == VideoCodecMode::kRealtimeVideo) { - ++frames_received_since_last_key_; - int64_t now_ms = rtc::TimeMillis(); - if (last_frame_received_ms_ != -1 && - (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) { - // Add limit to prevent triggering a key for every frame for very low - // framerates (e.g. if frame diff > kFrameDiffThresholdMs). - if (frames_received_since_last_key_ > kMinKeyFrameInterval) { - ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_); - send_key_frame = true; - } - frames_received_since_last_key_ = 0; - } - last_frame_received_ms_ = now_ms; - } - - frames_received_++; - if (!DeliverPendingOutputs(jni)) { - if (!ProcessHWError(true /* reset_if_fallback_unavailable */)) { - return sw_fallback_required_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE - : WEBRTC_VIDEO_CODEC_ERROR; - } - } - if (frames_encoded_ < kMaxEncodedLogFrames) { - ALOGD << "Encoder frame in # " << (frames_received_ - 1) - << ". TS: " << static_cast(current_timestamp_us_ / 1000) - << ". Q: " << input_frame_infos_.size() << ". Fps: " << last_set_fps_ - << ". Kbps: " << last_set_bitrate_kbps_; - } - - if (drop_next_input_frame_) { - ALOGW << "Encoder drop frame - failed callback."; - drop_next_input_frame_ = false; - current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; - frames_dropped_media_encoder_++; - return WEBRTC_VIDEO_CODEC_OK; - } - - RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count"; - - // Check if we accumulated too many frames in encoder input buffers and drop - // frame if so. - if (input_frame_infos_.size() > MAX_ENCODER_Q_SIZE) { - ALOGD << "Already " << input_frame_infos_.size() - << " frames in the queue, dropping" - ". TS: " - << static_cast(current_timestamp_us_ / 1000) - << ". Fps: " << last_set_fps_ - << ". Consecutive drops: " << consecutive_full_queue_frame_drops_; - current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; - consecutive_full_queue_frame_drops_++; - if (consecutive_full_queue_frame_drops_ >= - ENCODER_STALL_FRAMEDROP_THRESHOLD) { - ALOGE << "Encoder got stuck."; - return ProcessHWErrorOnEncode(); - } - frames_dropped_media_encoder_++; - return WEBRTC_VIDEO_CODEC_OK; - } - consecutive_full_queue_frame_drops_ = 0; - - rtc::scoped_refptr input_buffer(frame.video_frame_buffer()); - - VideoFrame input_frame = VideoFrame::Builder() - .set_video_frame_buffer(input_buffer) - .set_timestamp_rtp(frame.timestamp()) - .set_timestamp_ms(frame.render_time_ms()) - .set_rotation(frame.rotation()) - .set_id(frame.id()) - .build(); - - if (!MaybeReconfigureEncoder(jni, input_frame)) { - ALOGE << "Failed to reconfigure encoder."; - return WEBRTC_VIDEO_CODEC_ERROR; - } - - const bool key_frame = - frame_types->front() != VideoFrameType::kVideoFrameDelta || - send_key_frame; - bool encode_status = true; - - int j_input_buffer_index = -1; - if (!use_surface_) { - j_input_buffer_index = Java_MediaCodecVideoEncoder_dequeueInputBuffer( - jni, j_media_codec_video_encoder_); - if (CheckException(jni)) { - ALOGE << "Exception in dequeu input buffer."; - return ProcessHWErrorOnEncode(); - } - if (j_input_buffer_index == -1) { - // Video codec falls behind - no input buffer available. - ALOGW << "Encoder drop frame - no input buffers available"; - if (frames_received_ > 1) { - current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; - frames_dropped_media_encoder_++; - } else { - // Input buffers are not ready after codec initialization, HW is still - // allocating thme - this is expected and should not result in drop - // frame report. - frames_received_ = 0; - } - return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. - } else if (j_input_buffer_index == -2) { - return ProcessHWErrorOnEncode(); - } - } - - if (input_frame.video_frame_buffer()->type() != - VideoFrameBuffer::Type::kNative) { - encode_status = - EncodeByteBuffer(jni, key_frame, input_frame, j_input_buffer_index); - } else { - ScopedJavaLocalRef j_frame = NativeToJavaVideoFrame(jni, frame); - encode_status = - EncodeJavaFrame(jni, key_frame, j_frame, j_input_buffer_index); - ReleaseJavaVideoFrame(jni, j_frame); - } - - if (!encode_status) { - ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp(); - return ProcessHWErrorOnEncode(); - } - - // Save input image timestamps for later output. - input_frame_infos_.emplace_back(frame_input_time_ms, input_frame.timestamp(), - input_frame.render_time_ms(), - input_frame.rotation()); - - last_input_timestamp_ms_ = - current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec; - - current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; - - // Start the polling loop if it is not started. - if (encode_task_) { - TaskQueueBase::Current()->PostDelayedTask(std::move(encode_task_), - kMediaCodecPollMs); - } - - if (!DeliverPendingOutputs(jni)) { - return ProcessHWErrorOnEncode(); - } - return WEBRTC_VIDEO_CODEC_OK; -} - -bool MediaCodecVideoEncoder::MaybeReconfigureEncoder(JNIEnv* jni, - const VideoFrame& frame) { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - - bool is_texture = IsTextureFrame(jni, frame); - const bool reconfigure_due_to_format = is_texture != use_surface_; - const bool reconfigure_due_to_size = - frame.width() != width_ || frame.height() != height_; - - if (reconfigure_due_to_format) { - ALOGD << "Reconfigure encoder due to format change. " - << (use_surface_ ? "Reconfiguring to encode from byte buffer." - : "Reconfiguring to encode from texture."); - LogStatistics(true); - } - if (reconfigure_due_to_size) { - ALOGW << "Reconfigure encoder due to frame resolution change from " - << width_ << " x " << height_ << " to " << frame.width() << " x " - << frame.height(); - LogStatistics(true); - width_ = frame.width(); - height_ = frame.height(); - } - - if (!reconfigure_due_to_format && !reconfigure_due_to_size) - return true; - - Release(); - - return InitEncodeInternal(width_, height_, 0, 0, is_texture) == - WEBRTC_VIDEO_CODEC_OK; -} - -bool MediaCodecVideoEncoder::IsTextureFrame(JNIEnv* jni, - const VideoFrame& frame) { - if (frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) { - return false; - } - return Java_MediaCodecVideoEncoder_isTextureBuffer( - jni, static_cast(frame.video_frame_buffer().get()) - ->video_frame_buffer()); -} - -bool MediaCodecVideoEncoder::EncodeByteBuffer(JNIEnv* jni, - bool key_frame, - const VideoFrame& frame, - int input_buffer_index) { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - RTC_CHECK(!use_surface_); - - rtc::scoped_refptr i420_buffer = - frame.video_frame_buffer()->ToI420(); - if (!FillInputBuffer(jni, input_buffer_index, i420_buffer->DataY(), - i420_buffer->StrideY(), i420_buffer->DataU(), - i420_buffer->StrideU(), i420_buffer->DataV(), - i420_buffer->StrideV())) { - return false; - } - bool encode_status = Java_MediaCodecVideoEncoder_encodeBuffer( - jni, j_media_codec_video_encoder_, key_frame, input_buffer_index, - yuv_size_, current_timestamp_us_); - if (CheckException(jni)) { - ALOGE << "Exception in encode buffer."; - ProcessHWError(true /* reset_if_fallback_unavailable */); - return false; - } - return encode_status; -} - -bool MediaCodecVideoEncoder::FillInputBuffer(JNIEnv* jni, - int input_buffer_index, - uint8_t const* buffer_y, - int stride_y, - uint8_t const* buffer_u, - int stride_u, - uint8_t const* buffer_v, - int stride_v) { - uint8_t* yuv_buffer = reinterpret_cast( - jni->GetDirectBufferAddress(input_buffers_[input_buffer_index].obj())); - if (CheckException(jni)) { - ALOGE << "Exception in get direct buffer address."; - ProcessHWError(true /* reset_if_fallback_unavailable */); - return false; - } - RTC_CHECK(yuv_buffer) << "Indirect buffer??"; - - RTC_CHECK(!libyuv::ConvertFromI420(buffer_y, stride_y, buffer_u, stride_u, - buffer_v, stride_v, yuv_buffer, width_, - width_, height_, encoder_fourcc_)) - << "ConvertFromI420 failed"; - return true; -} - -bool MediaCodecVideoEncoder::EncodeJavaFrame(JNIEnv* jni, - bool key_frame, - const JavaRef& frame, - int input_buffer_index) { - bool encode_status = Java_MediaCodecVideoEncoder_encodeFrame( - jni, j_media_codec_video_encoder_, jlongFromPointer(this), key_frame, - frame, input_buffer_index, current_timestamp_us_); - if (CheckException(jni)) { - ALOGE << "Exception in encode frame."; - ProcessHWError(true /* reset_if_fallback_unavailable */); - return false; - } - return encode_status; -} - -int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback( - EncodedImageCallback* callback) { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - callback_ = callback; - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t MediaCodecVideoEncoder::Release() { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - if (!inited_) { - return WEBRTC_VIDEO_CODEC_OK; - } - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ALOGD << "EncoderRelease: Frames received: " << frames_received_ - << ". Encoded: " << frames_encoded_ - << ". Dropped: " << frames_dropped_media_encoder_; - encode_task_.reset(nullptr); - weak_factory_.reset(nullptr); - ScopedLocalRefFrame local_ref_frame(jni); - input_buffers_.clear(); - Java_MediaCodecVideoEncoder_release(jni, j_media_codec_video_encoder_); - if (CheckException(jni)) { - ALOGE << "Exception in release."; - ProcessHWError(false /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - { -#if RTC_DCHECK_IS_ON - rtc::CritScope lock(&inited_crit_); -#endif - inited_ = false; - } - use_surface_ = false; - ALOGD << "EncoderRelease done."; - // It's legal to move the encoder to another queue now. - encoder_queue_checker_.Detach(); - return WEBRTC_VIDEO_CODEC_OK; -} - -void MediaCodecVideoEncoder::SetRates(const RateControlParameters& parameters) { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - const uint32_t new_bit_rate = parameters.bitrate.get_sum_kbps(); - if (sw_fallback_required_) - return; - uint32_t frame_rate = static_cast(parameters.framerate_fps + 0.5); - frame_rate = - (frame_rate < MAX_ALLOWED_VIDEO_FPS) ? frame_rate : MAX_ALLOWED_VIDEO_FPS; - if (last_set_bitrate_kbps_ == new_bit_rate && last_set_fps_ == frame_rate) { - return; - } - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - if (new_bit_rate > 0) { - last_set_bitrate_kbps_ = new_bit_rate; - } - if (frame_rate > 0) { - last_set_fps_ = frame_rate; - } - bool ret = Java_MediaCodecVideoEncoder_setRates( - jni, j_media_codec_video_encoder_, - rtc::dchecked_cast(last_set_bitrate_kbps_), - rtc::dchecked_cast(last_set_fps_)); - if (CheckException(jni) || !ret) { - ProcessHWError(true /* reset_if_fallback_unavailable */); - } -} - -VideoEncoder::EncoderInfo MediaCodecVideoEncoder::GetEncoderInfo() const { - return encoder_info_; -} - -bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { - RTC_DCHECK_RUN_ON(&encoder_queue_checker_); - - while (true) { - ScopedJavaLocalRef j_output_buffer_info = - Java_MediaCodecVideoEncoder_dequeueOutputBuffer( - jni, j_media_codec_video_encoder_); - if (CheckException(jni)) { - ALOGE << "Exception in set dequeue output buffer."; - ProcessHWError(true /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - if (IsNull(jni, j_output_buffer_info)) { - break; - } - - int output_buffer_index = - Java_OutputBufferInfo_getIndex(jni, j_output_buffer_info); - if (output_buffer_index == -1) { - ProcessHWError(true /* reset_if_fallback_unavailable */); - return false; - } - - // Get key and config frame flags. - ScopedJavaLocalRef j_output_buffer = - Java_OutputBufferInfo_getBuffer(jni, j_output_buffer_info); - bool key_frame = - Java_OutputBufferInfo_isKeyFrame(jni, j_output_buffer_info); - - // Get frame timestamps from a queue - for non config frames only. - int64_t encoding_start_time_ms = 0; - int64_t frame_encoding_time_ms = 0; - last_output_timestamp_ms_ = - Java_OutputBufferInfo_getPresentationTimestampUs(jni, - j_output_buffer_info) / - rtc::kNumMicrosecsPerMillisec; - if (!input_frame_infos_.empty()) { - const InputFrameInfo& frame_info = input_frame_infos_.front(); - output_timestamp_ = frame_info.frame_timestamp; - output_render_time_ms_ = frame_info.frame_render_time_ms; - output_rotation_ = frame_info.rotation; - encoding_start_time_ms = frame_info.encode_start_time; - input_frame_infos_.pop_front(); - } - - // Extract payload. - size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer.obj()); - uint8_t* payload = reinterpret_cast( - jni->GetDirectBufferAddress(j_output_buffer.obj())); - if (CheckException(jni)) { - ALOGE << "Exception in get direct buffer address."; - ProcessHWError(true /* reset_if_fallback_unavailable */); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - // Callback - return encoded frame. - const VideoCodecType codec_type = GetCodecType(); - EncodedImageCallback::Result callback_result( - EncodedImageCallback::Result::OK); - if (callback_) { - auto image = std::make_unique(); - // The corresponding (and deprecated) java classes are not prepared for - // late calls to releaseOutputBuffer, so to keep things simple, make a - // copy here, and call releaseOutputBuffer before returning. - image->SetEncodedData(EncodedImageBuffer::Create(payload, payload_size)); - image->_encodedWidth = width_; - image->_encodedHeight = height_; - image->SetTimestamp(output_timestamp_); - image->capture_time_ms_ = output_render_time_ms_; - image->rotation_ = output_rotation_; - image->content_type_ = (codec_mode_ == VideoCodecMode::kScreensharing) - ? VideoContentType::SCREENSHARE - : VideoContentType::UNSPECIFIED; - image->timing_.flags = VideoSendTiming::kInvalid; - image->_frameType = (key_frame ? VideoFrameType::kVideoFrameKey - : VideoFrameType::kVideoFrameDelta); - image->_completeFrame = true; - CodecSpecificInfo info; - memset(&info, 0, sizeof(info)); - info.codecType = codec_type; - if (codec_type == kVideoCodecVP8) { - info.codecSpecific.VP8.nonReference = false; - info.codecSpecific.VP8.temporalIdx = kNoTemporalIdx; - info.codecSpecific.VP8.layerSync = false; - info.codecSpecific.VP8.keyIdx = kNoKeyIdx; - } else if (codec_type == kVideoCodecVP9) { - if (key_frame) { - gof_idx_ = 0; - } - info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true; - info.codecSpecific.VP9.flexible_mode = false; - info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; - info.codecSpecific.VP9.temporal_idx = kNoTemporalIdx; - info.codecSpecific.VP9.temporal_up_switch = true; - info.codecSpecific.VP9.inter_layer_predicted = false; - info.codecSpecific.VP9.gof_idx = - static_cast(gof_idx_++ % gof_.num_frames_in_gof); - info.codecSpecific.VP9.num_spatial_layers = 1; - info.codecSpecific.VP9.first_frame_in_picture = true; - info.codecSpecific.VP9.end_of_picture = true; - info.codecSpecific.VP9.spatial_layer_resolution_present = false; - if (info.codecSpecific.VP9.ss_data_available) { - info.codecSpecific.VP9.spatial_layer_resolution_present = true; - info.codecSpecific.VP9.width[0] = width_; - info.codecSpecific.VP9.height[0] = height_; - info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); - } - } - - // Generate a header describing a single fragment. - RTPFragmentationHeader header; - memset(&header, 0, sizeof(header)); - if (codec_type == kVideoCodecVP8 || codec_type == kVideoCodecVP9) { - header.VerifyAndAllocateFragmentationHeader(1); - header.fragmentationOffset[0] = 0; - header.fragmentationLength[0] = image->size(); - if (codec_type == kVideoCodecVP8) { - int qp; - if (vp8::GetQp(payload, payload_size, &qp)) { - current_acc_qp_ += qp; - image->qp_ = qp; - } - } else if (codec_type == kVideoCodecVP9) { - int qp; - if (vp9::GetQp(payload, payload_size, &qp)) { - current_acc_qp_ += qp; - image->qp_ = qp; - } - } - } else if (codec_type == kVideoCodecH264) { - h264_bitstream_parser_.ParseBitstream(payload, payload_size); - int qp; - if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { - current_acc_qp_ += qp; - image->qp_ = qp; - } - // For H.264 search for start codes. - const std::vector nalu_idxs = - H264::FindNaluIndices(payload, payload_size); - if (nalu_idxs.empty()) { - ALOGE << "Start code is not found!"; - ALOGE << "Data:" << image->data()[0] << " " << image->data()[1] << " " - << image->data()[2] << " " << image->data()[3] << " " - << image->data()[4] << " " << image->data()[5]; - ProcessHWError(true /* reset_if_fallback_unavailable */); - return false; - } - header.VerifyAndAllocateFragmentationHeader(nalu_idxs.size()); - for (size_t i = 0; i < nalu_idxs.size(); i++) { - header.fragmentationOffset[i] = nalu_idxs[i].payload_start_offset; - header.fragmentationLength[i] = nalu_idxs[i].payload_size; - } - } - - callback_result = callback_->OnEncodedImage(*image, &info, &header); - } - - // Return output buffer back to the encoder. - bool success = Java_MediaCodecVideoEncoder_releaseOutputBuffer( - jni, j_media_codec_video_encoder_, output_buffer_index); - if (CheckException(jni) || !success) { - ProcessHWError(true /* reset_if_fallback_unavailable */); - return false; - } - - // Print per frame statistics. - if (encoding_start_time_ms > 0) { - frame_encoding_time_ms = rtc::TimeMillis() - encoding_start_time_ms; - } - if (frames_encoded_ < kMaxEncodedLogFrames) { - int current_latency = static_cast(last_input_timestamp_ms_ - - last_output_timestamp_ms_); - ALOGD << "Encoder frame out # " << frames_encoded_ - << ". Key: " << key_frame << ". Size: " << payload_size - << ". TS: " << static_cast(last_output_timestamp_ms_) - << ". Latency: " << current_latency - << ". EncTime: " << frame_encoding_time_ms; - } - - // Calculate and print encoding statistics - every 3 seconds. - frames_encoded_++; - current_frames_++; - current_bytes_ += payload_size; - current_encoding_time_ms_ += frame_encoding_time_ms; - LogStatistics(false); - - // Errors in callback_result are currently ignored. - if (callback_result.drop_next_frame) - drop_next_input_frame_ = true; - } - return true; -} - -void MediaCodecVideoEncoder::LogStatistics(bool force_log) { - int statistic_time_ms = rtc::TimeMillis() - stat_start_time_ms_; - if ((statistic_time_ms >= kMediaCodecStatisticsIntervalMs || force_log) && - statistic_time_ms > 0) { - // Prevent division by zero. - int current_frames_divider = current_frames_ != 0 ? current_frames_ : 1; - - int current_bitrate = current_bytes_ * 8 / statistic_time_ms; - int current_fps = - (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms; - ALOGD << "Encoded frames: " << frames_encoded_ - << ". Bitrate: " << current_bitrate - << ", target: " << last_set_bitrate_kbps_ - << " kbps" - ", fps: " - << current_fps << ", encTime: " - << (current_encoding_time_ms_ / current_frames_divider) - << ". QP: " << (current_acc_qp_ / current_frames_divider) - << " for last " << statistic_time_ms << " ms."; - stat_start_time_ms_ = rtc::TimeMillis(); - current_frames_ = 0; - current_bytes_ = 0; - current_acc_qp_ = 0; - current_encoding_time_ms_ = 0; - } -} - -VideoEncoder::ScalingSettings -MediaCodecVideoEncoder::GetScalingSettingsInternal() const { - if (!scale_) - return VideoEncoder::ScalingSettings::kOff; - - const VideoCodecType codec_type = GetCodecType(); - if (field_trial::IsEnabled(kCustomQPThresholdsFieldTrial)) { - std::string experiment_string = - field_trial::FindFullName(kCustomQPThresholdsFieldTrial); - ALOGD << "QP custom thresholds: " << experiment_string << " for codec " - << codec_type; - int low_vp8_qp_threshold; - int high_vp8_qp_threshold; - int low_h264_qp_threshold; - int high_h264_qp_threshold; - int parsed_values = sscanf(experiment_string.c_str(), "Enabled-%u,%u,%u,%u", - &low_vp8_qp_threshold, &high_vp8_qp_threshold, - &low_h264_qp_threshold, &high_h264_qp_threshold); - if (parsed_values == 4) { - RTC_CHECK_GT(high_vp8_qp_threshold, low_vp8_qp_threshold); - RTC_CHECK_GT(low_vp8_qp_threshold, 0); - RTC_CHECK_GT(high_h264_qp_threshold, low_h264_qp_threshold); - RTC_CHECK_GT(low_h264_qp_threshold, 0); - if (codec_type == kVideoCodecVP8) { - return VideoEncoder::ScalingSettings(low_vp8_qp_threshold, - high_vp8_qp_threshold); - } else if (codec_type == kVideoCodecH264) { - return VideoEncoder::ScalingSettings(low_h264_qp_threshold, - high_h264_qp_threshold); - } - } - } - if (codec_type == kVideoCodecVP8) { - // Same as in vp8_impl.cc. - static const int kLowVp8QpThreshold = 29; - static const int kHighVp8QpThreshold = 95; - - return VideoEncoder::ScalingSettings(kLowVp8QpThreshold, - kHighVp8QpThreshold); - } else if (codec_type == kVideoCodecVP9) { - // QP is obtained from VP9-bitstream, so the QP corresponds to the bitstream - // range of [0, 255] and not the user-level range of [0,63]. - static const int kLowVp9QpThreshold = 96; - static const int kHighVp9QpThreshold = 185; - - return VideoEncoder::ScalingSettings(kLowVp9QpThreshold, - kHighVp9QpThreshold); - } else if (codec_type == kVideoCodecH264) { - // Same as in h264_encoder_impl.cc. - static const int kLowH264QpThreshold = 24; - static const int kHighH264QpThreshold = 37; - - return VideoEncoder::ScalingSettings(kLowH264QpThreshold, - kHighH264QpThreshold); - } - return VideoEncoder::ScalingSettings::kOff; -} - -static void JNI_MediaCodecVideoEncoder_FillInputBuffer( - JNIEnv* jni, - jlong native_encoder, - jint input_buffer, - const JavaParamRef& j_buffer_y, - jint stride_y, - const JavaParamRef& j_buffer_u, - jint stride_u, - const JavaParamRef& j_buffer_v, - jint stride_v) { - uint8_t* buffer_y = - static_cast(jni->GetDirectBufferAddress(j_buffer_y.obj())); - uint8_t* buffer_u = - static_cast(jni->GetDirectBufferAddress(j_buffer_u.obj())); - uint8_t* buffer_v = - static_cast(jni->GetDirectBufferAddress(j_buffer_v.obj())); - - RTC_DCHECK(buffer_y) << "GetDirectBufferAddress returned null. Ensure that " - "getDataY returns a direct ByteBuffer."; - RTC_DCHECK(buffer_u) << "GetDirectBufferAddress returned null. Ensure that " - "getDataU returns a direct ByteBuffer."; - RTC_DCHECK(buffer_v) << "GetDirectBufferAddress returned null. Ensure that " - "getDataV returns a direct ByteBuffer."; - - reinterpret_cast(native_encoder) - ->FillInputBuffer(jni, input_buffer, buffer_y, stride_y, buffer_u, - stride_u, buffer_v, stride_v); -} - -static jlong JNI_MediaCodecVideoEncoder_CreateEncoder( - JNIEnv* env, - const JavaParamRef& format, - jboolean has_egl_context) { - ScopedLocalRefFrame local_ref_frame(env); - return jlongFromPointer(new MediaCodecVideoEncoder( - env, VideoCodecInfoToSdpVideoFormat(env, format), has_egl_context)); -} - -} // namespace jni -} // namespace webrtc