diff --git a/webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java b/webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java index 72544bd83f..923ea91441 100644 --- a/webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java +++ b/webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java @@ -52,6 +52,12 @@ public class MediaCodecVideoEncoder { private static final int DEQUEUE_TIMEOUT = 0; // Non-blocking, no wait. private static final int BITRATE_ADJUSTMENT_FPS = 30; private static final int MAXIMUM_INITIAL_FPS = 30; + private static final double BITRATE_CORRECTION_SEC = 3.0; + // Maximum bitrate correction scale - no more than 2 times. + private static final double BITRATE_CORRECTION_MAX_SCALE = 2; + // Amount of correction steps to reach correction maximum scale. + private static final int BITRATE_CORRECTION_STEPS = 10; + // Active running encoder instance. Set in initEncode() (called from native code) // and reset to null in release() call. private static MediaCodecVideoEncoder runningInstance = null; @@ -73,6 +79,19 @@ public class MediaCodecVideoEncoder { private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9"; private static final String H264_MIME_TYPE = "video/avc"; + // Type of bitrate adjustment for video encoder. + public enum BitrateAdjustmentType { + // No adjustment - video encoder has no known bitrate problem. + NO_ADJUSTMENT, + // Framerate based bitrate adjustment is required - HW encoder does not use frame + // timestamps to calculate frame bitrate budget and instead is relying on initial + // fps configuration assuming that all frames are coming at fixed initial frame rate. + FRAMERATE_ADJUSTMENT, + // Dynamic bitrate adjustment is required - HW encoder used frame timestamps, but actual + // bitrate deviates too much from the target value. + DYNAMIC_ADJUSTMENT + } + // Class describing supported media codec properties. private static class MediaCodecProperties { public final String codecPrefix; @@ -81,39 +100,39 @@ public class MediaCodecVideoEncoder { // Flag if encoder implementation does not use frame timestamps to calculate frame bitrate // budget and instead is relying on initial fps configuration assuming that all frames are // coming at fixed initial frame rate. Bitrate adjustment is required for this case. - public final boolean bitrateAdjustmentRequired; + public final BitrateAdjustmentType bitrateAdjustmentType; MediaCodecProperties( - String codecPrefix, int minSdk, boolean bitrateAdjustmentRequired) { + String codecPrefix, int minSdk, BitrateAdjustmentType bitrateAdjustmentType) { this.codecPrefix = codecPrefix; this.minSdk = minSdk; - this.bitrateAdjustmentRequired = bitrateAdjustmentRequired; + this.bitrateAdjustmentType = bitrateAdjustmentType; } } // List of supported HW VP8 encoders. private static final MediaCodecProperties qcomVp8HwProperties = new MediaCodecProperties( - "OMX.qcom.", Build.VERSION_CODES.KITKAT, false /* bitrateAdjustmentRequired */); + "OMX.qcom.", Build.VERSION_CODES.KITKAT, BitrateAdjustmentType.NO_ADJUSTMENT); private static final MediaCodecProperties exynosVp8HwProperties = new MediaCodecProperties( - "OMX.Exynos.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */); + "OMX.Exynos.", Build.VERSION_CODES.M, BitrateAdjustmentType.DYNAMIC_ADJUSTMENT); private static final MediaCodecProperties[] vp8HwList = new MediaCodecProperties[] { qcomVp8HwProperties, exynosVp8HwProperties }; // List of supported HW VP9 encoders. private static final MediaCodecProperties qcomVp9HwProperties = new MediaCodecProperties( - "OMX.qcom.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */); + "OMX.qcom.", Build.VERSION_CODES.M, BitrateAdjustmentType.NO_ADJUSTMENT); private static final MediaCodecProperties exynosVp9HwProperties = new MediaCodecProperties( - "OMX.Exynos.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */); + "OMX.Exynos.", Build.VERSION_CODES.M, BitrateAdjustmentType.NO_ADJUSTMENT); private static final MediaCodecProperties[] vp9HwList = new MediaCodecProperties[] { qcomVp9HwProperties, exynosVp9HwProperties }; // List of supported HW H.264 encoders. private static final MediaCodecProperties qcomH264HwProperties = new MediaCodecProperties( - "OMX.qcom.", Build.VERSION_CODES.KITKAT, false /* bitrateAdjustmentRequired */); + "OMX.qcom.", Build.VERSION_CODES.KITKAT, BitrateAdjustmentType.NO_ADJUSTMENT); private static final MediaCodecProperties exynosH264HwProperties = new MediaCodecProperties( - "OMX.Exynos.", Build.VERSION_CODES.LOLLIPOP, true /* bitrateAdjustmentRequired */); + "OMX.Exynos.", Build.VERSION_CODES.LOLLIPOP, BitrateAdjustmentType.FRAMERATE_ADJUSTMENT); private static final MediaCodecProperties[] h264HwList = new MediaCodecProperties[] { qcomH264HwProperties, exynosH264HwProperties }; @@ -146,7 +165,15 @@ public class MediaCodecVideoEncoder { }; private VideoCodecType type; private int colorFormat; // Used by native code. - private boolean bitrateAdjustmentRequired; + + // Variables used for dynamic bitrate adjustment. + private BitrateAdjustmentType bitrateAdjustmentType = BitrateAdjustmentType.NO_ADJUSTMENT; + private double bitrateAccumulator; + private double bitrateAccumulatorMax; + private double bitrateObservationTimeMs; + private int bitrateAdjustmentScaleExp; + private int targetBitrateBps; + private int targetFps; // SPS and PPS NALs (Config frame) for H.264. private ByteBuffer configData = null; @@ -213,14 +240,15 @@ public class MediaCodecVideoEncoder { // Helper struct for findHwEncoder() below. private static class EncoderProperties { - public EncoderProperties(String codecName, int colorFormat, boolean bitrateAdjustment) { + public EncoderProperties( + String codecName, int colorFormat, BitrateAdjustmentType bitrateAdjustmentType) { this.codecName = codecName; this.colorFormat = colorFormat; - this.bitrateAdjustment = bitrateAdjustment; + this.bitrateAdjustmentType = bitrateAdjustmentType; } public final String codecName; // OpenMax component name for HW codec. public final int colorFormat; // Color format supported by codec. - public final boolean bitrateAdjustment; // true if bitrate adjustment workaround is required. + public final BitrateAdjustmentType bitrateAdjustmentType; // Bitrate adjustment type } private static EncoderProperties findHwEncoder( @@ -264,7 +292,7 @@ public class MediaCodecVideoEncoder { // Check if this is supported HW encoder. boolean supportedCodec = false; - boolean bitrateAdjustmentRequired = false; + BitrateAdjustmentType bitrateAdjustmentType = BitrateAdjustmentType.NO_ADJUSTMENT; for (MediaCodecProperties codecProperties : supportedHwCodecProperties) { if (name.startsWith(codecProperties.codecPrefix)) { if (Build.VERSION.SDK_INT < codecProperties.minSdk) { @@ -272,9 +300,10 @@ public class MediaCodecVideoEncoder { Build.VERSION.SDK_INT); continue; } - if (codecProperties.bitrateAdjustmentRequired) { - Logging.w(TAG, "Codec " + name + " does not use frame timestamps."); - bitrateAdjustmentRequired = true; + if (codecProperties.bitrateAdjustmentType != BitrateAdjustmentType.NO_ADJUSTMENT) { + bitrateAdjustmentType = codecProperties.bitrateAdjustmentType; + Logging.w(TAG, "Codec " + name + + " requires bitrate adjustment: " + bitrateAdjustmentType); } supportedCodec = true; break; @@ -300,9 +329,10 @@ public class MediaCodecVideoEncoder { for (int codecColorFormat : capabilities.colorFormats) { if (codecColorFormat == supportedColorFormat) { // Found supported HW encoder. - Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name + - ". Color: 0x" + Integer.toHexString(codecColorFormat)); - return new EncoderProperties(name, codecColorFormat, bitrateAdjustmentRequired); + Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name + + ". Color: 0x" + Integer.toHexString(codecColorFormat) + + ". Bitrate adjustment: " + bitrateAdjustmentType); + return new EncoderProperties(name, codecColorFormat, bitrateAdjustmentType); } } } @@ -375,23 +405,29 @@ public class MediaCodecVideoEncoder { } runningInstance = this; // Encoder is now running and can be queried for stack traces. colorFormat = properties.colorFormat; - bitrateAdjustmentRequired = properties.bitrateAdjustment; - if (bitrateAdjustmentRequired) { + bitrateAdjustmentType = properties.bitrateAdjustmentType; + if (bitrateAdjustmentType == BitrateAdjustmentType.FRAMERATE_ADJUSTMENT) { fps = BITRATE_ADJUSTMENT_FPS; } else { fps = Math.min(fps, MAXIMUM_INITIAL_FPS); } Logging.d(TAG, "Color format: " + colorFormat + - ". Bitrate adjustment: " + bitrateAdjustmentRequired + + ". Bitrate adjustment: " + bitrateAdjustmentType + ". Initial fps: " + fps); + targetBitrateBps = 1000 * kbps; + targetFps = fps; + bitrateAccumulatorMax = targetBitrateBps / 8.0; + bitrateAccumulator = 0; + bitrateObservationTimeMs = 0; + bitrateAdjustmentScaleExp = 0; mediaCodecThread = Thread.currentThread(); try { MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); - format.setInteger(MediaFormat.KEY_BIT_RATE, 1000 * kbps); + format.setInteger(MediaFormat.KEY_BIT_RATE, targetBitrateBps); format.setInteger("bitrate-mode", VIDEO_ControlRateConstant); format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); - format.setInteger(MediaFormat.KEY_FRAME_RATE, fps); + format.setInteger(MediaFormat.KEY_FRAME_RATE, targetFps); format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec); Logging.d(TAG, " Format: " + format); mediaCodec = createByCodecName(properties.codecName); @@ -529,17 +565,36 @@ public class MediaCodecVideoEncoder { private boolean setRates(int kbps, int frameRate) { checkOnMediaCodecThread(); - int codecBitrate = 1000 * kbps; - if (bitrateAdjustmentRequired && frameRate > 0) { - codecBitrate = BITRATE_ADJUSTMENT_FPS * codecBitrate / frameRate; - Logging.v(TAG, "setRates: " + kbps + " -> " + (codecBitrate / 1000) - + " kbps. Fps: " + frameRate); - } else { - Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + frameRate); + + int codecBitrateBps = 1000 * kbps; + if (bitrateAdjustmentType == BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) { + bitrateAccumulatorMax = codecBitrateBps / 8.0; + if (targetBitrateBps > 0 && codecBitrateBps < targetBitrateBps) { + // Rescale the accumulator level if the accumulator max decreases + bitrateAccumulator = bitrateAccumulator * codecBitrateBps / targetBitrateBps; + } } + targetBitrateBps = codecBitrateBps; + targetFps = frameRate; + + // Adjust actual encoder bitrate based on bitrate adjustment type. + if (bitrateAdjustmentType == BitrateAdjustmentType.FRAMERATE_ADJUSTMENT && targetFps > 0) { + codecBitrateBps = BITRATE_ADJUSTMENT_FPS * targetBitrateBps / targetFps; + Logging.v(TAG, "setRates: " + kbps + " -> " + (codecBitrateBps / 1000) + + " kbps. Fps: " + targetFps); + } else if (bitrateAdjustmentType == BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) { + Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + targetFps + + ". ExpScale: " + bitrateAdjustmentScaleExp); + if (bitrateAdjustmentScaleExp != 0) { + codecBitrateBps = (int)(codecBitrateBps * getBitrateScale(bitrateAdjustmentScaleExp)); + } + } else { + Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + targetFps); + } + try { Bundle params = new Bundle(); - params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, codecBitrate); + params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, codecBitrateBps); mediaCodec.setParameters(params); return true; } catch (IllegalStateException e) { @@ -608,6 +663,8 @@ public class MediaCodecVideoEncoder { ByteBuffer outputBuffer = outputBuffers[result].duplicate(); outputBuffer.position(info.offset); outputBuffer.limit(info.offset + info.size); + reportEncodedFrame(info.size); + // Check key frame flag. boolean isKeyFrame = (info.flags & MediaCodec.BUFFER_FLAG_SYNC_FRAME) != 0; @@ -646,6 +703,56 @@ public class MediaCodecVideoEncoder { } } + private double getBitrateScale(int bitrateAdjustmentScaleExp) { + return Math.pow(BITRATE_CORRECTION_MAX_SCALE, + (double)bitrateAdjustmentScaleExp / BITRATE_CORRECTION_STEPS); + } + + private void reportEncodedFrame(int size) { + if (targetFps == 0 || bitrateAdjustmentType != BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) { + return; + } + + // Accumulate the difference between actial and expected frame sizes. + double expectedBytesPerFrame = targetBitrateBps / (8.0 * targetFps); + bitrateAccumulator += (size - expectedBytesPerFrame); + bitrateObservationTimeMs += 1000.0 / targetFps; + + // Put a cap on the accumulator, i.e., don't let it grow beyond some level to avoid + // using too old data for bitrate adjustment. + double bitrateAccumulatorCap = BITRATE_CORRECTION_SEC * bitrateAccumulatorMax; + bitrateAccumulator = Math.min(bitrateAccumulator, bitrateAccumulatorCap); + bitrateAccumulator = Math.max(bitrateAccumulator, -bitrateAccumulatorCap); + + // Do bitrate adjustment every 3 seconds if actual encoder bitrate deviates too much + // form the target value. + if (bitrateObservationTimeMs > 1000 * BITRATE_CORRECTION_SEC) { + Logging.d(TAG, "Acc: " + (int)bitrateAccumulator + + ". Max: " + (int)bitrateAccumulatorMax + + ". ExpScale: " + bitrateAdjustmentScaleExp); + boolean bitrateAdjustmentScaleChanged = false; + if (bitrateAccumulator > bitrateAccumulatorMax) { + // Encoder generates too high bitrate - need to reduce the scale. + bitrateAccumulator = bitrateAccumulatorMax; + bitrateAdjustmentScaleExp--; + bitrateAdjustmentScaleChanged = true; + } else if (bitrateAccumulator < -bitrateAccumulatorMax) { + // Encoder generates too low bitrate - need to increase the scale. + bitrateAdjustmentScaleExp++; + bitrateAccumulator = -bitrateAccumulatorMax; + bitrateAdjustmentScaleChanged = true; + } + if (bitrateAdjustmentScaleChanged) { + bitrateAdjustmentScaleExp = Math.min(bitrateAdjustmentScaleExp, BITRATE_CORRECTION_STEPS); + bitrateAdjustmentScaleExp = Math.max(bitrateAdjustmentScaleExp, -BITRATE_CORRECTION_STEPS); + Logging.d(TAG, "Adjusting bitrate scale to " + bitrateAdjustmentScaleExp + + ". Value: " + getBitrateScale(bitrateAdjustmentScaleExp)); + setRates(targetBitrateBps / 1000, targetFps); + } + bitrateObservationTimeMs = 0; + } + } + // Release a dequeued output buffer back to the codec for re-use. Return // false if the codec is no longer operable. boolean releaseOutputBuffer(int index) {