From e2410e9ab474abc1ef43bc658f76971eaf94fb07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sami=20Kalliom=C3=A4ki?= Date: Fri, 2 Jun 2017 14:46:12 +0200 Subject: [PATCH] Interfaces for injectable video codecs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These interfaces will be used by the future refactoring that will allow clients to provide custom codec implementations. Change-Id: If199bc2807e1c27094c05983c62fa43d2eec5700 Bug: webrtc:7760 Reviewed-on: https://chromium-review.googlesource.com/522065 Commit-Queue: Sami Kalliomäki Reviewed-by: Magnus Jedvert Reviewed-by: Bjorn Mellem Cr-Commit-Position: refs/heads/master@{#18441} --- webrtc/sdk/android/BUILD.gn | 4 + .../android/api/org/webrtc/EncodedImage.java | 116 ++++++++++++++++ .../api/org/webrtc/VideoCodecInfo.java | 28 ++++ .../android/api/org/webrtc/VideoDecoder.java | 69 ++++++++++ .../android/api/org/webrtc/VideoEncoder.java | 126 ++++++++++++++++++ 5 files changed, 343 insertions(+) create mode 100644 webrtc/sdk/android/api/org/webrtc/EncodedImage.java create mode 100644 webrtc/sdk/android/api/org/webrtc/VideoCodecInfo.java create mode 100644 webrtc/sdk/android/api/org/webrtc/VideoDecoder.java create mode 100644 webrtc/sdk/android/api/org/webrtc/VideoEncoder.java diff --git a/webrtc/sdk/android/BUILD.gn b/webrtc/sdk/android/BUILD.gn index c766f76938..cd9de1747f 100644 --- a/webrtc/sdk/android/BUILD.gn +++ b/webrtc/sdk/android/BUILD.gn @@ -158,6 +158,7 @@ android_library("libjingle_peerconnection_java") { "api/org/webrtc/DtmfSender.java", "api/org/webrtc/EglBase.java", "api/org/webrtc/EglRenderer.java", + "api/org/webrtc/EncodedImage.java", "api/org/webrtc/FileVideoCapturer.java", "api/org/webrtc/GlRectDrawer.java", "api/org/webrtc/GlShader.java", @@ -189,6 +190,9 @@ android_library("libjingle_peerconnection_java") { "api/org/webrtc/SurfaceTextureHelper.java", "api/org/webrtc/SurfaceViewRenderer.java", "api/org/webrtc/VideoCapturer.java", + "api/org/webrtc/VideoCodecInfo.java", + "api/org/webrtc/VideoDecoder.java", + "api/org/webrtc/VideoEncoder.java", "api/org/webrtc/VideoFileRenderer.java", "api/org/webrtc/VideoFrame.java", "api/org/webrtc/VideoRenderer.java", diff --git a/webrtc/sdk/android/api/org/webrtc/EncodedImage.java b/webrtc/sdk/android/api/org/webrtc/EncodedImage.java new file mode 100644 index 0000000000..d0b70e36f0 --- /dev/null +++ b/webrtc/sdk/android/api/org/webrtc/EncodedImage.java @@ -0,0 +1,116 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +/** + * An encoded frame from a video stream. Used as an input for decoders and as an output for + * encoders. + */ +public class EncodedImage { + public enum FrameType { + EmptyFrame, + VideoFrameKey, + VideoFrameDelta, + } + + public final ByteBuffer buffer; + public final int encodedWidth; + public final int encodedHeight; + public final long timeStampMs; + public final long captureTimeMs; + public final FrameType frameType; + public final int rotation; + public final boolean completeFrame; + public final Integer qp; + + private EncodedImage(ByteBuffer buffer, int encodedWidth, int encodedHeight, long timeStampMs, + long captureTimeMs, FrameType frameType, int rotation, boolean completeFrame, Integer qp) { + this.buffer = buffer; + this.encodedWidth = encodedWidth; + this.encodedHeight = encodedHeight; + this.timeStampMs = timeStampMs; + this.captureTimeMs = captureTimeMs; + this.frameType = frameType; + this.rotation = rotation; + this.completeFrame = completeFrame; + this.qp = qp; + } + + public Builder builder() { + return new Builder(); + } + + public class Builder { + private ByteBuffer buffer; + private int encodedWidth; + private int encodedHeight; + private long timeStampMs; + private long captureTimeMs; + private EncodedImage.FrameType frameType; + private int rotation; + private boolean completeFrame; + private Integer qp; + + private Builder() {} + + public Builder setBuffer(ByteBuffer buffer) { + this.buffer = buffer; + return this; + } + + public Builder setEncodedWidth(int encodedWidth) { + this.encodedWidth = encodedWidth; + return this; + } + + public Builder setEncodedHeight(int encodedHeight) { + this.encodedHeight = encodedHeight; + return this; + } + + public Builder setTimeStampMs(long timeStampMs) { + this.timeStampMs = timeStampMs; + return this; + } + + public Builder setCaptureTimeMs(long captureTimeMs) { + this.captureTimeMs = captureTimeMs; + return this; + } + + public Builder setFrameType(EncodedImage.FrameType frameType) { + this.frameType = frameType; + return this; + } + + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + public Builder setCompleteFrame(boolean completeFrame) { + this.completeFrame = completeFrame; + return this; + } + + public Builder setQp(Integer qp) { + this.qp = qp; + return this; + } + + public EncodedImage createEncodedImage() { + return new EncodedImage(buffer, encodedWidth, encodedHeight, timeStampMs, captureTimeMs, + frameType, rotation, completeFrame, qp); + } + } +} diff --git a/webrtc/sdk/android/api/org/webrtc/VideoCodecInfo.java b/webrtc/sdk/android/api/org/webrtc/VideoCodecInfo.java new file mode 100644 index 0000000000..d8ac182e3f --- /dev/null +++ b/webrtc/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.util.Map; + +/** + * Represent a video codec as encoded in SDP. + */ +public class VideoCodecInfo { + public final int payload; + public final String name; + public final Map params; + + public VideoCodecInfo(int payload, String name, Map params) { + this.payload = payload; + this.name = name; + this.params = params; + } +} diff --git a/webrtc/sdk/android/api/org/webrtc/VideoDecoder.java b/webrtc/sdk/android/api/org/webrtc/VideoDecoder.java new file mode 100644 index 0000000000..e75975ce61 --- /dev/null +++ b/webrtc/sdk/android/api/org/webrtc/VideoDecoder.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Interface for a video decoder that can be used in WebRTC. All calls to the class will be made on + * a single decoding thread. + */ +public interface VideoDecoder { + /** Settings passed to the decoder by WebRTC. */ + public class Settings { + public final int numberOfCores; + + public Settings(int numberOfCores) { + this.numberOfCores = numberOfCores; + } + } + + /** Additional info for decoding. */ + public class DecodeInfo { + public final boolean isMissingFrames; + public final long renderTimeMs; + + public DecodeInfo(boolean isMissingFrames, long renderTimeMs) { + this.isMissingFrames = isMissingFrames; + this.renderTimeMs = renderTimeMs; + } + } + + public interface Callback { + /** + * Call to return a decoded frame. Can be called on any thread. + * + * @param frame Decoded frame + * @param decodeTimeMs Time it took to decode the frame in milliseconds or null if not available + * @param qp QP value of the decoded frame or null if not available + */ + void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp); + } + + /** + * Initializes the decoding process with specified settings. Will be called on the decoding thread + * before any decode calls. + */ + void initDecode(Settings settings, Callback decodeCallback); + /** + * Called when the decoder is no longer needed. Any more calls to decode will not be made. + */ + void release(); + /** + * Request the decoder to decode a frame. + */ + void decode(EncodedImage frame, DecodeInfo info); + /** + * The decoder should return true if it prefers late decoding. That is, it can not decode + * infinite number of frames before the decoded frame is consumed. + */ + boolean getPrefersLateDecoding(); + /** Should return a descriptive name for the implementation. */ + String getImplementationName(); +} diff --git a/webrtc/sdk/android/api/org/webrtc/VideoEncoder.java b/webrtc/sdk/android/api/org/webrtc/VideoEncoder.java new file mode 100644 index 0000000000..49c07fce4f --- /dev/null +++ b/webrtc/sdk/android/api/org/webrtc/VideoEncoder.java @@ -0,0 +1,126 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Interface for a video encoder that can be used with WebRTC. All calls will be made on the + * encoding thread. + */ +public interface VideoEncoder { + /** Settings passed to the encoder by WebRTC. */ + public class Settings { + public final int numberOfCores; + + public Settings(int numberOfCores) { + this.numberOfCores = numberOfCores; + } + } + + /** Additional info for encoding. */ + public class EncodeInfo { + public final EncodedImage.FrameType[] frameTypes; + + public EncodeInfo(EncodedImage.FrameType[] frameTypes) { + this.frameTypes = frameTypes; + } + } + + // TODO(sakal): Add values to these classes as necessary. + /** Codec specific information about the encoded frame. */ + public class CodecSpecificInfo {} + + public class CodecSpecificInfoVP8 extends CodecSpecificInfo {} + + public class CodecSpecificInfoVP9 extends CodecSpecificInfo {} + + public class CodecSpecificInfoH264 extends CodecSpecificInfo {} + + /** + * Represents bitrate allocated for an encoder to produce frames. Bitrate can be divided between + * spatial and temporal layers. + */ + public class BitrateAllocation { + // First index is the spatial layer and second the temporal layer. + public final long[][] bitratesBbs; + + /** + * Initializes the allocation with a two dimensional array of bitrates. The first index of the + * array is the spatial layer and the second index in the temporal layer. + */ + public BitrateAllocation(long[][] bitratesBbs) { + this.bitratesBbs = bitratesBbs; + } + + /** + * Gets the total bitrate allocated for all layers. + */ + public long getSum() { + long sum = 0; + for (long[] spatialLayer : bitratesBbs) { + for (long bitrate : spatialLayer) { + sum += bitrate; + } + } + return sum; + } + } + + /** Settings for WebRTC quality based scaling. */ + public class ScalingSettings { + public final boolean on; + public final int low; + public final int high; + + /** + * Creates quality based scaling settings. + * + * @param on True if quality scaling is turned on. + * @param low Average QP at which to scale up the resolution. + * @param high Average QP at which to scale down the resolution. + */ + public ScalingSettings(boolean on, int low, int high) { + this.on = on; + this.low = low; + this.high = high; + } + } + + public interface Callback { + /** Call to return an encoded frame. */ + void onEncodedFrame(EncodedImage frame, CodecSpecificInfo info); + } + + /** + * Initializes the encoding process. Call before any calls to encode. + */ + void initEncode(Settings settings, Callback encodeCallback); + /** + * Releases the encoder. No more calls to encode will be made after this call. + */ + void release(); + /** + * Requests the encoder to encode a frame. + */ + void encode(VideoFrame frame, EncodeInfo info); + /** + * Informs the encoder of the packet loss and the round-trip time of the network. + * + * @param packetLoss How many packets are lost on average per 255 packets. + * @param roundTripTimeMs Round-trip time of the network in milliseconds. + */ + void setChannelParameters(short packetLoss, long roundTripTimeMs); + /** Sets the bitrate allocation and the target framerate for the encoder. */ + void setRateAllocation(BitrateAllocation allocation, long framerate); + /** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */ + ScalingSettings getScalingSettings(); + /** Should return a descriptive name for the implementation. */ + String getImplementationName(); +}