From c21cf046181d0fc52ec27c463adb67d6f6a27cdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sami=20Kalliom=C3=A4ki?= Date: Wed, 10 Apr 2019 13:44:58 +0200 Subject: [PATCH] Move frame adaptation inside video processor. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:10530 Change-Id: Iba6a91bf3e1ec4b2821b554e9e28fd2ead662723 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131947 Reviewed-by: Magnus Jedvert Commit-Queue: Sami Kalliomäki Cr-Commit-Position: refs/heads/master@{#27542} --- .../api/org/webrtc/VideoProcessor.java | 51 +++++++++++++++++++ sdk/android/api/org/webrtc/VideoSource.java | 24 +++------ .../webrtc/NativeAndroidVideoTrackSource.java | 35 ++++--------- .../org/webrtc/NativeCapturerObserver.java | 2 +- .../src/jni/android_video_track_source.cc | 33 ++++++------ 5 files changed, 86 insertions(+), 59 deletions(-) diff --git a/sdk/android/api/org/webrtc/VideoProcessor.java b/sdk/android/api/org/webrtc/VideoProcessor.java index 1a2aca5cf1..3a89090e2d 100644 --- a/sdk/android/api/org/webrtc/VideoProcessor.java +++ b/sdk/android/api/org/webrtc/VideoProcessor.java @@ -17,9 +17,60 @@ import android.support.annotation.Nullable; * on to another object. This object is also allowed to observe capturer start/stop. */ public interface VideoProcessor extends CapturerObserver { + public static class FrameAdaptationParameters { + public final int cropX; + public final int cropY; + public final int cropWidth; + public final int cropHeight; + public final int scaleWidth; + public final int scaleHeight; + public final long timestampNs; + public final boolean drop; + + public FrameAdaptationParameters(int cropX, int cropY, int cropWidth, int cropHeight, + int scaleWidth, int scaleHeight, long timestampNs, boolean drop) { + this.cropX = cropX; + this.cropY = cropY; + this.cropWidth = cropWidth; + this.cropHeight = cropHeight; + this.scaleWidth = scaleWidth; + this.scaleHeight = scaleHeight; + this.timestampNs = timestampNs; + this.drop = drop; + } + } + + /** + * This is a chance to access an unadapted frame. The default implementation applies the + * adaptation and forwards the frame to {@link #onFrameCaptured(VideoFrame)}. + */ + default void onFrameCaptured(VideoFrame frame, FrameAdaptationParameters parameters) { + VideoFrame adaptedFrame = applyFrameAdaptationParameters(frame, parameters); + if (adaptedFrame != null) { + onFrameCaptured(adaptedFrame); + adaptedFrame.release(); + } + } + /** * Set the sink that receives the output from this processor. Null can be passed in to unregister * a sink. After this call returns, no frames should be delivered to an unregistered sink. */ void setSink(@Nullable VideoSink sink); + + /** + * Applies the frame adaptation parameters to a frame. Returns null if the frame is meant to be + * dropped. Returns a new frame. The caller is responsible for releasing the returned frame. + */ + public static @Nullable VideoFrame applyFrameAdaptationParameters( + VideoFrame frame, FrameAdaptationParameters parameters) { + if (parameters.drop) { + return null; + } + + final VideoFrame.Buffer adaptedBuffer = + frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth, + parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight); + return new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs); + } } diff --git a/sdk/android/api/org/webrtc/VideoSource.java b/sdk/android/api/org/webrtc/VideoSource.java index 847a7bd986..1b40231083 100644 --- a/sdk/android/api/org/webrtc/VideoSource.java +++ b/sdk/android/api/org/webrtc/VideoSource.java @@ -59,28 +59,20 @@ public class VideoSource extends MediaSource { @Override public void onFrameCaptured(VideoFrame frame) { - final NativeAndroidVideoTrackSource.FrameAdaptationParameters parameters = + final VideoProcessor.FrameAdaptationParameters parameters = nativeAndroidVideoTrackSource.adaptFrame(frame); - if (parameters == null) { - // Drop frame. - return; - } - - final VideoFrame.Buffer adaptedBuffer = - frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth, - parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight); - final VideoFrame adaptedFrame = - new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs); - synchronized (videoProcessorLock) { if (videoProcessor != null) { - videoProcessor.onFrameCaptured(adaptedFrame); - adaptedBuffer.release(); + videoProcessor.onFrameCaptured(frame, parameters); return; } } - nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame); - adaptedBuffer.release(); + + VideoFrame adaptedFrame = VideoProcessor.applyFrameAdaptationParameters(frame, parameters); + if (adaptedFrame != null) { + nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame); + adaptedFrame.release(); + } } }; diff --git a/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java b/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java index d19f6f0b8a..48690fb8b7 100644 --- a/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java +++ b/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java @@ -12,6 +12,7 @@ package org.webrtc; import android.support.annotation.Nullable; import org.webrtc.VideoFrame; +import org.webrtc.VideoProcessor; /** * This class is meant to be a simple layer that only handles the JNI wrapping of a C++ @@ -25,28 +26,6 @@ class NativeAndroidVideoTrackSource { // Pointer to webrtc::jni::AndroidVideoTrackSource. private final long nativeAndroidVideoTrackSource; - public static class FrameAdaptationParameters { - public final int cropX; - public final int cropY; - public final int cropWidth; - public final int cropHeight; - public final int scaleWidth; - public final int scaleHeight; - public final long timestampNs; - - @CalledByNative("FrameAdaptationParameters") - FrameAdaptationParameters(int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, - int scaleHeight, long timestampNs) { - this.cropX = cropX; - this.cropY = cropY; - this.cropWidth = cropWidth; - this.cropHeight = cropHeight; - this.scaleWidth = scaleWidth; - this.scaleHeight = scaleHeight; - this.timestampNs = timestampNs; - } - } - public NativeAndroidVideoTrackSource(long nativeAndroidVideoTrackSource) { this.nativeAndroidVideoTrackSource = nativeAndroidVideoTrackSource; } @@ -66,7 +45,7 @@ class NativeAndroidVideoTrackSource { * adaptation parameters before calling onFrameCaptured(). */ @Nullable - public FrameAdaptationParameters adaptFrame(VideoFrame frame) { + public VideoProcessor.FrameAdaptationParameters adaptFrame(VideoFrame frame) { return nativeAdaptFrame(nativeAndroidVideoTrackSource, frame.getBuffer().getWidth(), frame.getBuffer().getHeight(), frame.getRotation(), frame.getTimestampNs()); } @@ -93,13 +72,21 @@ class NativeAndroidVideoTrackSource { targetPortraitAspectRatio.height, maxPortraitPixelCount, maxFps); } + @CalledByNative + static VideoProcessor.FrameAdaptationParameters createFrameAdaptationParameters(int cropX, + int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight, long timestampNs, + boolean drop) { + return new VideoProcessor.FrameAdaptationParameters( + cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight, timestampNs, drop); + } + private static native void nativeSetState(long nativeAndroidVideoTrackSource, boolean isLive); private static native void nativeAdaptOutputFormat(long nativeAndroidVideoTrackSource, int landscapeWidth, int landscapeHeight, @Nullable Integer maxLandscapePixelCount, int portraitWidth, int portraitHeight, @Nullable Integer maxPortraitPixelCount, @Nullable Integer maxFps); @Nullable - private static native FrameAdaptationParameters nativeAdaptFrame( + private static native VideoProcessor.FrameAdaptationParameters nativeAdaptFrame( long nativeAndroidVideoTrackSource, int width, int height, int rotation, long timestampNs); private static native void nativeOnFrameCaptured( long nativeAndroidVideoTrackSource, int rotation, long timestampNs, VideoFrame.Buffer buffer); diff --git a/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java b/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java index f0c27c5503..c195fb3a4c 100644 --- a/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java +++ b/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java @@ -36,7 +36,7 @@ class NativeCapturerObserver implements CapturerObserver { @Override public void onFrameCaptured(VideoFrame frame) { - final NativeAndroidVideoTrackSource.FrameAdaptationParameters parameters = + final VideoProcessor.FrameAdaptationParameters parameters = nativeAndroidVideoTrackSource.adaptFrame(frame); if (parameters == null) { // Drop frame. diff --git a/sdk/android/src/jni/android_video_track_source.cc b/sdk/android/src/jni/android_video_track_source.cc index dbcbd6a6bc..973167a1ec 100644 --- a/sdk/android/src/jni/android_video_track_source.cc +++ b/sdk/android/src/jni/android_video_track_source.cc @@ -102,33 +102,30 @@ ScopedJavaLocalRef AndroidVideoTrackSource::AdaptFrame( camera_time_us, rtc::TimeMicros()) : j_timestamp_ns; - int adapted_width; - int adapted_height; - int crop_width; - int crop_height; - int crop_x; - int crop_y; + int adapted_width = 0; + int adapted_height = 0; + int crop_width = 0; + int crop_height = 0; + int crop_x = 0; + int crop_y = 0; + bool drop; // TODO(magjed): Move this logic to users of NativeAndroidVideoTrackSource // instead, in order to keep this native wrapping layer as thin as possible. if (rotation % 180 == 0) { - if (!rtc::AdaptedVideoTrackSource::AdaptFrame( - j_width, j_height, camera_time_us, &adapted_width, &adapted_height, - &crop_width, &crop_height, &crop_x, &crop_y)) { - return nullptr; - } + drop = !rtc::AdaptedVideoTrackSource::AdaptFrame( + j_width, j_height, camera_time_us, &adapted_width, &adapted_height, + &crop_width, &crop_height, &crop_x, &crop_y); } else { // Swap all width/height and x/y. - if (!rtc::AdaptedVideoTrackSource::AdaptFrame( - j_height, j_width, camera_time_us, &adapted_height, &adapted_width, - &crop_height, &crop_width, &crop_y, &crop_x)) { - return nullptr; - } + drop = !rtc::AdaptedVideoTrackSource::AdaptFrame( + j_height, j_width, camera_time_us, &adapted_height, &adapted_width, + &crop_height, &crop_width, &crop_y, &crop_x); } - return Java_FrameAdaptationParameters_Constructor( + return Java_NativeAndroidVideoTrackSource_createFrameAdaptationParameters( env, crop_x, crop_y, crop_width, crop_height, adapted_width, - adapted_height, aligned_timestamp_ns); + adapted_height, aligned_timestamp_ns, drop); } void AndroidVideoTrackSource::OnFrameCaptured(