From e9652ca6ec0c5291a55b2d13c27e55a57d234fbc Mon Sep 17 00:00:00 2001 From: Magnus Jedvert Date: Mon, 18 Feb 2019 16:12:00 +0100 Subject: [PATCH] Android: Add video processing interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL adds an API for injecting video processing after the WebRTC CPU and QP scaling step. Bug: webrtc:10247 Change-Id: I776498e1e9113f50e953ee411bbb31f181863312 Reviewed-on: https://webrtc-review.googlesource.com/c/119953 Commit-Queue: Magnus Jedvert Reviewed-by: Sami Kalliomäki Cr-Commit-Position: refs/heads/master@{#26740} --- sdk/android/BUILD.gn | 1 + .../api/org/webrtc/VideoProcessor.java | 25 ++++++++ sdk/android/api/org/webrtc/VideoSource.java | 61 ++++++++++++++++++- 3 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 sdk/android/api/org/webrtc/VideoProcessor.java diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index cbae01552c..01acfc511d 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -326,6 +326,7 @@ if (is_android) { "api/org/webrtc/StatsReport.java", "api/org/webrtc/TurnCustomizer.java", "api/org/webrtc/VideoSource.java", + "api/org/webrtc/VideoProcessor.java", "api/org/webrtc/VideoTrack.java", "src/java/org/webrtc/NativeLibrary.java", "src/java/org/webrtc/NativeCapturerObserver.java", diff --git a/sdk/android/api/org/webrtc/VideoProcessor.java b/sdk/android/api/org/webrtc/VideoProcessor.java new file mode 100644 index 0000000000..1a2aca5cf1 --- /dev/null +++ b/sdk/android/api/org/webrtc/VideoProcessor.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.support.annotation.Nullable; + +/** + * Lightweight abstraction for an object that can receive video frames, process them, and pass them + * on to another object. This object is also allowed to observe capturer start/stop. + */ +public interface VideoProcessor extends CapturerObserver { + /** + * Set the sink that receives the output from this processor. Null can be passed in to unregister + * a sink. After this call returns, no frames should be delivered to an unregistered sink. + */ + void setSink(@Nullable VideoSink sink); +} diff --git a/sdk/android/api/org/webrtc/VideoSource.java b/sdk/android/api/org/webrtc/VideoSource.java index 0939457c40..847a7bd986 100644 --- a/sdk/android/api/org/webrtc/VideoSource.java +++ b/sdk/android/api/org/webrtc/VideoSource.java @@ -30,15 +30,31 @@ public class VideoSource extends MediaSource { } private final NativeAndroidVideoTrackSource nativeAndroidVideoTrackSource; + private final Object videoProcessorLock = new Object(); + @Nullable private VideoProcessor videoProcessor; + private boolean isCapturerRunning; + private final CapturerObserver capturerObserver = new CapturerObserver() { @Override public void onCapturerStarted(boolean success) { nativeAndroidVideoTrackSource.setState(success); + synchronized (videoProcessorLock) { + isCapturerRunning = success; + if (videoProcessor != null) { + videoProcessor.onCapturerStarted(success); + } + } } @Override public void onCapturerStopped() { nativeAndroidVideoTrackSource.setState(/* isLive= */ false); + synchronized (videoProcessorLock) { + isCapturerRunning = false; + if (videoProcessor != null) { + videoProcessor.onCapturerStopped(); + } + } } @Override @@ -53,9 +69,17 @@ public class VideoSource extends MediaSource { final VideoFrame.Buffer adaptedBuffer = frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth, parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight); - // TODO(magjed): Add video processing hook here. - nativeAndroidVideoTrackSource.onFrameCaptured( - new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs)); + final VideoFrame adaptedFrame = + new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs); + + synchronized (videoProcessorLock) { + if (videoProcessor != null) { + videoProcessor.onFrameCaptured(adaptedFrame); + adaptedBuffer.release(); + return; + } + } + nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame); adaptedBuffer.release(); } }; @@ -98,6 +122,31 @@ public class VideoSource extends MediaSource { maxLandscapePixelCount, targetPortraitAspectRatio, maxPortraitPixelCount, maxFps); } + /** + * Hook for injecting a custom video processor before frames are passed onto WebRTC. The frames + * will be cropped and scaled depending on CPU and network conditions before they are passed to + * the video processor. Frames will be delivered to the video processor on the same thread they + * are passed to this object. The video processor is allowed to deliver the processed frames + * back on any thread. + */ + public void setVideoProcessor(@Nullable VideoProcessor newVideoProcessor) { + synchronized (videoProcessorLock) { + if (videoProcessor != null) { + videoProcessor.setSink(/* sink= */ null); + if (isCapturerRunning) { + videoProcessor.onCapturerStopped(); + } + } + videoProcessor = newVideoProcessor; + if (newVideoProcessor != null) { + newVideoProcessor.setSink(nativeAndroidVideoTrackSource::onFrameCaptured); + if (isCapturerRunning) { + newVideoProcessor.onCapturerStarted(/* success= */ true); + } + } + } + } + public CapturerObserver getCapturerObserver() { return capturerObserver; } @@ -106,4 +155,10 @@ public class VideoSource extends MediaSource { long getNativeVideoTrackSource() { return getNativeMediaSource(); } + + @Override + public void dispose() { + setVideoProcessor(/* newVideoProcessor= */ null); + super.dispose(); + } }