diff --git a/sdk/android/api/org/webrtc/VideoSource.java b/sdk/android/api/org/webrtc/VideoSource.java index f783a57ea8..0939457c40 100644 --- a/sdk/android/api/org/webrtc/VideoSource.java +++ b/sdk/android/api/org/webrtc/VideoSource.java @@ -43,7 +43,20 @@ public class VideoSource extends MediaSource { @Override public void onFrameCaptured(VideoFrame frame) { - nativeAndroidVideoTrackSource.onFrameCaptured(frame); + final NativeAndroidVideoTrackSource.FrameAdaptationParameters parameters = + nativeAndroidVideoTrackSource.adaptFrame(frame); + if (parameters == null) { + // Drop frame. + return; + } + + final VideoFrame.Buffer adaptedBuffer = + frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth, + parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight); + // TODO(magjed): Add video processing hook here. + nativeAndroidVideoTrackSource.onFrameCaptured( + new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs)); + adaptedBuffer.release(); } }; diff --git a/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java b/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java index da963a8ebc..d19f6f0b8a 100644 --- a/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java +++ b/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java @@ -17,11 +17,36 @@ import org.webrtc.VideoFrame; * This class is meant to be a simple layer that only handles the JNI wrapping of a C++ * AndroidVideoTrackSource, that can easily be mocked out in Java unit tests. Refrain from adding * any unnecessary logic to this class. + * This class is thred safe and methods can be called from any thread, but if frames A, B, ..., are + * sent to adaptFrame(), the adapted frames adaptedA, adaptedB, ..., needs to be passed in the same + * order to onFrameCaptured(). */ class NativeAndroidVideoTrackSource { // Pointer to webrtc::jni::AndroidVideoTrackSource. private final long nativeAndroidVideoTrackSource; + public static class FrameAdaptationParameters { + public final int cropX; + public final int cropY; + public final int cropWidth; + public final int cropHeight; + public final int scaleWidth; + public final int scaleHeight; + public final long timestampNs; + + @CalledByNative("FrameAdaptationParameters") + FrameAdaptationParameters(int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, + int scaleHeight, long timestampNs) { + this.cropX = cropX; + this.cropY = cropY; + this.cropWidth = cropWidth; + this.cropHeight = cropHeight; + this.scaleWidth = scaleWidth; + this.scaleHeight = scaleHeight; + this.timestampNs = timestampNs; + } + } + public NativeAndroidVideoTrackSource(long nativeAndroidVideoTrackSource) { this.nativeAndroidVideoTrackSource = nativeAndroidVideoTrackSource; } @@ -34,11 +59,25 @@ class NativeAndroidVideoTrackSource { nativeSetState(nativeAndroidVideoTrackSource, isLive); } - /** Pass a frame to the native AndroidVideoTrackSource. */ + /** + * This function should be called before delivering any frame to determine if the frame should be + * dropped or what the cropping and scaling parameters should be. If the return value is null, the + * frame should be dropped, otherwise the frame should be adapted in accordance to the frame + * adaptation parameters before calling onFrameCaptured(). + */ + @Nullable + public FrameAdaptationParameters adaptFrame(VideoFrame frame) { + return nativeAdaptFrame(nativeAndroidVideoTrackSource, frame.getBuffer().getWidth(), + frame.getBuffer().getHeight(), frame.getRotation(), frame.getTimestampNs()); + } + + /** + * Pass an adapted frame to the native AndroidVideoTrackSource. Note that adaptFrame() is + * expected to be called first and that the passed frame conforms to those parameters. + */ public void onFrameCaptured(VideoFrame frame) { - nativeOnFrameCaptured(nativeAndroidVideoTrackSource, frame.getBuffer().getWidth(), - frame.getBuffer().getHeight(), frame.getRotation(), frame.getTimestampNs(), - frame.getBuffer()); + nativeOnFrameCaptured(nativeAndroidVideoTrackSource, frame.getRotation(), + frame.getTimestampNs(), frame.getBuffer()); } /** @@ -59,6 +98,9 @@ class NativeAndroidVideoTrackSource { int landscapeWidth, int landscapeHeight, @Nullable Integer maxLandscapePixelCount, int portraitWidth, int portraitHeight, @Nullable Integer maxPortraitPixelCount, @Nullable Integer maxFps); - private static native void nativeOnFrameCaptured(long nativeAndroidVideoTrackSource, int width, - int height, int rotation, long timestampNs, VideoFrame.Buffer buffer); + @Nullable + private static native FrameAdaptationParameters nativeAdaptFrame( + long nativeAndroidVideoTrackSource, int width, int height, int rotation, long timestampNs); + private static native void nativeOnFrameCaptured( + long nativeAndroidVideoTrackSource, int rotation, long timestampNs, VideoFrame.Buffer buffer); } diff --git a/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java b/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java index 69a0d54534..f0c27c5503 100644 --- a/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java +++ b/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java @@ -36,6 +36,18 @@ class NativeCapturerObserver implements CapturerObserver { @Override public void onFrameCaptured(VideoFrame frame) { - nativeAndroidVideoTrackSource.onFrameCaptured(frame); + final NativeAndroidVideoTrackSource.FrameAdaptationParameters parameters = + nativeAndroidVideoTrackSource.adaptFrame(frame); + if (parameters == null) { + // Drop frame. + return; + } + + final VideoFrame.Buffer adaptedBuffer = + frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth, + parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight); + nativeAndroidVideoTrackSource.onFrameCaptured( + new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs)); + adaptedBuffer.release(); } } diff --git a/sdk/android/src/jni/android_video_track_source.cc b/sdk/android/src/jni/android_video_track_source.cc index a9fb10d7ca..dbcbd6a6bc 100644 --- a/sdk/android/src/jni/android_video_track_source.cc +++ b/sdk/android/src/jni/android_video_track_source.cc @@ -86,21 +86,21 @@ bool AndroidVideoTrackSource::remote() const { return false; } -void AndroidVideoTrackSource::OnFrameCaptured( +ScopedJavaLocalRef AndroidVideoTrackSource::AdaptFrame( JNIEnv* env, const JavaRef& j_caller, jint j_width, jint j_height, jint j_rotation, - jlong j_timestamp_ns, - const JavaRef& j_video_frame_buffer) { + jlong j_timestamp_ns) { const VideoRotation rotation = jintToVideoRotation(j_rotation); - int64_t camera_time_us = j_timestamp_ns / rtc::kNumNanosecsPerMicrosec; - int64_t translated_camera_time_us = - align_timestamps_ ? timestamp_aligner_.TranslateTimestamp( - camera_time_us, rtc::TimeMicros()) - : camera_time_us; + const int64_t camera_time_us = j_timestamp_ns / rtc::kNumNanosecsPerMicrosec; + const int64_t aligned_timestamp_ns = + align_timestamps_ ? rtc::kNumNanosecsPerMicrosec * + timestamp_aligner_.TranslateTimestamp( + camera_time_us, rtc::TimeMicros()) + : j_timestamp_ns; int adapted_width; int adapted_height; @@ -109,35 +109,46 @@ void AndroidVideoTrackSource::OnFrameCaptured( int crop_x; int crop_y; + // TODO(magjed): Move this logic to users of NativeAndroidVideoTrackSource + // instead, in order to keep this native wrapping layer as thin as possible. if (rotation % 180 == 0) { - if (!AdaptFrame(j_width, j_height, camera_time_us, &adapted_width, - &adapted_height, &crop_width, &crop_height, &crop_x, - &crop_y)) { - return; + if (!rtc::AdaptedVideoTrackSource::AdaptFrame( + j_width, j_height, camera_time_us, &adapted_width, &adapted_height, + &crop_width, &crop_height, &crop_x, &crop_y)) { + return nullptr; } } else { // Swap all width/height and x/y. - if (!AdaptFrame(j_height, j_width, camera_time_us, &adapted_height, - &adapted_width, &crop_height, &crop_width, &crop_y, - &crop_x)) { - return; + if (!rtc::AdaptedVideoTrackSource::AdaptFrame( + j_height, j_width, camera_time_us, &adapted_height, &adapted_width, + &crop_height, &crop_width, &crop_y, &crop_x)) { + return nullptr; } } + return Java_FrameAdaptationParameters_Constructor( + env, crop_x, crop_y, crop_width, crop_height, adapted_width, + adapted_height, aligned_timestamp_ns); +} + +void AndroidVideoTrackSource::OnFrameCaptured( + JNIEnv* env, + const JavaRef& j_caller, + jint j_rotation, + jlong j_timestamp_ns, + const JavaRef& j_video_frame_buffer) { rtc::scoped_refptr buffer = - AndroidVideoBuffer::Create(env, j_video_frame_buffer) - ->CropAndScale(env, crop_x, crop_y, crop_width, crop_height, - adapted_width, adapted_height); + AndroidVideoBuffer::Create(env, j_video_frame_buffer); + const VideoRotation rotation = jintToVideoRotation(j_rotation); // AdaptedVideoTrackSource handles applying rotation for I420 frames. - if (apply_rotation() && rotation != kVideoRotation_0) { + if (apply_rotation() && rotation != kVideoRotation_0) buffer = buffer->ToI420(); - } OnFrame(VideoFrame::Builder() .set_video_frame_buffer(buffer) .set_rotation(rotation) - .set_timestamp_us(translated_camera_time_us) + .set_timestamp_us(j_timestamp_ns / rtc::kNumNanosecsPerMicrosec) .build()); } diff --git a/sdk/android/src/jni/android_video_track_source.h b/sdk/android/src/jni/android_video_track_source.h index 5640c1db9f..331a308bc9 100644 --- a/sdk/android/src/jni/android_video_track_source.h +++ b/sdk/android/src/jni/android_video_track_source.h @@ -24,6 +24,11 @@ namespace webrtc { namespace jni { +// This class needs to be used in conjunction with the Java corresponding class +// NativeAndroidVideoTrackSource. This class is thred safe and methods can be +// called from any thread, but if frames A, B, ..., are sent to adaptFrame(), +// the adapted frames adaptedA, adaptedB, ..., needs to be passed in the same +// order to onFrameCaptured(). class AndroidVideoTrackSource : public rtc::AdaptedVideoTrackSource { public: AndroidVideoTrackSource(rtc::Thread* signaling_thread, @@ -45,10 +50,25 @@ class AndroidVideoTrackSource : public rtc::AdaptedVideoTrackSource { bool remote() const override; + // This function should be called before delivering any frame to determine if + // the frame should be dropped or what the cropping and scaling parameters + // should be. This function is thread safe and can be called from any thread. + // This function returns + // NativeAndroidVideoTrackSource.FrameAdaptationParameters, or null if the + // frame should be dropped. + ScopedJavaLocalRef AdaptFrame(JNIEnv* env, + const JavaRef& j_caller, + jint j_width, + jint j_height, + jint j_rotation, + jlong j_timestamp_ns); + + // This function converts and passes the frame on to the rest of the C++ + // WebRTC layer. Note that GetFrameAdaptationParameters() is expected to be + // called first and that the delivered frame conforms to those parameters. + // This function is thread safe and can be called from any thread. void OnFrameCaptured(JNIEnv* env, const JavaRef& j_caller, - jint j_width, - jint j_height, jint j_rotation, jlong j_timestamp_ns, const JavaRef& j_video_frame_buffer);