From be910460e082af04489a76a3b9fee571756af10b Mon Sep 17 00:00:00 2001 From: sakal Date: Fri, 11 Aug 2017 00:26:05 -0700 Subject: [PATCH] Add support for capturers to capture VideoFrames. BUG=webrtc:7749, webrtc:7760 Review-Url: https://codereview.webrtc.org/2982213002 Cr-Commit-Position: refs/heads/master@{#19318} --- .../android/api/org/webrtc/VideoCapturer.java | 4 +- .../CameraVideoCapturerTestFixtures.java | 6 +++ .../src/org/webrtc/FileVideoCapturerTest.java | 5 ++ .../AndroidVideoTrackSourceObserver.java | 8 ++++ .../src/jni/androidvideotracksource.cc | 48 +++++++++++++++++++ .../android/src/jni/androidvideotracksource.h | 12 +++++ .../src/jni/androidvideotracksource_jni.cc | 16 +++++++ .../sdk/android/src/jni/native_handle_impl.cc | 42 +++++++++++++--- .../sdk/android/src/jni/native_handle_impl.h | 23 +++++++++ 9 files changed, 157 insertions(+), 7 deletions(-) diff --git a/webrtc/sdk/android/api/org/webrtc/VideoCapturer.java b/webrtc/sdk/android/api/org/webrtc/VideoCapturer.java index 43473faa17..804560a3c3 100644 --- a/webrtc/sdk/android/api/org/webrtc/VideoCapturer.java +++ b/webrtc/sdk/android/api/org/webrtc/VideoCapturer.java @@ -11,7 +11,6 @@ package org.webrtc; import android.content.Context; - import java.util.List; // Base interface for all VideoCapturers to implement. @@ -31,6 +30,9 @@ public interface VideoCapturer { // owned by VideoCapturer. void onTextureFrameCaptured(int width, int height, int oesTextureId, float[] transformMatrix, int rotation, long timestamp); + + // Delivers a captured frame. Called on a Java thread owned by VideoCapturer. + void onFrameCaptured(VideoFrame frame); } /** diff --git a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java index 91bb39fbb7..d9af85b96c 100644 --- a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java +++ b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java @@ -135,6 +135,7 @@ class CameraVideoCapturerTestFixtures { frameLock.notify(); } } + @Override public void onTextureFrameCaptured(int width, int height, int oesTextureId, float[] transformMatrix, int rotation, long timeStamp) { @@ -148,6 +149,11 @@ class CameraVideoCapturerTestFixtures { } } + @Override + public void onFrameCaptured(VideoFrame frame) { + // Empty on purpose. + } + public boolean waitForCapturerToStart() throws InterruptedException { Logging.d(TAG, "Waiting for the capturer to start"); synchronized (capturerStartLock) { diff --git a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java index 5481818627..8127f8f48c 100644 --- a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java +++ b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java @@ -65,6 +65,11 @@ public class FileVideoCapturerTest { // Empty on purpose. } + @Override + public void onFrameCaptured(VideoFrame frame) { + // Empty on purpose. + } + public synchronized ArrayList getMinimumFramesBlocking(int minFrames) throws InterruptedException { while (frameDatas.size() < minFrames) { diff --git a/webrtc/sdk/android/src/java/org/webrtc/AndroidVideoTrackSourceObserver.java b/webrtc/sdk/android/src/java/org/webrtc/AndroidVideoTrackSourceObserver.java index 323bd5be2a..123204791f 100644 --- a/webrtc/sdk/android/src/java/org/webrtc/AndroidVideoTrackSourceObserver.java +++ b/webrtc/sdk/android/src/java/org/webrtc/AndroidVideoTrackSourceObserver.java @@ -43,10 +43,18 @@ class AndroidVideoTrackSourceObserver implements VideoCapturer.CapturerObserver nativeSource, width, height, oesTextureId, transformMatrix, rotation, timestamp); } + @Override + public void onFrameCaptured(VideoFrame frame) { + nativeOnFrameCaptured(nativeSource, frame.getBuffer().getWidth(), frame.getBuffer().getHeight(), + frame.getRotation(), frame.getTimestampNs(), frame.getBuffer()); + } + private native void nativeCapturerStarted(long nativeSource, boolean success); private native void nativeCapturerStopped(long nativeSource); private native void nativeOnByteBufferFrameCaptured(long nativeSource, byte[] data, int length, int width, int height, int rotation, long timeStamp); private native void nativeOnTextureFrameCaptured(long nativeSource, int width, int height, int oesTextureId, float[] transformMatrix, int rotation, long timestamp); + private native void nativeOnFrameCaptured(long nativeSource, int width, int height, int rotation, + long timestampNs, VideoFrame.Buffer frame); } diff --git a/webrtc/sdk/android/src/jni/androidvideotracksource.cc b/webrtc/sdk/android/src/jni/androidvideotracksource.cc index 770962de6b..0444b1d9d7 100644 --- a/webrtc/sdk/android/src/jni/androidvideotracksource.cc +++ b/webrtc/sdk/android/src/jni/androidvideotracksource.cc @@ -13,6 +13,7 @@ #include #include "webrtc/rtc_base/logging.h" +#include "webrtc/sdk/android/src/jni/classreferenceholder.h" namespace { // MediaCodec wants resolution to be divisible by 2. @@ -32,9 +33,16 @@ AndroidVideoTrackSource::AndroidVideoTrackSource( new rtc::RefCountedObject( jni, j_surface_texture_helper)), + video_buffer_factory_(jni), is_screencast_(is_screencast) { LOG(LS_INFO) << "AndroidVideoTrackSource ctor"; camera_thread_checker_.DetachFromThread(); + + jclass j_video_frame_buffer_class = + webrtc_jni::FindClass(jni, "org/webrtc/VideoFrame$Buffer"); + j_crop_and_scale_id_ = + jni->GetMethodID(j_video_frame_buffer_class, "cropAndScale", + "(IIIIII)Lorg/webrtc/VideoFrame$Buffer;"); } void AndroidVideoTrackSource::SetState(SourceState state) { @@ -152,6 +160,46 @@ void AndroidVideoTrackSource::OnTextureFrameCaptured( rotation, translated_camera_time_us)); } +void AndroidVideoTrackSource::OnFrameCaptured(JNIEnv* jni, + int width, + int height, + int64_t timestamp_ns, + VideoRotation rotation, + jobject j_video_frame_buffer) { + RTC_DCHECK(camera_thread_checker_.CalledOnValidThread()); + + int64_t camera_time_us = timestamp_ns / rtc::kNumNanosecsPerMicrosec; + int64_t translated_camera_time_us = + timestamp_aligner_.TranslateTimestamp(camera_time_us, rtc::TimeMicros()); + + int adapted_width; + int adapted_height; + int crop_width; + int crop_height; + int crop_x; + int crop_y; + + if (!AdaptFrame(width, height, camera_time_us, &adapted_width, + &adapted_height, &crop_width, &crop_height, &crop_x, + &crop_y)) { + return; + } + + jobject j_adapted_video_frame_buffer = jni->CallObjectMethod( + j_video_frame_buffer, j_crop_and_scale_id_, crop_x, crop_y, crop_width, + crop_height, adapted_width, adapted_height); + + rtc::scoped_refptr buffer = + video_buffer_factory_.WrapBuffer(jni, j_adapted_video_frame_buffer); + + // AdaptedVideoTrackSource handles applying rotation for I420 frames. + if (apply_rotation()) { + buffer = buffer->ToI420(); + } + + OnFrame(VideoFrame(buffer, rotation, translated_camera_time_us)); +} + void AndroidVideoTrackSource::OnOutputFormatRequest(int width, int height, int fps) { diff --git a/webrtc/sdk/android/src/jni/androidvideotracksource.h b/webrtc/sdk/android/src/jni/androidvideotracksource.h index 292c7e0e51..92bc71b24a 100644 --- a/webrtc/sdk/android/src/jni/androidvideotracksource.h +++ b/webrtc/sdk/android/src/jni/androidvideotracksource.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_API_ANDROID_JNI_ANDROIDVIDEOTRACKSOURCE_H_ #define WEBRTC_API_ANDROID_JNI_ANDROIDVIDEOTRACKSOURCE_H_ +#include + #include "webrtc/common_video/include/i420_buffer_pool.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/media/base/adaptedvideotracksource.h" @@ -59,6 +61,13 @@ class AndroidVideoTrackSource : public rtc::AdaptedVideoTrackSource { int64_t timestamp_ns, const webrtc_jni::NativeHandleImpl& handle); + void OnFrameCaptured(JNIEnv* jni, + int width, + int height, + int64_t timestamp_ns, + VideoRotation rotation, + jobject j_video_frame_buffer); + void OnOutputFormatRequest(int width, int height, int fps); rtc::scoped_refptr @@ -75,7 +84,10 @@ class AndroidVideoTrackSource : public rtc::AdaptedVideoTrackSource { NV12ToI420Scaler nv12toi420_scaler_; I420BufferPool buffer_pool_; rtc::scoped_refptr surface_texture_helper_; + webrtc_jni::AndroidVideoBufferFactory video_buffer_factory_; const bool is_screencast_; + + jmethodID j_crop_and_scale_id_; }; } // namespace webrtc diff --git a/webrtc/sdk/android/src/jni/androidvideotracksource_jni.cc b/webrtc/sdk/android/src/jni/androidvideotracksource_jni.cc index 62f5f82574..56335227f8 100644 --- a/webrtc/sdk/android/src/jni/androidvideotracksource_jni.cc +++ b/webrtc/sdk/android/src/jni/androidvideotracksource_jni.cc @@ -69,6 +69,22 @@ JOW_OBSERVER_METHOD(void, nativeOnTextureFrameCaptured) NativeHandleImpl(jni, j_oes_texture_id, j_transform_matrix)); } +JOW_OBSERVER_METHOD(void, nativeOnFrameCaptured) +(JNIEnv* jni, + jclass, + jlong j_source, + jint j_width, + jint j_height, + jint j_rotation, + jlong j_timestamp_ns, + jobject j_video_frame_buffer) { + webrtc::AndroidVideoTrackSource* source = + AndroidVideoTrackSourceFromJavaProxy(j_source); + source->OnFrameCaptured(jni, j_width, j_height, j_timestamp_ns, + jintToVideoRotation(j_rotation), + j_video_frame_buffer); +} + JOW_OBSERVER_METHOD(void, nativeCapturerStarted) (JNIEnv* jni, jclass, jlong j_source, jboolean j_success) { LOG(LS_INFO) << "AndroidVideoTrackSourceObserve_nativeCapturerStarted"; diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.cc b/webrtc/sdk/android/src/jni/native_handle_impl.cc index eebfba4035..cd78a27584 100644 --- a/webrtc/sdk/android/src/jni/native_handle_impl.cc +++ b/webrtc/sdk/android/src/jni/native_handle_impl.cc @@ -322,18 +322,39 @@ rtc::scoped_refptr AndroidTextureBuffer::ToI420() { return copy; } +rtc::scoped_refptr AndroidVideoBuffer::WrapReference( + JNIEnv* jni, + jmethodID j_release_id, + int width, + int height, + jobject j_video_frame_buffer) { + return new rtc::RefCountedObject( + jni, j_release_id, width, height, j_video_frame_buffer); +} + AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni, jmethodID j_retain_id, jmethodID j_release_id, int width, int height, jobject j_video_frame_buffer) + : AndroidVideoBuffer(jni, + j_release_id, + width, + height, + j_video_frame_buffer) { + jni->CallVoidMethod(j_video_frame_buffer, j_retain_id); +} + +AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni, + jmethodID j_release_id, + int width, + int height, + jobject j_video_frame_buffer) : j_release_id_(j_release_id), width_(width), height_(height), - j_video_frame_buffer_(jni, j_video_frame_buffer) { - jni->CallVoidMethod(j_video_frame_buffer, j_retain_id); -} + j_video_frame_buffer_(jni, j_video_frame_buffer) {} AndroidVideoBuffer::~AndroidVideoBuffer() { JNIEnv* jni = AttachCurrentThreadIfNeeded(); @@ -422,15 +443,24 @@ webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame( uint32_t timestamp_ns = jni->CallLongMethod(j_video_frame, j_get_timestamp_ns_id_); rtc::scoped_refptr buffer = - CreateBuffer(j_video_frame_buffer); + CreateBuffer(jni, j_video_frame_buffer); return webrtc::VideoFrame(buffer, timestamp_rtp, timestamp_ns / rtc::kNumNanosecsPerMillisec, static_cast(rotation)); } -rtc::scoped_refptr AndroidVideoBufferFactory::CreateBuffer( +rtc::scoped_refptr AndroidVideoBufferFactory::WrapBuffer( + JNIEnv* jni, + jobject j_video_frame_buffer) const { + int width = jni->CallIntMethod(j_video_frame_buffer, j_get_width_id_); + int height = jni->CallIntMethod(j_video_frame_buffer, j_get_height_id_); + return AndroidVideoBuffer::WrapReference(jni, j_release_id_, width, height, + j_video_frame_buffer); +} + +rtc::scoped_refptr AndroidVideoBufferFactory::CreateBuffer( + JNIEnv* jni, jobject j_video_frame_buffer) const { - JNIEnv* jni = AttachCurrentThreadIfNeeded(); int width = jni->CallIntMethod(j_video_frame_buffer, j_get_width_id_); int height = jni->CallIntMethod(j_video_frame_buffer, j_get_height_id_); return new rtc::RefCountedObject( diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.h b/webrtc/sdk/android/src/jni/native_handle_impl.h index 834441a7c6..fc674db9d4 100644 --- a/webrtc/sdk/android/src/jni/native_handle_impl.h +++ b/webrtc/sdk/android/src/jni/native_handle_impl.h @@ -103,12 +103,28 @@ class AndroidTextureBuffer : public AndroidVideoFrameBuffer { class AndroidVideoBuffer : public AndroidVideoFrameBuffer { public: + // Wraps an existing reference to a Java VideoBuffer. Retain will not be + // called but release will be called when the C++ object is destroyed. + static rtc::scoped_refptr WrapReference( + JNIEnv* jni, + jmethodID j_release_id, + int width, + int height, + jobject j_video_frame_buffer); + AndroidVideoBuffer(JNIEnv* jni, jmethodID j_retain_id, jmethodID j_release_id, int width, int height, jobject j_video_frame_buffer); + // Should not be called directly. Wraps a reference. Use + // AndroidVideoBuffer::WrapReference instead for clarity. + AndroidVideoBuffer(JNIEnv* jni, + jmethodID j_release_id, + int width, + int height, + jobject j_video_frame_buffer); ~AndroidVideoBuffer() override; jobject video_frame_buffer() const; @@ -140,7 +156,14 @@ class AndroidVideoBufferFactory { jobject j_video_frame, uint32_t timestamp_rtp) const; + // Wraps a buffer to AndroidVideoBuffer without incrementing the reference + // count. + rtc::scoped_refptr WrapBuffer( + JNIEnv* jni, + jobject j_video_frame_buffer) const; + rtc::scoped_refptr CreateBuffer( + JNIEnv* jni, jobject j_video_frame_buffer) const; private: