diff --git a/webrtc/sdk/android/api/org/webrtc/VideoCapturer.java b/webrtc/sdk/android/api/org/webrtc/VideoCapturer.java
index 43473faa17..804560a3c3 100644
--- a/webrtc/sdk/android/api/org/webrtc/VideoCapturer.java
+++ b/webrtc/sdk/android/api/org/webrtc/VideoCapturer.java
@@ -11,7 +11,6 @@
package org.webrtc;
import android.content.Context;
-
import java.util.List;
// Base interface for all VideoCapturers to implement.
@@ -31,6 +30,9 @@ public interface VideoCapturer {
// owned by VideoCapturer.
void onTextureFrameCaptured(int width, int height, int oesTextureId, float[] transformMatrix,
int rotation, long timestamp);
+
+ // Delivers a captured frame. Called on a Java thread owned by VideoCapturer.
+ void onFrameCaptured(VideoFrame frame);
}
/**
diff --git a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java
index 91bb39fbb7..d9af85b96c 100644
--- a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java
+++ b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java
@@ -135,6 +135,7 @@ class CameraVideoCapturerTestFixtures {
frameLock.notify();
}
}
+
@Override
public void onTextureFrameCaptured(int width, int height, int oesTextureId,
float[] transformMatrix, int rotation, long timeStamp) {
@@ -148,6 +149,11 @@ class CameraVideoCapturerTestFixtures {
}
}
+ @Override
+ public void onFrameCaptured(VideoFrame frame) {
+ // Empty on purpose.
+ }
+
public boolean waitForCapturerToStart() throws InterruptedException {
Logging.d(TAG, "Waiting for the capturer to start");
synchronized (capturerStartLock) {
diff --git a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java
index 5481818627..8127f8f48c 100644
--- a/webrtc/sdk/android/instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java
+++ b/webrtc/sdk/android/instrumentationtests/src/org/webrtc/FileVideoCapturerTest.java
@@ -65,6 +65,11 @@ public class FileVideoCapturerTest {
// Empty on purpose.
}
+ @Override
+ public void onFrameCaptured(VideoFrame frame) {
+ // Empty on purpose.
+ }
+
public synchronized ArrayList getMinimumFramesBlocking(int minFrames)
throws InterruptedException {
while (frameDatas.size() < minFrames) {
diff --git a/webrtc/sdk/android/src/java/org/webrtc/AndroidVideoTrackSourceObserver.java b/webrtc/sdk/android/src/java/org/webrtc/AndroidVideoTrackSourceObserver.java
index 323bd5be2a..123204791f 100644
--- a/webrtc/sdk/android/src/java/org/webrtc/AndroidVideoTrackSourceObserver.java
+++ b/webrtc/sdk/android/src/java/org/webrtc/AndroidVideoTrackSourceObserver.java
@@ -43,10 +43,18 @@ class AndroidVideoTrackSourceObserver implements VideoCapturer.CapturerObserver
nativeSource, width, height, oesTextureId, transformMatrix, rotation, timestamp);
}
+ @Override
+ public void onFrameCaptured(VideoFrame frame) {
+ nativeOnFrameCaptured(nativeSource, frame.getBuffer().getWidth(), frame.getBuffer().getHeight(),
+ frame.getRotation(), frame.getTimestampNs(), frame.getBuffer());
+ }
+
private native void nativeCapturerStarted(long nativeSource, boolean success);
private native void nativeCapturerStopped(long nativeSource);
private native void nativeOnByteBufferFrameCaptured(long nativeSource, byte[] data, int length,
int width, int height, int rotation, long timeStamp);
private native void nativeOnTextureFrameCaptured(long nativeSource, int width, int height,
int oesTextureId, float[] transformMatrix, int rotation, long timestamp);
+ private native void nativeOnFrameCaptured(long nativeSource, int width, int height, int rotation,
+ long timestampNs, VideoFrame.Buffer frame);
}
diff --git a/webrtc/sdk/android/src/jni/androidvideotracksource.cc b/webrtc/sdk/android/src/jni/androidvideotracksource.cc
index 770962de6b..0444b1d9d7 100644
--- a/webrtc/sdk/android/src/jni/androidvideotracksource.cc
+++ b/webrtc/sdk/android/src/jni/androidvideotracksource.cc
@@ -13,6 +13,7 @@
#include
#include "webrtc/rtc_base/logging.h"
+#include "webrtc/sdk/android/src/jni/classreferenceholder.h"
namespace {
// MediaCodec wants resolution to be divisible by 2.
@@ -32,9 +33,16 @@ AndroidVideoTrackSource::AndroidVideoTrackSource(
new rtc::RefCountedObject(
jni,
j_surface_texture_helper)),
+ video_buffer_factory_(jni),
is_screencast_(is_screencast) {
LOG(LS_INFO) << "AndroidVideoTrackSource ctor";
camera_thread_checker_.DetachFromThread();
+
+ jclass j_video_frame_buffer_class =
+ webrtc_jni::FindClass(jni, "org/webrtc/VideoFrame$Buffer");
+ j_crop_and_scale_id_ =
+ jni->GetMethodID(j_video_frame_buffer_class, "cropAndScale",
+ "(IIIIII)Lorg/webrtc/VideoFrame$Buffer;");
}
void AndroidVideoTrackSource::SetState(SourceState state) {
@@ -152,6 +160,46 @@ void AndroidVideoTrackSource::OnTextureFrameCaptured(
rotation, translated_camera_time_us));
}
+void AndroidVideoTrackSource::OnFrameCaptured(JNIEnv* jni,
+ int width,
+ int height,
+ int64_t timestamp_ns,
+ VideoRotation rotation,
+ jobject j_video_frame_buffer) {
+ RTC_DCHECK(camera_thread_checker_.CalledOnValidThread());
+
+ int64_t camera_time_us = timestamp_ns / rtc::kNumNanosecsPerMicrosec;
+ int64_t translated_camera_time_us =
+ timestamp_aligner_.TranslateTimestamp(camera_time_us, rtc::TimeMicros());
+
+ int adapted_width;
+ int adapted_height;
+ int crop_width;
+ int crop_height;
+ int crop_x;
+ int crop_y;
+
+ if (!AdaptFrame(width, height, camera_time_us, &adapted_width,
+ &adapted_height, &crop_width, &crop_height, &crop_x,
+ &crop_y)) {
+ return;
+ }
+
+ jobject j_adapted_video_frame_buffer = jni->CallObjectMethod(
+ j_video_frame_buffer, j_crop_and_scale_id_, crop_x, crop_y, crop_width,
+ crop_height, adapted_width, adapted_height);
+
+ rtc::scoped_refptr buffer =
+ video_buffer_factory_.WrapBuffer(jni, j_adapted_video_frame_buffer);
+
+ // AdaptedVideoTrackSource handles applying rotation for I420 frames.
+ if (apply_rotation()) {
+ buffer = buffer->ToI420();
+ }
+
+ OnFrame(VideoFrame(buffer, rotation, translated_camera_time_us));
+}
+
void AndroidVideoTrackSource::OnOutputFormatRequest(int width,
int height,
int fps) {
diff --git a/webrtc/sdk/android/src/jni/androidvideotracksource.h b/webrtc/sdk/android/src/jni/androidvideotracksource.h
index 292c7e0e51..92bc71b24a 100644
--- a/webrtc/sdk/android/src/jni/androidvideotracksource.h
+++ b/webrtc/sdk/android/src/jni/androidvideotracksource.h
@@ -11,6 +11,8 @@
#ifndef WEBRTC_API_ANDROID_JNI_ANDROIDVIDEOTRACKSOURCE_H_
#define WEBRTC_API_ANDROID_JNI_ANDROIDVIDEOTRACKSOURCE_H_
+#include
+
#include "webrtc/common_video/include/i420_buffer_pool.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/media/base/adaptedvideotracksource.h"
@@ -59,6 +61,13 @@ class AndroidVideoTrackSource : public rtc::AdaptedVideoTrackSource {
int64_t timestamp_ns,
const webrtc_jni::NativeHandleImpl& handle);
+ void OnFrameCaptured(JNIEnv* jni,
+ int width,
+ int height,
+ int64_t timestamp_ns,
+ VideoRotation rotation,
+ jobject j_video_frame_buffer);
+
void OnOutputFormatRequest(int width, int height, int fps);
rtc::scoped_refptr
@@ -75,7 +84,10 @@ class AndroidVideoTrackSource : public rtc::AdaptedVideoTrackSource {
NV12ToI420Scaler nv12toi420_scaler_;
I420BufferPool buffer_pool_;
rtc::scoped_refptr surface_texture_helper_;
+ webrtc_jni::AndroidVideoBufferFactory video_buffer_factory_;
const bool is_screencast_;
+
+ jmethodID j_crop_and_scale_id_;
};
} // namespace webrtc
diff --git a/webrtc/sdk/android/src/jni/androidvideotracksource_jni.cc b/webrtc/sdk/android/src/jni/androidvideotracksource_jni.cc
index 62f5f82574..56335227f8 100644
--- a/webrtc/sdk/android/src/jni/androidvideotracksource_jni.cc
+++ b/webrtc/sdk/android/src/jni/androidvideotracksource_jni.cc
@@ -69,6 +69,22 @@ JOW_OBSERVER_METHOD(void, nativeOnTextureFrameCaptured)
NativeHandleImpl(jni, j_oes_texture_id, j_transform_matrix));
}
+JOW_OBSERVER_METHOD(void, nativeOnFrameCaptured)
+(JNIEnv* jni,
+ jclass,
+ jlong j_source,
+ jint j_width,
+ jint j_height,
+ jint j_rotation,
+ jlong j_timestamp_ns,
+ jobject j_video_frame_buffer) {
+ webrtc::AndroidVideoTrackSource* source =
+ AndroidVideoTrackSourceFromJavaProxy(j_source);
+ source->OnFrameCaptured(jni, j_width, j_height, j_timestamp_ns,
+ jintToVideoRotation(j_rotation),
+ j_video_frame_buffer);
+}
+
JOW_OBSERVER_METHOD(void, nativeCapturerStarted)
(JNIEnv* jni, jclass, jlong j_source, jboolean j_success) {
LOG(LS_INFO) << "AndroidVideoTrackSourceObserve_nativeCapturerStarted";
diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.cc b/webrtc/sdk/android/src/jni/native_handle_impl.cc
index eebfba4035..cd78a27584 100644
--- a/webrtc/sdk/android/src/jni/native_handle_impl.cc
+++ b/webrtc/sdk/android/src/jni/native_handle_impl.cc
@@ -322,18 +322,39 @@ rtc::scoped_refptr AndroidTextureBuffer::ToI420() {
return copy;
}
+rtc::scoped_refptr AndroidVideoBuffer::WrapReference(
+ JNIEnv* jni,
+ jmethodID j_release_id,
+ int width,
+ int height,
+ jobject j_video_frame_buffer) {
+ return new rtc::RefCountedObject(
+ jni, j_release_id, width, height, j_video_frame_buffer);
+}
+
AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni,
jmethodID j_retain_id,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer)
+ : AndroidVideoBuffer(jni,
+ j_release_id,
+ width,
+ height,
+ j_video_frame_buffer) {
+ jni->CallVoidMethod(j_video_frame_buffer, j_retain_id);
+}
+
+AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni,
+ jmethodID j_release_id,
+ int width,
+ int height,
+ jobject j_video_frame_buffer)
: j_release_id_(j_release_id),
width_(width),
height_(height),
- j_video_frame_buffer_(jni, j_video_frame_buffer) {
- jni->CallVoidMethod(j_video_frame_buffer, j_retain_id);
-}
+ j_video_frame_buffer_(jni, j_video_frame_buffer) {}
AndroidVideoBuffer::~AndroidVideoBuffer() {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
@@ -422,15 +443,24 @@ webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame(
uint32_t timestamp_ns =
jni->CallLongMethod(j_video_frame, j_get_timestamp_ns_id_);
rtc::scoped_refptr buffer =
- CreateBuffer(j_video_frame_buffer);
+ CreateBuffer(jni, j_video_frame_buffer);
return webrtc::VideoFrame(buffer, timestamp_rtp,
timestamp_ns / rtc::kNumNanosecsPerMillisec,
static_cast(rotation));
}
-rtc::scoped_refptr AndroidVideoBufferFactory::CreateBuffer(
+rtc::scoped_refptr AndroidVideoBufferFactory::WrapBuffer(
+ JNIEnv* jni,
+ jobject j_video_frame_buffer) const {
+ int width = jni->CallIntMethod(j_video_frame_buffer, j_get_width_id_);
+ int height = jni->CallIntMethod(j_video_frame_buffer, j_get_height_id_);
+ return AndroidVideoBuffer::WrapReference(jni, j_release_id_, width, height,
+ j_video_frame_buffer);
+}
+
+rtc::scoped_refptr AndroidVideoBufferFactory::CreateBuffer(
+ JNIEnv* jni,
jobject j_video_frame_buffer) const {
- JNIEnv* jni = AttachCurrentThreadIfNeeded();
int width = jni->CallIntMethod(j_video_frame_buffer, j_get_width_id_);
int height = jni->CallIntMethod(j_video_frame_buffer, j_get_height_id_);
return new rtc::RefCountedObject(
diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.h b/webrtc/sdk/android/src/jni/native_handle_impl.h
index 834441a7c6..fc674db9d4 100644
--- a/webrtc/sdk/android/src/jni/native_handle_impl.h
+++ b/webrtc/sdk/android/src/jni/native_handle_impl.h
@@ -103,12 +103,28 @@ class AndroidTextureBuffer : public AndroidVideoFrameBuffer {
class AndroidVideoBuffer : public AndroidVideoFrameBuffer {
public:
+ // Wraps an existing reference to a Java VideoBuffer. Retain will not be
+ // called but release will be called when the C++ object is destroyed.
+ static rtc::scoped_refptr WrapReference(
+ JNIEnv* jni,
+ jmethodID j_release_id,
+ int width,
+ int height,
+ jobject j_video_frame_buffer);
+
AndroidVideoBuffer(JNIEnv* jni,
jmethodID j_retain_id,
jmethodID j_release_id,
int width,
int height,
jobject j_video_frame_buffer);
+ // Should not be called directly. Wraps a reference. Use
+ // AndroidVideoBuffer::WrapReference instead for clarity.
+ AndroidVideoBuffer(JNIEnv* jni,
+ jmethodID j_release_id,
+ int width,
+ int height,
+ jobject j_video_frame_buffer);
~AndroidVideoBuffer() override;
jobject video_frame_buffer() const;
@@ -140,7 +156,14 @@ class AndroidVideoBufferFactory {
jobject j_video_frame,
uint32_t timestamp_rtp) const;
+ // Wraps a buffer to AndroidVideoBuffer without incrementing the reference
+ // count.
+ rtc::scoped_refptr WrapBuffer(
+ JNIEnv* jni,
+ jobject j_video_frame_buffer) const;
+
rtc::scoped_refptr CreateBuffer(
+ JNIEnv* jni,
jobject j_video_frame_buffer) const;
private: