From 16032126eda1d723492248859e40c26cdb1e93ee Mon Sep 17 00:00:00 2001 From: Sami Kalliomaki Date: Wed, 20 Jul 2016 16:13:08 +0200 Subject: [PATCH] This implementation greatly simplifies Android video capturing stack. The old stack will be removed soon in a separate CL. Constraints will not be supported in the new implementation. Apps can request a format directly and the closest supported format will be selected. Changes needed from the apps: 1. Use the new createVideoSource without constraints. 2. Call startCapture manually. 3. Don't call videoSource.stop/restart, use startCapture/stopCapture instead. R=magjed@webrtc.org TBR=kjellander@webrtc.org Review URL: https://codereview.webrtc.org/2127893002 . Cr-Commit-Position: refs/heads/master@{#13504} --- webrtc/api/BUILD.gn | 3 + webrtc/api/OWNERS | 3 + .../java/src/org/webrtc/Camera2Capturer.java | 1 + .../src/org/webrtc/PeerConnectionFactory.java | 23 +- .../java/src/org/webrtc/VideoCapturer.java | 55 ++++ .../src/org/webrtc/VideoCapturerAndroid.java | 3 +- .../java/src/org/webrtc/VideoSource.java | 5 - webrtc/api/android/jni/OWNERS | 2 + .../jni/androidvideotracksource_jni.cc | 88 ++++++ webrtc/api/android/jni/peerconnection_jni.cc | 64 ++++- .../CameraVideoCapturerTestFixtures.java | 5 + webrtc/api/androidvideotracksource.cc | 260 ++++++++++++++++++ webrtc/api/androidvideotracksource.h | 124 +++++++++ webrtc/api/api.gyp | 3 + .../appspot/apprtc/PeerConnectionClient.java | 98 +++---- 15 files changed, 664 insertions(+), 73 deletions(-) create mode 100644 webrtc/api/android/jni/androidvideotracksource_jni.cc create mode 100644 webrtc/api/androidvideotracksource.cc create mode 100644 webrtc/api/androidvideotracksource.h diff --git a/webrtc/api/BUILD.gn b/webrtc/api/BUILD.gn index 391af74e7e..526e82c094 100644 --- a/webrtc/api/BUILD.gn +++ b/webrtc/api/BUILD.gn @@ -156,6 +156,7 @@ if (is_android && !build_with_chromium) { "android/jni/androidnetworkmonitor_jni.h", "android/jni/androidvideocapturer_jni.cc", "android/jni/androidvideocapturer_jni.h", + "android/jni/androidvideotracksource_jni.cc", "android/jni/classreferenceholder.cc", "android/jni/classreferenceholder.h", "android/jni/jni_helpers.cc", @@ -167,6 +168,8 @@ if (is_android && !build_with_chromium) { "android/jni/surfacetexturehelper_jni.h", "androidvideocapturer.cc", "androidvideocapturer.h", + "androidvideotracksource.cc", + "androidvideotracksource.h", ] configs += [ diff --git a/webrtc/api/OWNERS b/webrtc/api/OWNERS index d6441d3790..2160ffe125 100644 --- a/webrtc/api/OWNERS +++ b/webrtc/api/OWNERS @@ -6,6 +6,9 @@ tkchin@webrtc.org tommi@webrtc.org deadbeef@webrtc.org +per-file androidvideotracksource.*=sakal@webrtc.org +per-file androidvideotracksource.*=magjed@webrtc.org + # These are for the common case of adding or renaming files. If you're doing # structural changes, please get a review from a reviewer in this file. per-file *.gyp=* diff --git a/webrtc/api/android/java/src/org/webrtc/Camera2Capturer.java b/webrtc/api/android/java/src/org/webrtc/Camera2Capturer.java index 8110945a33..c3c912acd5 100644 --- a/webrtc/api/android/java/src/org/webrtc/Camera2Capturer.java +++ b/webrtc/api/android/java/src/org/webrtc/Camera2Capturer.java @@ -575,6 +575,7 @@ public class Camera2Capturer implements if (eventsHandler != null) { eventsHandler.onCameraClosed(); } + capturerObserver.onCapturerStopped(); } } diff --git a/webrtc/api/android/java/src/org/webrtc/PeerConnectionFactory.java b/webrtc/api/android/java/src/org/webrtc/PeerConnectionFactory.java index 2edd56e841..8e3e11d9b9 100644 --- a/webrtc/api/android/java/src/org/webrtc/PeerConnectionFactory.java +++ b/webrtc/api/android/java/src/org/webrtc/PeerConnectionFactory.java @@ -111,8 +111,9 @@ public class PeerConnectionFactory { nativeCreateLocalMediaStream(nativeFactory, label)); } - // The VideoSource takes ownership of |capturer|, so capturer.release() should not be called - // manually after this. + // The VideoSource takes ownership of |capturer|, so capturer.dispose() should not be called + // manually after this. Video capturer is automatically started so there is no need to call + // startCapture after this method. public VideoSource createVideoSource( VideoCapturer capturer, MediaConstraints constraints) { final EglBase.Context eglContext = @@ -121,6 +122,17 @@ public class PeerConnectionFactory { eglContext, capturer, constraints)); } + public VideoSource createVideoSource(VideoCapturer capturer) { + final EglBase.Context eglContext = + localEglbase == null ? null : localEglbase.getEglBaseContext(); + long nativeAndroidVideoTrackSource = nativeCreateVideoSource2(nativeFactory, eglContext); + VideoCapturer.CapturerObserver capturerObserver + = new VideoCapturer.AndroidVideoTrackSourceObserver(nativeAndroidVideoTrackSource); + nativeInitializeVideoCapturer(nativeFactory, capturer, nativeAndroidVideoTrackSource, + capturerObserver); + return new VideoSource(nativeAndroidVideoTrackSource); + } + public VideoTrack createVideoTrack(String id, VideoSource source) { return new VideoTrack(nativeCreateVideoTrack( nativeFactory, id, source.nativeSource)); @@ -239,6 +251,13 @@ public class PeerConnectionFactory { long nativeFactory, EglBase.Context eglContext, VideoCapturer videoCapturer, MediaConstraints constraints); + private static native long nativeCreateVideoSource2( + long nativeFactory, EglBase.Context eglContext); + + private static native void nativeInitializeVideoCapturer( + long native_factory, VideoCapturer j_video_capturer, long native_source, + VideoCapturer.CapturerObserver j_frame_observer); + private static native long nativeCreateVideoTrack( long nativeFactory, String id, long nativeVideoSource); diff --git a/webrtc/api/android/java/src/org/webrtc/VideoCapturer.java b/webrtc/api/android/java/src/org/webrtc/VideoCapturer.java index afa3a0563b..c92f82ac18 100644 --- a/webrtc/api/android/java/src/org/webrtc/VideoCapturer.java +++ b/webrtc/api/android/java/src/org/webrtc/VideoCapturer.java @@ -21,6 +21,7 @@ public interface VideoCapturer { // Notify if the camera have been started successfully or not. // Called on a Java thread owned by VideoCapturer. void onCapturerStarted(boolean success); + void onCapturerStopped(); // Delivers a captured frame. Called on a Java thread owned by VideoCapturer. void onByteBufferFrameCaptured(byte[] data, int width, int height, int rotation, @@ -52,6 +53,9 @@ public interface VideoCapturer { nativeCapturerStarted(nativeCapturer, success); } + @Override + public void onCapturerStopped() {} + @Override public void onByteBufferFrameCaptured(byte[] data, int width, int height, int rotation, long timeStamp) { @@ -82,6 +86,57 @@ public interface VideoCapturer { int width, int height, int framerate); } + // An implementation of CapturerObserver that forwards all calls from + // Java to the C layer. + static class AndroidVideoTrackSourceObserver implements CapturerObserver { + // Pointer to VideoTrackSourceProxy proxying AndroidVideoTrackSource. + private final long nativeSource; + + public AndroidVideoTrackSourceObserver(long nativeSource) { + this.nativeSource = nativeSource; + } + + @Override + public void onCapturerStarted(boolean success) { + nativeCapturerStarted(nativeSource, success); + } + + @Override + public void onCapturerStopped() { + nativeCapturerStopped(nativeSource); + } + + @Override + public void onByteBufferFrameCaptured(byte[] data, int width, int height, + int rotation, long timeStamp) { + nativeOnByteBufferFrameCaptured(nativeSource, data, data.length, width, height, rotation, + timeStamp); + } + + @Override + public void onTextureFrameCaptured( + int width, int height, int oesTextureId, float[] transformMatrix, int rotation, + long timestamp) { + nativeOnTextureFrameCaptured(nativeSource, width, height, oesTextureId, transformMatrix, + rotation, timestamp); + } + + @Override + public void onOutputFormatRequest(int width, int height, int framerate) { + nativeOnOutputFormatRequest(nativeSource, width, height, framerate); + } + + private native void nativeCapturerStarted(long nativeSource, + boolean success); + private native void nativeCapturerStopped(long nativeSource); + private native void nativeOnByteBufferFrameCaptured(long nativeSource, + byte[] data, int length, int width, int height, int rotation, long timeStamp); + private native void nativeOnTextureFrameCaptured(long nativeSource, int width, int height, + int oesTextureId, float[] transformMatrix, int rotation, long timestamp); + private native void nativeOnOutputFormatRequest(long nativeSource, + int width, int height, int framerate); + } + /** * Returns a list with all the formats this VideoCapturer supports. */ diff --git a/webrtc/api/android/java/src/org/webrtc/VideoCapturerAndroid.java b/webrtc/api/android/java/src/org/webrtc/VideoCapturerAndroid.java index 1e9c7ee712..d3a9faff21 100644 --- a/webrtc/api/android/java/src/org/webrtc/VideoCapturerAndroid.java +++ b/webrtc/api/android/java/src/org/webrtc/VideoCapturerAndroid.java @@ -23,9 +23,9 @@ import java.nio.ByteBuffer; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; // Android specific implementation of VideoCapturer. // An instance of this class can be created by an application using @@ -484,6 +484,7 @@ public class VideoCapturerAndroid implements eventsHandler.onCameraError("Camera stop timeout"); } } + frameObserver.onCapturerStopped(); Logging.d(TAG, "stopCapture done"); } diff --git a/webrtc/api/android/java/src/org/webrtc/VideoSource.java b/webrtc/api/android/java/src/org/webrtc/VideoSource.java index 3864ccbb40..ba5d2ca6af 100644 --- a/webrtc/api/android/java/src/org/webrtc/VideoSource.java +++ b/webrtc/api/android/java/src/org/webrtc/VideoSource.java @@ -36,11 +36,6 @@ public class VideoSource extends MediaSource { restart(nativeSource); } - @Override - public void dispose() { - super.dispose(); - } - private static native void stop(long nativeSource); private static native void restart(long nativeSource); } diff --git a/webrtc/api/android/jni/OWNERS b/webrtc/api/android/jni/OWNERS index 4178fd6920..14c6742b58 100644 --- a/webrtc/api/android/jni/OWNERS +++ b/webrtc/api/android/jni/OWNERS @@ -2,6 +2,8 @@ per-file androidvideocapturer*=magjed@webrtc.org per-file androidmediaencoder*=magjed@webrtc.org per-file androidmediadecoder*=magjed@webrtc.org per-file androidmediacodeccommon.h=magjed@webrtc.org +per-file androidvideotracksource_jni.cc=magjed@webrtc.org +per-file androidvideotracksource_jni.cc=sakal@webrtc.org per-file surfacetexturehelper*=magjed@webrtc.org per-file native_handle_impl*=magjed@webrtc.org # Video related parts of peerconnection only. diff --git a/webrtc/api/android/jni/androidvideotracksource_jni.cc b/webrtc/api/android/jni/androidvideotracksource_jni.cc new file mode 100644 index 0000000000..5ec281ab98 --- /dev/null +++ b/webrtc/api/android/jni/androidvideotracksource_jni.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/api/android/jni/classreferenceholder.h" +#include "webrtc/api/androidvideotracksource.h" +#include "webrtc/api/videosourceproxy.h" + +// Identifiers are over 80 characters long so this is needed to fit them on one +// line. +#define JOW_OBSERVER_METHOD(rettype, name) \ + JOW(rettype, VideoCapturer_00024AndroidVideoTrackSourceObserver_##name) + +namespace webrtc_jni { + +static webrtc::AndroidVideoTrackSource* AndroidVideoTrackSourceFromJavaProxy( + jlong j_proxy) { + auto proxy_source = reinterpret_cast(j_proxy); + return reinterpret_cast( + proxy_source->internal()); +} + +JOW_OBSERVER_METHOD(void, nativeOnByteBufferFrameCaptured) +(JNIEnv* jni, + jclass, + jlong j_source, + jbyteArray j_frame, + jint length, + jint width, + jint height, + jint rotation, + jlong timestamp) { + webrtc::AndroidVideoTrackSource* source = + AndroidVideoTrackSourceFromJavaProxy(j_source); + jbyte* bytes = jni->GetByteArrayElements(j_frame, nullptr); + source->OnByteBufferFrameCaptured(bytes, length, width, height, rotation, + timestamp); + jni->ReleaseByteArrayElements(j_frame, bytes, JNI_ABORT); +} + +JOW_OBSERVER_METHOD(void, nativeOnTextureFrameCaptured) +(JNIEnv* jni, + jclass, + jlong j_source, + jint j_width, + jint j_height, + jint j_oes_texture_id, + jfloatArray j_transform_matrix, + jint j_rotation, + jlong j_timestamp) { + webrtc::AndroidVideoTrackSource* source = + AndroidVideoTrackSourceFromJavaProxy(j_source); + source->OnTextureFrameCaptured( + j_width, j_height, j_rotation, j_timestamp, + NativeHandleImpl(jni, j_oes_texture_id, j_transform_matrix)); +} + +JOW_OBSERVER_METHOD(void, nativeCapturerStarted) +(JNIEnv* jni, jclass, jlong j_source, jboolean j_success) { + LOG(LS_INFO) << "AndroidVideoTrackSourceObserve_nativeCapturerStarted"; + webrtc::AndroidVideoTrackSource* source = + AndroidVideoTrackSourceFromJavaProxy(j_source); + source->SetState(webrtc::AndroidVideoTrackSource::SourceState::kLive); +} + +JOW_OBSERVER_METHOD(void, nativeCapturerStopped) +(JNIEnv* jni, jclass, jlong j_source) { + LOG(LS_INFO) << "AndroidVideoTrackSourceObserve_nativeCapturerStopped"; + webrtc::AndroidVideoTrackSource* source = + AndroidVideoTrackSourceFromJavaProxy(j_source); + source->SetState(webrtc::AndroidVideoTrackSource::SourceState::kEnded); +} + +JOW_OBSERVER_METHOD(void, nativeOnOutputFormatRequest) +(JNIEnv* jni, jclass, jlong j_source, jint j_width, jint j_height, jint j_fps) { + LOG(LS_INFO) << "AndroidVideoTrackSourceObserve_nativeOnOutputFormatRequest"; + webrtc::AndroidVideoTrackSource* source = + AndroidVideoTrackSourceFromJavaProxy(j_source); + source->OnOutputFormatRequest(j_width, j_height, j_fps); +} + +} // namespace webrtc_jni diff --git a/webrtc/api/android/jni/peerconnection_jni.cc b/webrtc/api/android/jni/peerconnection_jni.cc index fefb44785a..8b5a69251b 100644 --- a/webrtc/api/android/jni/peerconnection_jni.cc +++ b/webrtc/api/android/jni/peerconnection_jni.cc @@ -44,6 +44,7 @@ #include #include "webrtc/api/androidvideocapturer.h" +#include "webrtc/api/androidvideotracksource.h" #include "webrtc/api/android/jni/androidmediadecoder_jni.h" #include "webrtc/api/android/jni/androidmediaencoder_jni.h" #include "webrtc/api/android/jni/androidnetworkmonitor_jni.h" @@ -55,6 +56,7 @@ #include "webrtc/api/peerconnectioninterface.h" #include "webrtc/api/rtpreceiverinterface.h" #include "webrtc/api/rtpsenderinterface.h" +#include "webrtc/api/videosourceproxy.h" #include "webrtc/api/webrtcsdp.h" #include "webrtc/base/bind.h" #include "webrtc/base/checks.h" @@ -116,6 +118,7 @@ static char *field_trials_init_string = NULL; // Set in PeerConnectionFactory_initializeAndroidGlobals(). static bool factory_static_initialized = false; static bool video_hw_acceleration_enabled = true; +static jobject j_application_context = nullptr; // Return the (singleton) Java Enum object corresponding to |index|; // |state_class_fragment| is something like "MediaSource$State". @@ -931,7 +934,7 @@ JOW(void, PeerConnection_freeObserver)(JNIEnv*, jclass, jlong j_p) { } JOW(void, MediaSource_free)(JNIEnv*, jclass, jlong j_p) { - CHECK_RELEASE(reinterpret_cast(j_p)); + reinterpret_cast(j_p)->Release(); } JOW(void, VideoRenderer_freeWrappedVideoRenderer)(JNIEnv*, jclass, jlong j_p) { @@ -985,14 +988,20 @@ JOW(jlong, PeerConnectionFactory_nativeCreateObserver)( return (jlong)new PCOJava(jni, j_observer); } -JOW(jboolean, PeerConnectionFactory_initializeAndroidGlobals)( - JNIEnv* jni, jclass, jobject context, - jboolean initialize_audio, jboolean initialize_video, - jboolean video_hw_acceleration) { +JOW(jboolean, PeerConnectionFactory_initializeAndroidGlobals) +(JNIEnv* jni, + jclass, + jobject context, + jboolean initialize_audio, + jboolean initialize_video, + jboolean video_hw_acceleration) { bool failure = false; video_hw_acceleration_enabled = video_hw_acceleration; AndroidNetworkMonitor::SetAndroidContext(jni, context); if (!factory_static_initialized) { + RTC_DCHECK(j_application_context == nullptr); + j_application_context = NewGlobalRef(jni, context); + if (initialize_video) { failure |= AndroidVideoCapturerJni::SetAndroidObjects(jni, context); } @@ -1075,6 +1084,8 @@ class OwnedFactoryAndThreads { } PeerConnectionFactoryInterface* factory() { return factory_; } + Thread* signaling_thread() { return signaling_thread_.get(); } + Thread* worker_thread() { return worker_thread_.get(); } WebRtcVideoEncoderFactory* encoder_factory() { return encoder_factory_; } WebRtcVideoDecoderFactory* decoder_factory() { return decoder_factory_; } rtc::NetworkMonitorFactory* network_monitor_factory() { @@ -1270,6 +1281,49 @@ JOW(jlong, PeerConnectionFactory_nativeCreateVideoSource)( return (jlong)source.release(); } +JOW(jlong, PeerConnectionFactory_nativeCreateVideoSource2) +(JNIEnv* jni, jclass, jlong native_factory, jobject j_egl_context) { + OwnedFactoryAndThreads* factory = + reinterpret_cast(native_factory); + + rtc::scoped_refptr source( + new rtc::RefCountedObject( + factory->signaling_thread(), jni, j_egl_context)); + rtc::scoped_refptr proxy_source = + webrtc::VideoTrackSourceProxy::Create(factory->signaling_thread(), + factory->worker_thread(), source); + + return (jlong)proxy_source.release(); +} + +JOW(void, PeerConnectionFactory_nativeInitializeVideoCapturer) +(JNIEnv* jni, + jclass, + jlong native_factory, + jobject j_video_capturer, + jlong native_source, + jobject j_frame_observer) { + LOG(LS_INFO) << "PeerConnectionFactory_nativeInitializeVideoCapturer"; + rtc::scoped_refptr factory( + factoryFromJava(native_factory)); + auto proxy_source = + reinterpret_cast(native_source); + auto source = reinterpret_cast( + proxy_source->internal()); + rtc::scoped_refptr surface_texture_helper = + source->surface_texture_helper(); + jni->CallVoidMethod( + j_video_capturer, + GetMethodID(jni, FindClass(jni, "org/webrtc/VideoCapturer"), "initialize", + "(Lorg/webrtc/SurfaceTextureHelper;Landroid/content/" + "Context;Lorg/webrtc/VideoCapturer$CapturerObserver;)V"), + surface_texture_helper + ? surface_texture_helper->GetJavaSurfaceTextureHelper() + : nullptr, + j_application_context, j_frame_observer); + CHECK_EXCEPTION(jni) << "error during VideoCapturer.initialize()"; +} + JOW(jlong, PeerConnectionFactory_nativeCreateVideoTrack)( JNIEnv* jni, jclass, jlong native_factory, jstring id, jlong native_source) { diff --git a/webrtc/api/androidtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java b/webrtc/api/androidtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java index 729e99a8d9..4c933db0e3 100644 --- a/webrtc/api/androidtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java +++ b/webrtc/api/androidtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java @@ -104,6 +104,11 @@ class CameraVideoCapturerTestFixtures { } } + @Override + public void onCapturerStopped() { + Logging.d(TAG, "onCapturerStopped"); + } + @Override public void onByteBufferFrameCaptured(byte[] frame, int width, int height, int rotation, long timeStamp) { diff --git a/webrtc/api/androidvideotracksource.cc b/webrtc/api/androidvideotracksource.cc new file mode 100644 index 0000000000..c52e4f84ff --- /dev/null +++ b/webrtc/api/androidvideotracksource.cc @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/api/androidvideotracksource.h" + +#include + +namespace webrtc { + +AndroidVideoTrackSource::AndroidVideoTrackSource(rtc::Thread* signaling_thread, + JNIEnv* jni, + jobject j_egl_context) + : signaling_thread_(signaling_thread), + surface_texture_helper_(webrtc_jni::SurfaceTextureHelper::create( + jni, + "Camera SurfaceTextureHelper", + j_egl_context)) { + LOG(LS_INFO) << "AndroidVideoTrackSource ctor"; + worker_thread_checker_.DetachFromThread(); + camera_thread_checker_.DetachFromThread(); +} + +bool AndroidVideoTrackSource::GetStats(AndroidVideoTrackSource::Stats* stats) { + rtc::CritScope lock(&stats_crit_); + + if (!stats_) { + return false; + } + + *stats = *stats_; + return true; +} + +void AndroidVideoTrackSource::SetState(SourceState state) { + if (rtc::Thread::Current() != signaling_thread_) { + invoker_.AsyncInvoke( + RTC_FROM_HERE, signaling_thread_, + rtc::Bind(&AndroidVideoTrackSource::SetState, this, state)); + return; + } + + if (state_ != state) { + state_ = state; + FireOnChanged(); + } +} + +void AndroidVideoTrackSource::AddOrUpdateSink( + rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); + + broadcaster_.AddOrUpdateSink(sink, wants); + OnSinkWantsChanged(broadcaster_.wants()); +} + +void AndroidVideoTrackSource::RemoveSink( + rtc::VideoSinkInterface* sink) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); + + broadcaster_.RemoveSink(sink); + OnSinkWantsChanged(broadcaster_.wants()); +} + +void AndroidVideoTrackSource::OnSinkWantsChanged( + const rtc::VideoSinkWants& wants) { + { + rtc::CritScope lock(&apply_rotation_crit_); + apply_rotation_ = wants.rotation_applied; + } + + video_adapter_.OnResolutionRequest(wants.max_pixel_count, + wants.max_pixel_count_step_up); +} + +void AndroidVideoTrackSource::OnByteBufferFrameCaptured(const void* frame_data, + int length, + int width, + int height, + int rotation, + int64_t timestamp_ns) { + RTC_DCHECK(camera_thread_checker_.CalledOnValidThread()); + RTC_DCHECK(rotation == 0 || rotation == 90 || rotation == 180 || + rotation == 270); + + int adapted_width; + int adapted_height; + int crop_width; + int crop_height; + int crop_x; + int crop_y; + int64_t translated_camera_time_us; + + if (!AdaptFrame(width, height, timestamp_ns / rtc::kNumNanosecsPerMicrosec, + &adapted_width, &adapted_height, &crop_width, &crop_height, + &crop_x, &crop_y, &translated_camera_time_us)) { + return; + } + + int rotated_width = crop_width; + int rotated_height = crop_height; + + rtc::CritScope lock(&apply_rotation_crit_); + if (apply_rotation_ && (rotation == 90 || rotation == 270)) { + std::swap(adapted_width, adapted_height); + std::swap(rotated_width, rotated_height); + } + + rtc::scoped_refptr buffer = + pre_scale_pool_.CreateBuffer(rotated_width, rotated_height); + + const uint8_t* y_plane = static_cast(frame_data); + const uint8_t* uv_plane = y_plane + width * height; + int uv_width = (width + 1) / 2; + + RTC_CHECK_GE(length, width * height + 2 * uv_width * ((height + 1) / 2)); + + // Can only crop at even pixels. + crop_x &= ~1; + crop_y &= ~1; + + libyuv::NV12ToI420Rotate( + y_plane + width * crop_y + crop_x, width, + uv_plane + uv_width * crop_y + crop_x, width, buffer->MutableDataY(), + buffer->StrideY(), + // Swap U and V, since we have NV21, not NV12. + buffer->MutableDataV(), buffer->StrideV(), buffer->MutableDataU(), + buffer->StrideU(), crop_width, crop_height, + static_cast(apply_rotation_ ? rotation : 0)); + + if (adapted_width != buffer->width() || adapted_height != buffer->height()) { + rtc::scoped_refptr scaled_buffer( + post_scale_pool_.CreateBuffer(adapted_width, adapted_height)); + scaled_buffer->ScaleFrom(buffer); + buffer = scaled_buffer; + } + + OnFrame(cricket::WebRtcVideoFrame( + buffer, + apply_rotation_ ? webrtc::kVideoRotation_0 + : static_cast(rotation), + translated_camera_time_us), + width, height); +} + +void AndroidVideoTrackSource::OnTextureFrameCaptured( + int width, + int height, + int rotation, + int64_t timestamp_ns, + const webrtc_jni::NativeHandleImpl& handle) { + RTC_DCHECK(camera_thread_checker_.CalledOnValidThread()); + RTC_DCHECK(rotation == 0 || rotation == 90 || rotation == 180 || + rotation == 270); + + int adapted_width; + int adapted_height; + int crop_width; + int crop_height; + int crop_x; + int crop_y; + int64_t translated_camera_time_us; + + if (!AdaptFrame(width, height, timestamp_ns / rtc::kNumNanosecsPerMicrosec, + &adapted_width, &adapted_height, &crop_width, &crop_height, + &crop_x, &crop_y, &translated_camera_time_us)) { + surface_texture_helper_->ReturnTextureFrame(); + return; + } + + webrtc_jni::Matrix matrix = handle.sampling_matrix; + + matrix.Crop(crop_width / static_cast(width), + crop_height / static_cast(height), + crop_x / static_cast(width), + crop_y / static_cast(height)); + + rtc::CritScope lock(&apply_rotation_crit_); + if (apply_rotation_) { + if (rotation == webrtc::kVideoRotation_90 || + rotation == webrtc::kVideoRotation_270) { + std::swap(adapted_width, adapted_height); + } + matrix.Rotate(static_cast(rotation)); + } + + OnFrame(cricket::WebRtcVideoFrame( + surface_texture_helper_->CreateTextureFrame( + adapted_width, adapted_height, + webrtc_jni::NativeHandleImpl(handle.oes_texture_id, matrix)), + apply_rotation_ ? webrtc::kVideoRotation_0 + : static_cast(rotation), + translated_camera_time_us), + width, height); +} + +void AndroidVideoTrackSource::OnFrame(const cricket::VideoFrame& frame, + int width, + int height) { + { + rtc::CritScope lock(&stats_crit_); + stats_ = rtc::Optional({width, height}); + } + + broadcaster_.OnFrame(frame); +} + +void AndroidVideoTrackSource::OnOutputFormatRequest(int width, + int height, + int fps) { + RTC_DCHECK(camera_thread_checker_.CalledOnValidThread()); + + cricket::VideoFormat format(width, height, + cricket::VideoFormat::FpsToInterval(fps), 0); + video_adapter_.OnOutputFormatRequest(format); +} + +bool AndroidVideoTrackSource::AdaptFrame(int width, + int height, + int64_t camera_time_us, + int* out_width, + int* out_height, + int* crop_width, + int* crop_height, + int* crop_x, + int* crop_y, + int64_t* translated_camera_time_us) { + RTC_DCHECK(camera_thread_checker_.CalledOnValidThread()); + + int64_t system_time_us = rtc::TimeMicros(); + + int64_t offset_us = + timestamp_aligner_.UpdateOffset(camera_time_us, system_time_us); + + if (!broadcaster_.frame_wanted()) { + return false; + } + + if (!video_adapter_.AdaptFrameResolution( + width, height, camera_time_us * rtc::kNumNanosecsPerMicrosec, + crop_width, crop_height, out_width, out_height)) { + // VideoAdapter dropped the frame. + return false; + } + *crop_x = (width - *crop_width) / 2; + *crop_y = (height - *crop_height) / 2; + + *translated_camera_time_us = timestamp_aligner_.ClipTimestamp( + camera_time_us + offset_us, system_time_us); + return true; +} + +} // namespace webrtc diff --git a/webrtc/api/androidvideotracksource.h b/webrtc/api/androidvideotracksource.h new file mode 100644 index 0000000000..cc6ead4014 --- /dev/null +++ b/webrtc/api/androidvideotracksource.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_API_ANDROIDVIDEOTRACKSOURCE_H_ +#define WEBRTC_API_ANDROIDVIDEOTRACKSOURCE_H_ + +#include "webrtc/api/android/jni/native_handle_impl.h" +#include "webrtc/api/android/jni/surfacetexturehelper_jni.h" +#include "webrtc/api/mediastreaminterface.h" +#include "webrtc/api/notifier.h" +#include "webrtc/base/asyncinvoker.h" +#include "webrtc/base/checks.h" +#include "webrtc/base/thread_checker.h" +#include "webrtc/base/timestampaligner.h" +#include "webrtc/common_video/include/i420_buffer_pool.h" +#include "webrtc/media/base/videoadapter.h" +#include "webrtc/media/base/videobroadcaster.h" +#include "webrtc/media/base/videosinkinterface.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +namespace webrtc { + +class AndroidVideoTrackSource : public Notifier { + public: + AndroidVideoTrackSource(rtc::Thread* signaling_thread, + JNIEnv* jni, + jobject j_egl_context); + + // Not used on Android. + // TODO(sakal/magjed): Try to remove this from the interface. + void Stop() override { RTC_NOTREACHED(); }; + // Not used on Android. + // TODO(sakal/magjed): Try to remove this from the interface. + void Restart() override { RTC_NOTREACHED(); } + + // Currently, none of the Android implementations are screencast. + bool is_screencast() const override { return false; } + + // Indicates that the encoder should denoise video before encoding it. + // If it is not set, the default configuration is used which is different + // depending on video codec. + rtc::Optional needs_denoising() const override { + return rtc::Optional(false); + } + + // Returns false if no stats are available, e.g, for a remote + // source, or a source which has not seen its first frame yet. + // Should avoid blocking. + bool GetStats(Stats* stats) override; + + // Called by the native capture observer + void SetState(SourceState state); + + SourceState state() const override { return state_; } + + bool remote() const override { return false; } + + void AddOrUpdateSink(rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) override; + void RemoveSink(rtc::VideoSinkInterface* sink) override; + + void OnByteBufferFrameCaptured(const void* frame_data, + int length, + int width, + int height, + int rotation, + int64_t timestamp_ns); + + void OnTextureFrameCaptured(int width, + int height, + int rotation, + int64_t timestamp_ns, + const webrtc_jni::NativeHandleImpl& handle); + + void OnOutputFormatRequest(int width, int height, int fps); + + rtc::scoped_refptr + surface_texture_helper() { + return surface_texture_helper_; + } + + private: + rtc::Thread* signaling_thread_; + rtc::AsyncInvoker invoker_; + rtc::ThreadChecker worker_thread_checker_; + rtc::ThreadChecker camera_thread_checker_; + rtc::CriticalSection stats_crit_; + rtc::Optional stats_ GUARDED_BY(stats_crit_); + SourceState state_; + rtc::VideoBroadcaster broadcaster_; + rtc::TimestampAligner timestamp_aligner_; + cricket::VideoAdapter video_adapter_; + rtc::CriticalSection apply_rotation_crit_; + bool apply_rotation_ GUARDED_BY(apply_rotation_crit_); + webrtc::I420BufferPool pre_scale_pool_; + webrtc::I420BufferPool post_scale_pool_; + rtc::scoped_refptr surface_texture_helper_; + + void OnFrame(const cricket::VideoFrame& frame, int width, int height); + + void OnSinkWantsChanged(const rtc::VideoSinkWants& wants); + + bool AdaptFrame(int width, + int height, + int64_t camera_time_us, + int* out_width, + int* out_height, + int* crop_width, + int* crop_height, + int* crop_x, + int* crop_y, + int64_t* translated_camera_time_us); +}; + +} // namespace webrtc + +#endif // WEBRTC_API_ANDROIDVIDEOTRACKSOURCE_H_ diff --git a/webrtc/api/api.gyp b/webrtc/api/api.gyp index 15c88dd0f3..86a55fd9d2 100644 --- a/webrtc/api/api.gyp +++ b/webrtc/api/api.gyp @@ -45,6 +45,7 @@ 'android/jni/androidnetworkmonitor_jni.h', 'android/jni/androidvideocapturer_jni.cc', 'android/jni/androidvideocapturer_jni.h', + 'android/jni/androidvideotracksource_jni.cc', 'android/jni/classreferenceholder.cc', 'android/jni/classreferenceholder.h', 'android/jni/jni_helpers.cc', @@ -56,6 +57,8 @@ 'android/jni/surfacetexturehelper_jni.h', 'androidvideocapturer.cc', 'androidvideocapturer.h', + 'androidvideotracksource.cc', + 'androidvideotracksource.h', ], 'include_dirs': [ '<(libyuv_dir)/include', diff --git a/webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java b/webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java index 0f7ff3532d..f12d5ce48f 100644 --- a/webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java +++ b/webrtc/examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java @@ -107,14 +107,16 @@ public class PeerConnectionClient { private boolean videoCallEnabled; private boolean preferIsac; private String preferredVideoCodec; - private boolean videoSourceStopped; + private boolean videoCapturerStopped; private boolean isError; private Timer statsTimer; private VideoRenderer.Callbacks localRender; private VideoRenderer.Callbacks remoteRender; private SignalingParameters signalingParameters; private MediaConstraints pcConstraints; - private MediaConstraints videoConstraints; + private int videoWidth; + private int videoHeight; + private int videoFps; private MediaConstraints audioConstraints; private ParcelFileDescriptor aecDumpFileDescriptor; private MediaConstraints sdpMediaConstraints; @@ -260,7 +262,7 @@ public class PeerConnectionClient { factory = null; peerConnection = null; preferIsac = false; - videoSourceStopped = false; + videoCapturerStopped = false; isError = false; queuedRemoteCandidates = null; localSdp = null; // either offer or answer SDP @@ -399,42 +401,24 @@ public class PeerConnectionClient { } // Create video constraints if video call is enabled. if (videoCallEnabled) { - videoConstraints = new MediaConstraints(); - int videoWidth = peerConnectionParameters.videoWidth; - int videoHeight = peerConnectionParameters.videoHeight; + videoWidth = peerConnectionParameters.videoWidth; + videoHeight = peerConnectionParameters.videoHeight; + videoFps = peerConnectionParameters.videoFps; - // If VP8 HW video encoder is supported and video resolution is not - // specified force it to HD. - if ((videoWidth == 0 || videoHeight == 0) - && peerConnectionParameters.videoCodecHwAcceleration - && MediaCodecVideoEncoder.isVp8HwSupported()) { + // If video resolution is not specified, default to HD. + if (videoWidth == 0 || videoHeight == 0) { videoWidth = HD_VIDEO_WIDTH; videoHeight = HD_VIDEO_HEIGHT; } - // Add video resolution constraints. - if (videoWidth > 0 && videoHeight > 0) { - videoWidth = Math.min(videoWidth, MAX_VIDEO_WIDTH); - videoHeight = Math.min(videoHeight, MAX_VIDEO_HEIGHT); - videoConstraints.mandatory.add(new KeyValuePair( - MIN_VIDEO_WIDTH_CONSTRAINT, Integer.toString(videoWidth))); - videoConstraints.mandatory.add(new KeyValuePair( - MAX_VIDEO_WIDTH_CONSTRAINT, Integer.toString(videoWidth))); - videoConstraints.mandatory.add(new KeyValuePair( - MIN_VIDEO_HEIGHT_CONSTRAINT, Integer.toString(videoHeight))); - videoConstraints.mandatory.add(new KeyValuePair( - MAX_VIDEO_HEIGHT_CONSTRAINT, Integer.toString(videoHeight))); + // If fps is not specified, default to 30. + if (videoFps == 0) { + videoFps = 30; } - // Add fps constraints. - int videoFps = peerConnectionParameters.videoFps; - if (videoFps > 0) { - videoFps = Math.min(videoFps, MAX_VIDEO_FPS); - videoConstraints.mandatory.add(new KeyValuePair( - MIN_VIDEO_FPS_CONSTRAINT, Integer.toString(videoFps))); - videoConstraints.mandatory.add(new KeyValuePair( - MAX_VIDEO_FPS_CONSTRAINT, Integer.toString(videoFps))); - } + videoWidth = Math.min(videoWidth, MAX_VIDEO_WIDTH); + videoHeight = Math.min(videoHeight, MAX_VIDEO_HEIGHT); + videoFps = Math.min(videoFps, MAX_VIDEO_FPS); } // Create audio constraints. @@ -502,9 +486,6 @@ public class PeerConnectionClient { Log.d(TAG, "Create peer connection."); Log.d(TAG, "PCConstraints: " + pcConstraints.toString()); - if (videoConstraints != null) { - Log.d(TAG, "VideoConstraints: " + videoConstraints.toString()); - } queuedRemoteCandidates = new LinkedList(); if (videoCallEnabled) { @@ -592,6 +573,16 @@ public class PeerConnectionClient { audioSource.dispose(); audioSource = null; } + Log.d(TAG, "Stopping capture."); + if (videoCapturer != null) { + try { + videoCapturer.stopCapture(); + } catch(InterruptedException e) { + throw new RuntimeException(e); + } + videoCapturer.dispose(); + videoCapturer = null; + } Log.d(TAG, "Closing video source."); if (videoSource != null) { videoSource.dispose(); @@ -613,24 +604,8 @@ public class PeerConnectionClient { if (!videoCallEnabled) { return false; } - int minWidth = 0; - int minHeight = 0; - for (KeyValuePair keyValuePair : videoConstraints.mandatory) { - if (keyValuePair.getKey().equals("minWidth")) { - try { - minWidth = Integer.parseInt(keyValuePair.getValue()); - } catch (NumberFormatException e) { - Log.e(TAG, "Can not parse video width from video constraints"); - } - } else if (keyValuePair.getKey().equals("minHeight")) { - try { - minHeight = Integer.parseInt(keyValuePair.getValue()); - } catch (NumberFormatException e) { - Log.e(TAG, "Can not parse video height from video constraints"); - } - } - } - return minWidth * minHeight >= 1280 * 720; + + return videoWidth * videoHeight >= 1280 * 720; } private void getStats() { @@ -791,10 +766,12 @@ public class PeerConnectionClient { executor.execute(new Runnable() { @Override public void run() { - if (videoSource != null && !videoSourceStopped) { + if (videoCapturer != null && !videoCapturerStopped) { Log.d(TAG, "Stop video source."); - videoSource.stop(); - videoSourceStopped = true; + try { + videoCapturer.stopCapture(); + } catch (InterruptedException e) {} + videoCapturerStopped = true; } } }); @@ -804,10 +781,10 @@ public class PeerConnectionClient { executor.execute(new Runnable() { @Override public void run() { - if (videoSource != null && videoSourceStopped) { + if (videoCapturer != null && videoCapturerStopped) { Log.d(TAG, "Restart video source."); - videoSource.restart(); - videoSourceStopped = false; + videoCapturer.startCapture(videoWidth, videoHeight, videoFps); + videoCapturerStopped = false; } } }); @@ -834,7 +811,8 @@ public class PeerConnectionClient { } private VideoTrack createVideoTrack(VideoCapturer capturer) { - videoSource = factory.createVideoSource(capturer, videoConstraints); + videoSource = factory.createVideoSource(capturer); + capturer.startCapture(videoWidth, videoHeight, videoFps); localVideoTrack = factory.createVideoTrack(VIDEO_TRACK_ID, videoSource); localVideoTrack.setEnabled(renderVideo);