From f7030d4ed700f66494ede4702ceed5176c931097 Mon Sep 17 00:00:00 2001 From: "glaznev@webrtc.org" Date: Fri, 17 Oct 2014 16:25:06 +0000 Subject: [PATCH] Query Android device orientation on every camera frame received. Remove orientation listener from Android camera, since device orientation change events are not well synchronized with actual device display orientation. Plus these event may not be delivered at all if device is in stationary position causing initial camera frames appear rotated. BUG= R=braveyao@webrtc.org Review URL: https://webrtc-codereview.appspot.com/23009004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@7467 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../videoengine/VideoCaptureAndroid.java | 71 +++++++++++-------- .../android/video_capture_android.cc | 45 +++++------- .../android/video_capture_android.h | 2 + .../test/video_capture_unittest.cc | 8 ++- 4 files changed, 68 insertions(+), 58 deletions(-) diff --git a/webrtc/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java b/webrtc/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java index c1db65ac0c..926b350e19 100644 --- a/webrtc/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java +++ b/webrtc/modules/video_capture/android/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java @@ -24,9 +24,10 @@ import android.os.Handler; import android.os.Looper; import android.os.SystemClock; import android.util.Log; -import android.view.OrientationEventListener; +import android.view.Surface; import android.view.SurfaceHolder.Callback; import android.view.SurfaceHolder; +import android.view.WindowManager; // Wrapper for android Camera, with support for direct local preview rendering. // Threading notes: this class is called from ViE C++ code, and from Camera & @@ -44,10 +45,9 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { private Camera camera; // Only non-null while capturing. private CameraThread cameraThread; private Handler cameraThreadHandler; + private Context context; private final int id; private final Camera.CameraInfo info; - private final OrientationEventListener orientationListener; - private boolean orientationListenerEnabled; private final long native_capturer; // |VideoCaptureAndroid*| in C++. private SurfaceTexture cameraSurfaceTexture; private int[] cameraGlTextures = null; @@ -70,34 +70,13 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { public VideoCaptureAndroid(int id, long native_capturer) { this.id = id; this.native_capturer = native_capturer; + this.context = GetContext(); this.info = new Camera.CameraInfo(); Camera.getCameraInfo(id, info); - - // Must be the last thing in the ctor since we pass a reference to |this|! - final VideoCaptureAndroid self = this; - orientationListener = new OrientationEventListener(GetContext()) { - @Override public void onOrientationChanged(int degrees) { - if (!self.orientationListenerEnabled) { - return; - } - if (degrees == OrientationEventListener.ORIENTATION_UNKNOWN) { - return; - } - if (info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) { - degrees = (info.orientation - degrees + 360) % 360; - } else { // back-facing - degrees = (info.orientation + degrees) % 360; - } - self.OnOrientationChanged(self.native_capturer, degrees); - } - }; - // Don't add any code here; see the comment above |self| above! } // Return the global application context. private static native Context GetContext(); - // Request frame rotation post-capture. - private native void OnOrientationChanged(long captureObject, int degrees); private class CameraThread extends Thread { private Exchanger handlerExchanger; @@ -137,8 +116,6 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { } }); boolean startResult = exchange(result, false); // |false| is a dummy value. - orientationListenerEnabled = true; - orientationListener.enable(); return startResult; } @@ -184,6 +161,8 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { } } + Log.d(TAG, "Camera orientation: " + info.orientation + + " .Device orientation: " + getDeviceOrientation()); Camera.Parameters parameters = camera.getParameters(); Log.d(TAG, "isVideoStabilizationSupported: " + parameters.isVideoStabilizationSupported()); @@ -223,8 +202,6 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { // Called by native code. Returns true when camera is known to be stopped. private synchronized boolean stopCapture() { Log.d(TAG, "stopCapture"); - orientationListener.disable(); - orientationListenerEnabled = false; final Exchanger result = new Exchanger(); cameraThreadHandler.post(new Runnable() { @Override public void run() { @@ -279,8 +256,32 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { return; } + private int getDeviceOrientation() { + int orientation = 0; + if (context != null) { + WindowManager wm = (WindowManager) context.getSystemService( + Context.WINDOW_SERVICE); + switch(wm.getDefaultDisplay().getRotation()) { + case Surface.ROTATION_90: + orientation = 90; + break; + case Surface.ROTATION_180: + orientation = 180; + break; + case Surface.ROTATION_270: + orientation = 270; + break; + case Surface.ROTATION_0: + default: + orientation = 0; + break; + } + } + return orientation; + } + private native void ProvideCameraFrame( - byte[] data, int length, long timeStamp, long captureObject); + byte[] data, int length, int rotation, long timeStamp, long captureObject); // Called on cameraThread so must not "synchronized". @Override @@ -306,7 +307,15 @@ public class VideoCaptureAndroid implements PreviewCallback, Callback { } } lastCaptureTimeMs = captureTimeMs; - ProvideCameraFrame(data, data.length, captureTimeMs, native_capturer); + + int rotation = getDeviceOrientation(); + if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { + rotation = 360 - rotation; + } + rotation = (info.orientation + rotation) % 360; + + ProvideCameraFrame(data, data.length, rotation, + captureTimeMs, native_capturer); camera.addCallbackBuffer(data); } diff --git a/webrtc/modules/video_capture/android/video_capture_android.cc b/webrtc/modules/video_capture/android/video_capture_android.cc index 4bc14e5ae1..14dda7f9c6 100644 --- a/webrtc/modules/video_capture/android/video_capture_android.cc +++ b/webrtc/modules/video_capture/android/video_capture_android.cc @@ -37,6 +37,7 @@ void JNICALL ProvideCameraFrame( jobject, jbyteArray javaCameraFrame, jint length, + jint rotation, jlong timeStamp, jlong context) { webrtc::videocapturemodule::VideoCaptureAndroid* captureModule = @@ -44,30 +45,10 @@ void JNICALL ProvideCameraFrame( context); jbyte* cameraFrame = env->GetByteArrayElements(javaCameraFrame, NULL); captureModule->OnIncomingFrame( - reinterpret_cast(cameraFrame), length, 0); + reinterpret_cast(cameraFrame), length, rotation, 0); env->ReleaseByteArrayElements(javaCameraFrame, cameraFrame, JNI_ABORT); } -// Called by Java when the device orientation has changed. -void JNICALL OnOrientationChanged( - JNIEnv* env, jobject, jlong context, jint degrees) { - webrtc::videocapturemodule::VideoCaptureAndroid* captureModule = - reinterpret_cast( - context); - degrees = (360 + degrees) % 360; - assert(degrees >= 0 && degrees < 360); - VideoCaptureRotation rotation = - (degrees <= 45 || degrees > 315) ? kCameraRotate0 : - (degrees > 45 && degrees <= 135) ? kCameraRotate90 : - (degrees > 135 && degrees <= 225) ? kCameraRotate180 : - (degrees > 225 && degrees <= 315) ? kCameraRotate270 : - kCameraRotate0; // Impossible. - int32_t status = - captureModule->VideoCaptureImpl::SetCaptureRotation(rotation); - RTC_UNUSED(status); - assert(status == 0); -} - int32_t SetCaptureAndroidVM(JavaVM* javaVM, jobject context) { if (javaVM) { assert(!g_jvm); @@ -88,14 +69,11 @@ int32_t SetCaptureAndroidVM(JavaVM* javaVM, jobject context) { {"GetContext", "()Landroid/content/Context;", reinterpret_cast(&GetContext)}, - {"OnOrientationChanged", - "(JI)V", - reinterpret_cast(&OnOrientationChanged)}, {"ProvideCameraFrame", - "([BIJJ)V", + "([BIIJJ)V", reinterpret_cast(&ProvideCameraFrame)}}; if (ats.env()->RegisterNatives(g_java_capturer_class, - native_methods, 3) != 0) + native_methods, 2) != 0) assert(false); } else { if (g_jvm) { @@ -129,9 +107,23 @@ VideoCaptureModule* VideoCaptureImpl::Create( int32_t VideoCaptureAndroid::OnIncomingFrame(uint8_t* videoFrame, int32_t videoFrameLength, + int32_t degrees, int64_t captureTime) { if (!_captureStarted) return 0; + VideoCaptureRotation current_rotation = + (degrees <= 45 || degrees > 315) ? kCameraRotate0 : + (degrees > 45 && degrees <= 135) ? kCameraRotate90 : + (degrees > 135 && degrees <= 225) ? kCameraRotate180 : + (degrees > 225 && degrees <= 315) ? kCameraRotate270 : + kCameraRotate0; // Impossible. + if (_rotation != current_rotation) { + LOG(LS_INFO) << "New camera rotation: " << degrees; + _rotation = current_rotation; + int32_t status = VideoCaptureImpl::SetCaptureRotation(_rotation); + if (status != 0) + return status; + } return IncomingFrame( videoFrame, videoFrameLength, _captureCapability, captureTime); } @@ -165,6 +157,7 @@ int32_t VideoCaptureAndroid::Init(const int32_t id, _jCapturer = env->NewGlobalRef( env->NewObject(g_java_capturer_class, ctor, camera_id, j_this)); assert(_jCapturer); + _rotation = kCameraRotate0; return 0; } diff --git a/webrtc/modules/video_capture/android/video_capture_android.h b/webrtc/modules/video_capture/android/video_capture_android.h index 3ab7189b42..f45b726f18 100644 --- a/webrtc/modules/video_capture/android/video_capture_android.h +++ b/webrtc/modules/video_capture/android/video_capture_android.h @@ -32,6 +32,7 @@ class VideoCaptureAndroid : public VideoCaptureImpl { int32_t OnIncomingFrame(uint8_t* videoFrame, int32_t videoFrameLength, + int32_t degrees, int64_t captureTime = 0); protected: @@ -40,6 +41,7 @@ class VideoCaptureAndroid : public VideoCaptureImpl { DeviceInfoAndroid _deviceInfo; jobject _jCapturer; // Global ref to Java VideoCaptureAndroid object. VideoCaptureCapability _captureCapability; + VideoCaptureRotation _rotation; bool _captureStarted; }; diff --git a/webrtc/modules/video_capture/test/video_capture_unittest.cc b/webrtc/modules/video_capture/test/video_capture_unittest.cc index 4c2263d04f..59d58e7647 100644 --- a/webrtc/modules/video_capture/test/video_capture_unittest.cc +++ b/webrtc/modules/video_capture/test/video_capture_unittest.cc @@ -108,9 +108,14 @@ class TestVideoCaptureCallback : public VideoCaptureDataCallback { virtual void OnIncomingCapturedFrame(const int32_t id, webrtc::I420VideoFrame& videoFrame) { CriticalSectionScoped cs(capture_cs_.get()); - int height = videoFrame.height(); int width = videoFrame.width(); +#if ANDROID + // Android camera frames may be rotated depending on test device + // orientation. + EXPECT_TRUE(height == capability_.height || height == capability_.width); + EXPECT_TRUE(width == capability_.width || width == capability_.height); +#else if (rotate_frame_ == webrtc::kCameraRotate90 || rotate_frame_ == webrtc::kCameraRotate270) { EXPECT_EQ(width, capability_.height); @@ -119,6 +124,7 @@ class TestVideoCaptureCallback : public VideoCaptureDataCallback { EXPECT_EQ(height, capability_.height); EXPECT_EQ(width, capability_.width); } +#endif // RenderTimstamp should be the time now. EXPECT_TRUE( videoFrame.render_time_ms() >= TickTime::MillisecondTimestamp()-30 &&