From 3354419a2d53c3bfee8077cfbb0d86022b03e94a Mon Sep 17 00:00:00 2001 From: Per Date: Thu, 2 Apr 2015 12:30:51 +0200 Subject: [PATCH] Zero copy AndroidVideeCapturer. This cl uses the YV12 buffers from Java without a copy if no rotation is needed. Buffers are returned to the camera when the encoder and renderers no longer needs them. This add a new frame type WrappedI420Buffer based in that allows for wrapping existing memory buffers and getting a notification when it is no longer used. AndroidVideoCapturer::FrameFactory::CreateAliasedFrame wraps frame received from Java. For each wrapped frame a new reference to AndroidVideoCapturerDelegate is held to ensure that the delegate can not be destroyed until all frames have been returned. Some overlap exist in webrtcvideoframe.cc and webrtcvideengine.cc with https://webrtc-codereview.appspot.com/47399004/ that is expected to be landed before this cl. BUG=1128 R=glaznev@webrtc.org, magjed@webrtc.org TBR=mflodman@webrtc.org // For changes in webrtc/common_video/video_frame_buffer Review URL: https://webrtc-codereview.appspot.com/49459004 Cr-Commit-Position: refs/heads/master@{#8923} --- .../org/webrtc/VideoCapturerAndroidTest.java | 55 +++++++- talk/app/webrtc/androidvideocapturer.cc | 71 ++++++++--- talk/app/webrtc/androidvideocapturer.h | 15 +-- .../java/jni/androidvideocapturer_jni.cc | 19 +-- .../java/jni/androidvideocapturer_jni.h | 8 +- .../app/webrtc/java/jni/peerconnection_jni.cc | 4 +- .../src/org/webrtc/VideoCapturerAndroid.java | 120 +++++++++++++----- webrtc/base/asyncinvoker.cc | 2 + webrtc/base/callback.h | 1 - webrtc/base/refcount.h | 41 ++++++ .../interface/video_frame_buffer.h | 38 ++++++ webrtc/common_video/video_frame_buffer.cc | 82 ++++++++++++ 12 files changed, 386 insertions(+), 70 deletions(-) diff --git a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java index 24294ab8f7..169a678a58 100644 --- a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java +++ b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java @@ -34,6 +34,7 @@ import org.webrtc.VideoCapturerAndroid.CaptureFormat; import org.webrtc.VideoRenderer.I420Frame; import java.util.ArrayList; +import java.util.List; @SuppressWarnings("deprecation") public class VideoCapturerAndroidTest extends ActivityTestCase { @@ -41,9 +42,6 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { private int framesRendered = 0; private Object frameLock = 0; - private void setSize(int width, int height) { - } - @Override public void renderFrame(I420Frame frame) { synchronized (frameLock) { @@ -73,6 +71,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { private Object frameLock = 0; private Object capturerStartLock = 0; private boolean captureStartResult = false; + private List timestamps = new ArrayList(); @Override public void OnCapturerStarted(boolean success) { @@ -88,6 +87,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { synchronized (frameLock) { ++framesCaptured; frameSize = length; + timestamps.add(timeStamp); frameLock.notify(); } } @@ -111,6 +111,14 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { return frameSize; } } + + List getCopyAndResetListOftimeStamps() { + synchronized (frameLock) { + ArrayList list = new ArrayList(timestamps); + timestamps.clear(); + return list; + } + } } // Return true if the device under test have at least two cameras. @@ -257,4 +265,45 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { } capturer.dispose(); } + + @SmallTest + // This test what happens if buffers are returned after the capturer have + // been stopped and restarted. It does not test or use the C++ layer. + public void testReturnBufferLate() throws Exception { + FakeCapturerObserver observer = new FakeCapturerObserver(); + + String deviceName = VideoCapturerAndroid.getDeviceName(0); + ArrayList formats = + VideoCapturerAndroid.getSupportedFormats(0); + VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName); + + VideoCapturerAndroid.CaptureFormat format = formats.get(0); + capturer.startCapture(format.width, format.height, format.maxFramerate, + getInstrumentation().getContext(), observer); + assertTrue(observer.WaitForCapturerToStart()); + + observer.WaitForNextCapturedFrame(); + capturer.stopCapture(); + List listOftimestamps = observer.getCopyAndResetListOftimeStamps(); + assertTrue(listOftimestamps.size() >= 1); + + format = formats.get(1); + capturer.startCapture(format.width, format.height, format.maxFramerate, + getInstrumentation().getContext(), observer); + observer.WaitForCapturerToStart(); + observer.WaitForNextCapturedFrame(); + + for (Long timeStamp : listOftimestamps) { + capturer.returnBuffer(timeStamp); + } + + observer.WaitForNextCapturedFrame(); + capturer.stopCapture(); + + listOftimestamps = observer.getCopyAndResetListOftimeStamps(); + assertTrue(listOftimestamps.size() >= 2); + for (Long timeStamp : listOftimestamps) { + capturer.returnBuffer(timeStamp); + } + } } diff --git a/talk/app/webrtc/androidvideocapturer.cc b/talk/app/webrtc/androidvideocapturer.cc index 01619aff57..6711b6ed0a 100644 --- a/talk/app/webrtc/androidvideocapturer.cc +++ b/talk/app/webrtc/androidvideocapturer.cc @@ -27,6 +27,8 @@ #include "talk/app/webrtc/androidvideocapturer.h" #include "talk/media/webrtc/webrtcvideoframe.h" +#include "webrtc/base/bind.h" +#include "webrtc/base/callback.h" #include "webrtc/base/common.h" #include "webrtc/base/json.h" #include "webrtc/base/timeutils.h" @@ -36,6 +38,7 @@ namespace webrtc { using cricket::WebRtcVideoFrame; using rtc::scoped_ptr; +using rtc::scoped_refptr; // An implementation of cricket::VideoFrameFactory for frames that are not // guaranteed to outlive the created cricket::VideoFrame. @@ -45,7 +48,10 @@ using rtc::scoped_ptr; // CreateAliasedFrame for every frame. class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory { public: - FrameFactory(int width, int height) : start_time_(rtc::TimeNanos()) { + FrameFactory(int width, + int height, + const scoped_refptr& delegate) + : start_time_(rtc::TimeNanos()), delegate_(delegate) { // Create a CapturedFrame that only contains header information, not the // actual pixel data. captured_frame_.width = width; @@ -61,12 +67,11 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory { void UpdateCapturedFrame(void* frame_data, int length, int rotation, - int64 time_stamp_in_ms) { + int64 time_stamp_in_ns) { captured_frame_.fourcc = static_cast(cricket::FOURCC_YV12); captured_frame_.data = frame_data; captured_frame_.elapsed_time = rtc::TimeNanos() - start_time_; - captured_frame_.time_stamp = - time_stamp_in_ms * rtc::kNumNanosecsPerMillisec; + captured_frame_.time_stamp = time_stamp_in_ns; captured_frame_.rotation = rotation; captured_frame_.data_size = length; } @@ -84,23 +89,60 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory { // of |AndroidVideoCapturer::OnIncomingFrame_w|. // Check that captured_frame is actually our frame. DCHECK(captured_frame == &captured_frame_); + + if (!apply_rotation_ || captured_frame->rotation == kVideoRotation_0) { + DCHECK(captured_frame->fourcc == cricket::FOURCC_YV12); + const uint8_t* y_plane = static_cast(captured_frame_.data); + const int y_stride = captured_frame->width; + const uint8_t* v_plane = y_plane + + captured_frame->width * captured_frame->height; + const int uv_stride = (captured_frame->width + 1) / 2; + const int uv_height = (captured_frame->height + 1) / 2; + const uint8_t* u_plane = v_plane + uv_stride * uv_height; + + // Create a WrappedI420Buffer and bind the |no_longer_used| callback + // to the static method ReturnFrame. The |delegate_| is bound as an + // argument which means that the callback will hold a reference to + // |delegate_|. + rtc::scoped_refptr buffer( + new rtc::RefCountedObject( + dst_width, dst_height, captured_frame->width, + captured_frame->height, y_plane, y_stride, u_plane, uv_stride, + v_plane, uv_stride, + rtc::Bind(&AndroidVideoCapturer::FrameFactory::ReturnFrame, + delegate_, + captured_frame->time_stamp))); + return new WebRtcVideoFrame( + buffer, captured_frame->elapsed_time, + captured_frame->time_stamp, captured_frame->GetRotation()); + } + scoped_ptr frame(new WebRtcVideoFrame()); frame->Init(captured_frame, dst_width, dst_height, apply_rotation_); + // frame->Init copies the data in |captured_frame| so it is safe to return + // the buffer immediately. + delegate_->ReturnBuffer(captured_frame->time_stamp); return frame.release(); } + static void ReturnFrame(scoped_refptr delegate, + int64 time_stamp) { + delegate->ReturnBuffer(time_stamp); + } + private: uint64 start_time_; cricket::CapturedFrame captured_frame_; + scoped_refptr delegate_; }; AndroidVideoCapturer::AndroidVideoCapturer( - rtc::scoped_ptr delegate) + const rtc::scoped_refptr& delegate) : running_(false), - delegate_(delegate.Pass()), - worker_thread_(NULL), + delegate_(delegate), frame_factory_(NULL), current_state_(cricket::CS_STOPPED) { + thread_checker_.DetachFromThread(); std::string json_string = delegate_->GetSupportedFormats(); LOG(LS_INFO) << json_string; @@ -131,15 +173,13 @@ AndroidVideoCapturer::~AndroidVideoCapturer() { cricket::CaptureState AndroidVideoCapturer::Start( const cricket::VideoFormat& capture_format) { + DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(!running_); - DCHECK(worker_thread_ == nullptr || worker_thread_ == rtc::Thread::Current()); - // TODO(perkj): Better way to get a handle to the worker thread? - worker_thread_ = rtc::Thread::Current(); LOG(LS_INFO) << " AndroidVideoCapturer::Start w = " << capture_format.width << " h = " << capture_format.height; frame_factory_ = new AndroidVideoCapturer::FrameFactory( - capture_format.width, capture_format.height); + capture_format.width, capture_format.height, delegate_.get()); set_frame_factory(frame_factory_); running_ = true; @@ -152,7 +192,7 @@ cricket::CaptureState AndroidVideoCapturer::Start( } void AndroidVideoCapturer::Stop() { - DCHECK(worker_thread_->IsCurrent()); + DCHECK(thread_checker_.CalledOnValidThread()); LOG(LS_INFO) << " AndroidVideoCapturer::Stop "; DCHECK(running_); running_ = false; @@ -164,16 +204,18 @@ void AndroidVideoCapturer::Stop() { } bool AndroidVideoCapturer::IsRunning() { + DCHECK(thread_checker_.CalledOnValidThread()); return running_; } bool AndroidVideoCapturer::GetPreferredFourccs(std::vector* fourccs) { + DCHECK(thread_checker_.CalledOnValidThread()); fourccs->push_back(cricket::FOURCC_YV12); return true; } void AndroidVideoCapturer::OnCapturerStarted(bool success) { - DCHECK(worker_thread_->IsCurrent()); + DCHECK(thread_checker_.CalledOnValidThread()); cricket::CaptureState new_state = success ? cricket::CS_RUNNING : cricket::CS_FAILED; if (new_state == current_state_) @@ -190,10 +232,9 @@ void AndroidVideoCapturer::OnIncomingFrame(void* frame_data, int length, int rotation, int64 time_stamp) { - DCHECK(worker_thread_->IsCurrent()); + DCHECK(thread_checker_.CalledOnValidThread()); frame_factory_->UpdateCapturedFrame(frame_data, length, rotation, time_stamp); SignalFrameCaptured(this, frame_factory_->GetCapturedFrame()); - delegate_->ReturnBuffer(time_stamp); } } // namespace webrtc diff --git a/talk/app/webrtc/androidvideocapturer.h b/talk/app/webrtc/androidvideocapturer.h index 380313bf79..2dffb9c141 100644 --- a/talk/app/webrtc/androidvideocapturer.h +++ b/talk/app/webrtc/androidvideocapturer.h @@ -31,12 +31,13 @@ #include #include "talk/media/base/videocapturer.h" +#include "webrtc/base/thread_checker.h" namespace webrtc { class AndroidVideoCapturer; -class AndroidVideoCapturerDelegate { +class AndroidVideoCapturerDelegate : public rtc::RefCountInterface { public: virtual ~AndroidVideoCapturerDelegate() {} // Start capturing. The implementation of the delegate must call @@ -49,7 +50,7 @@ class AndroidVideoCapturerDelegate { virtual void Stop() = 0; // Notify that a frame received in OnIncomingFrame with |time_stamp| has been - // processed and can be returned. + // processed and can be returned. May be called on an arbitrary thread. virtual void ReturnBuffer(int64 time_stamp) = 0; // Must returns a JSON string "{{width=xxx, height=xxx, framerate = xxx}}" @@ -61,7 +62,7 @@ class AndroidVideoCapturerDelegate { class AndroidVideoCapturer : public cricket::VideoCapturer { public: explicit AndroidVideoCapturer( - rtc::scoped_ptr delegate); + const rtc::scoped_refptr& delegate); virtual ~AndroidVideoCapturer(); // Called from JNI when the capturer has been started. @@ -87,13 +88,9 @@ class AndroidVideoCapturer : public cricket::VideoCapturer { bool GetPreferredFourccs(std::vector* fourccs) override; bool running_; - rtc::scoped_ptr delegate_; + rtc::scoped_refptr delegate_; - // |worker_thread_| is the thread that calls Start and is used for - // communication with the Java capturer. - // Video frames are delivered to cricket::VideoCapturer::SignalFrameCaptured - // on this thread. - rtc::Thread* worker_thread_; + rtc::ThreadChecker thread_checker_; class FrameFactory; FrameFactory* frame_factory_; // Owned by cricket::VideoCapturer. diff --git a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc index dfe1b1b7f5..4fa23f3516 100644 --- a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc +++ b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc @@ -47,16 +47,15 @@ int AndroidVideoCapturerJni::SetAndroidObjects(JNIEnv* jni, } // static -rtc::scoped_ptr +rtc::scoped_refptr AndroidVideoCapturerJni::Create(JNIEnv* jni, jobject j_video_capture, jstring device_name) { - rtc::scoped_ptr capturer( - new AndroidVideoCapturerJni(jni, - j_video_capture)); + rtc::scoped_refptr capturer( + new rtc::RefCountedObject(jni, j_video_capture)); if (capturer->Init(device_name)) - return capturer.Pass(); + return capturer; return nullptr; } @@ -123,15 +122,19 @@ void AndroidVideoCapturerJni::Stop() { jni()->CallVoidMethod(*j_capturer_global_, m); CHECK_EXCEPTION(jni()) << "error during VideoCapturerAndroid.stopCapture"; DeleteGlobalRef(jni(), j_frame_observer_); - // Do not process frames in flight after stop have returned since - // the memory buffers they point to have been deleted. - rtc::MessageQueueManager::Clear(&invoker_); } void AndroidVideoCapturerJni::ReturnBuffer(int64 time_stamp) { + invoker_.AsyncInvoke( + thread_, + rtc::Bind(&AndroidVideoCapturerJni::ReturnBuffer_w, this, time_stamp)); +} + +void AndroidVideoCapturerJni::ReturnBuffer_w(int64 time_stamp) { jmethodID m = GetMethodID(jni(), *j_video_capturer_class_, "returnBuffer", "(J)V"); jni()->CallVoidMethod(*j_capturer_global_, m, time_stamp); + CHECK_EXCEPTION(jni()) << "error during VideoCapturerAndroid.returnBuffer"; } std::string AndroidVideoCapturerJni::GetSupportedFormats() { diff --git a/talk/app/webrtc/java/jni/androidvideocapturer_jni.h b/talk/app/webrtc/java/jni/androidvideocapturer_jni.h index e6717dbb1c..aea4377f4c 100644 --- a/talk/app/webrtc/java/jni/androidvideocapturer_jni.h +++ b/talk/app/webrtc/java/jni/androidvideocapturer_jni.h @@ -44,11 +44,10 @@ namespace webrtc_jni { class AndroidVideoCapturerJni : public webrtc::AndroidVideoCapturerDelegate { public: static int SetAndroidObjects(JNIEnv* jni, jobject appliction_context); - ~AndroidVideoCapturerJni(); // Creates a new instance of AndroidVideoCapturerJni. Returns a nullptr if // it can't be created. This happens if |device_name| is invalid. - static rtc::scoped_ptr Create( + static rtc::scoped_refptr Create( JNIEnv* jni, jobject j_video_capture, // Instance of VideoCapturerAndroid jstring device_name); // Name of the camera to use. @@ -67,9 +66,11 @@ class AndroidVideoCapturerJni : public webrtc::AndroidVideoCapturerDelegate { int length, int rotation, int64 time_stamp); +protected: + AndroidVideoCapturerJni(JNIEnv* jni, jobject j_video_capturer); + ~AndroidVideoCapturerJni(); private: - AndroidVideoCapturerJni(JNIEnv* jni, jobject j_video_capturer); bool Init(jstring device_name); void OnCapturerStarted_w(bool success); @@ -78,6 +79,7 @@ private: int length, int rotation, int64 time_stamp); + void ReturnBuffer_w(int64 time_stamp); JNIEnv* jni(); diff --git a/talk/app/webrtc/java/jni/peerconnection_jni.cc b/talk/app/webrtc/java/jni/peerconnection_jni.cc index 7e6072c7aa..d7fde20ace 100644 --- a/talk/app/webrtc/java/jni/peerconnection_jni.cc +++ b/talk/app/webrtc/java/jni/peerconnection_jni.cc @@ -1410,12 +1410,12 @@ JOW(jobject, VideoCapturer_nativeCreateVideoCapturer)( j_videocapturer_ctor); CHECK_EXCEPTION(jni) << "error during NewObject"; - rtc::scoped_ptr delegate = + rtc::scoped_refptr delegate = AndroidVideoCapturerJni::Create(jni, j_video_capturer, j_device_name); if (!delegate.get()) return nullptr; rtc::scoped_ptr capturer( - new webrtc::AndroidVideoCapturer(delegate.Pass())); + new webrtc::AndroidVideoCapturer(delegate)); #else std::string device_name = JavaToStdString(jni, j_device_name); diff --git a/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java b/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java index f8b03f91c1..e650770635 100644 --- a/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java +++ b/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java @@ -52,6 +52,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Exchanger; +import java.util.concurrent.TimeUnit; // Android specific implementation of VideoCapturer. // An instance of this class can be created by an application using @@ -80,7 +81,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private Camera.CameraInfo info; private SurfaceTexture cameraSurfaceTexture; private int[] cameraGlTextures = null; - private FramePool videoBuffers = null; + private final FramePool videoBuffers = new FramePool(); private int width; private int height; private int framerate; @@ -360,10 +361,6 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba cameraThread = new CameraThread(handlerExchanger); cameraThread.start(); cameraThreadHandler = exchange(handlerExchanger, null); - // We must guarantee that buffers sent to an observer are kept alive until - // stopCapture have completed. Therefore, create the buffers here and - // abandon them after the camera thread have been stopped. - videoBuffers = new FramePool(width, height, ImageFormat.YV12); cameraThreadHandler.post(new Runnable() { @Override public void run() { startCaptureOnCameraThread(width, height, framerate, frameObserver, @@ -440,7 +437,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Note: setRecordingHint(true) actually decrease frame rate on N5. // parameters.setRecordingHint(true); - videoBuffers.addBuffersAsCameraCallbackBuffers(camera); + videoBuffers.queueCameraBuffers(width, height, format, camera); camera.setPreviewCallbackWithBuffer(this); camera.startPreview(); frameObserver.OnCapturerStarted(true); @@ -467,7 +464,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba }); cameraThread.join(); cameraThreadHandler = null; - videoBuffers = null; + Log.d(TAG, "stopCapture done"); } private void stopCaptureOnCameraThread() { @@ -481,6 +478,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba try { camera.stopPreview(); camera.setPreviewCallbackWithBuffer(null); + videoBuffers.stopReturnBuffersToCamera(); camera.setPreviewTexture(null); cameraSurfaceTexture = null; @@ -505,11 +503,14 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } synchronized void returnBuffer(final long timeStamp) { + if (cameraThreadHandler == null) { + // The camera has been stopped. + videoBuffers.returnBuffer(timeStamp); + return; + } cameraThreadHandler.post(new Runnable() { @Override public void run() { - if (camera == null) - return; - videoBuffers.addBufferAsCameraCallbackBuffer(camera, timeStamp); + videoBuffers.returnBuffer(timeStamp); } }); } @@ -583,7 +584,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba throw new RuntimeException("Unexpected camera in callback!"); } - long captureTimeMs = SystemClock.elapsedRealtime(); + long captureTimeNs = + TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); int rotation = getDeviceOrientation(); if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { @@ -593,9 +595,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Mark the frame owning |data| as used. // Note that since data is directBuffer, // data.length >= videoBuffers.frameSize. - videoBuffers.reserveByteBuffer(data, captureTimeMs); + videoBuffers.reserveByteBuffer(data, captureTimeNs); frameObserver.OnFrameCaptured(data, videoBuffers.frameSize, rotation, - captureTimeMs); + captureTimeNs); } // runCameraThreadUntilIdle make sure all posted messages to the cameraThread @@ -632,14 +634,17 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // lower number means more sensitivity to processing time in the client (and // potentially stalling the capturer if it runs out of buffers to write to). private static int numCaptureBuffers = 3; - private final Frame cameraFrames[]; - public final int frameSize; + private final List cameraFrames = new ArrayList(); + public int frameSize = 0; + private Camera camera; private static class Frame { private final ByteBuffer buffer; public long timeStamp = -1; + public final int frameSize; Frame(int frameSize) { + this.frameSize = frameSize; buffer = ByteBuffer.allocateDirect(frameSize); } @@ -648,25 +653,60 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } } - FramePool(int width, int height, int format) { - cameraFrames = new Frame[numCaptureBuffers]; - frameSize = width * height * ImageFormat.getBitsPerPixel(format) / 8; - for (int i = 0; i < numCaptureBuffers; i++) { - cameraFrames[i] = new Frame(frameSize); + // Adds frames as callback buffers to |camera|. If a new frame size is + // required, new buffers are allocated and added. + void queueCameraBuffers(int width, int height, int format, Camera camera) { + if (this.camera != null) + throw new RuntimeException("camera already set."); + + this.camera = camera; + int newframeSize = + width * height * ImageFormat.getBitsPerPixel(format) / 8; + int numberOfEnquedCameraBuffers = 0; + if (newframeSize != frameSize) { + // Create new frames and add to the camera. + // The old frames will be released when frames are returned. + for (int i = 0; i < numCaptureBuffers; ++i) { + Frame frame = new Frame(newframeSize); + cameraFrames.add(frame); + this.camera.addCallbackBuffer(frame.data()); + } + numberOfEnquedCameraBuffers = numCaptureBuffers; + } else { + // Add all frames that have been returned. + for (Frame frame : cameraFrames) { + if (frame.timeStamp < 0) { + camera.addCallbackBuffer(frame.data()); + ++numberOfEnquedCameraBuffers; + } + } } + frameSize = newframeSize; + Log.d(TAG, "queueCameraBuffers enqued " + numberOfEnquedCameraBuffers + + " buffers of size " + frameSize + "."); } - // Add all free buffers to |camera| that are currently not sent to a client. - void addBuffersAsCameraCallbackBuffers(Camera camera) { + void stopReturnBuffersToCamera() { + this.camera = null; + String pendingTimeStamps = new String(); for (Frame frame : cameraFrames) { - if (frame.timeStamp < 0) - camera.addCallbackBuffer(frame.data()); + if (frame.timeStamp > -1) { + pendingTimeStamps+= " " + frame.timeStamp; + } } + Log.d(TAG, "stopReturnBuffersToCamera called." + + (pendingTimeStamps.isEmpty() ? + " All buffers have been returned." + : " Pending buffers " + pendingTimeStamps + ".")); + } void reserveByteBuffer(byte[] data, long timeStamp) { for (Frame frame : cameraFrames) { if (data == frame.data()) { + if (frame.timeStamp > 0) { + throw new RuntimeException("Frame already in use !"); + } frame.timeStamp = timeStamp; return; } @@ -674,16 +714,38 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba throw new RuntimeException("unknown data buffer?!?"); } - // Add the buffer with |timeStamp| to |camera|. - void addBufferAsCameraCallbackBuffer(Camera camera, long timeStamp) { + void returnBuffer(long timeStamp) { + Frame returnedFrame = null; for (Frame frame : cameraFrames) { if (timeStamp == frame.timeStamp) { frame.timeStamp = -1; - camera.addCallbackBuffer(frame.data()); - return; + returnedFrame = frame; + break; } } - throw new RuntimeException("unknown data buffer returned?!?"); + + if (returnedFrame == null) { + throw new RuntimeException("unknown data buffer with time stamp " + + timeStamp + "returned?!?"); + } + + if (camera != null && returnedFrame.frameSize == frameSize) { + camera.addCallbackBuffer(returnedFrame.data()); + return; + } + + if (returnedFrame.frameSize != frameSize) { + Log.d(TAG, "returnBuffer with time stamp "+ timeStamp + + " called with old frame size, " + returnedFrame.frameSize + "."); + // Since this frame has the wrong size, remove it from the list. Frames + // with the correct size is created in queueCameraBuffers so this must + // be an old buffer. + cameraFrames.remove(returnedFrame); + return; + } + + Log.d(TAG, "returnBuffer with time stamp "+ timeStamp + + " called after camera has been stopped."); } } diff --git a/webrtc/base/asyncinvoker.cc b/webrtc/base/asyncinvoker.cc index 048873412e..35ce2fb128 100644 --- a/webrtc/base/asyncinvoker.cc +++ b/webrtc/base/asyncinvoker.cc @@ -10,6 +10,8 @@ #include "webrtc/base/asyncinvoker.h" +#include "webrtc/base/logging.h" + namespace rtc { AsyncInvoker::AsyncInvoker() : destroying_(false) {} diff --git a/webrtc/base/callback.h b/webrtc/base/callback.h index 949510e950..7ffdcd7373 100644 --- a/webrtc/base/callback.h +++ b/webrtc/base/callback.h @@ -62,7 +62,6 @@ #ifndef WEBRTC_BASE_CALLBACK_H_ #define WEBRTC_BASE_CALLBACK_H_ -#include "webrtc/base/logging.h" #include "webrtc/base/refcount.h" #include "webrtc/base/scoped_ref_ptr.h" diff --git a/webrtc/base/refcount.h b/webrtc/base/refcount.h index eb097334f1..e7306d3df9 100644 --- a/webrtc/base/refcount.h +++ b/webrtc/base/refcount.h @@ -54,6 +54,47 @@ class RefCountedObject : public T { : T(p1, p2, p3, p4, p5), ref_count_(0) { } + template + RefCountedObject(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6) + : T(p1, p2, p3, p4, p5, p6), ref_count_(0) { + } + + template + RefCountedObject(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6, P7 p7) + : T(p1, p2, p3, p4, p5, p6, p7), ref_count_(0) { + } + + template + RefCountedObject(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6, P7 p7, P8 p8) + : T(p1, p2, p3, p4, p5, p6, p7, p8), ref_count_(0) { + } + + template + RefCountedObject( + P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6, P7 p7, P8 p8, P9 p9) + : T(p1, p2, p3, p4, p5, p6, p7, p8, p9), ref_count_(0) { + } + + template + RefCountedObject( + P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6, P7 p7, P8 p8, P9 p9, P10 p10) + : T(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10), ref_count_(0) { + } + + template + RefCountedObject( + P1 p1, P2 p2, P3 p3, P4 p4, P5 p5, P6 p6, P7 p7, P8 p8, P9 p9, P10 p10, + P11 p11) + : T(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11), ref_count_(0) { + } + virtual int AddRef() { return rtc::AtomicOps::Increment(&ref_count_); } diff --git a/webrtc/common_video/interface/video_frame_buffer.h b/webrtc/common_video/interface/video_frame_buffer.h index e340d8b351..e7cae48a9f 100644 --- a/webrtc/common_video/interface/video_frame_buffer.h +++ b/webrtc/common_video/interface/video_frame_buffer.h @@ -11,6 +11,7 @@ #ifndef WEBRTC_VIDEO_FRAME_BUFFER_H_ #define WEBRTC_VIDEO_FRAME_BUFFER_H_ +#include "webrtc/base/callback.h" #include "webrtc/base/refcount.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/base/scoped_ref_ptr.h" @@ -104,6 +105,43 @@ class TextureBuffer : public VideoFrameBuffer { const int height_; }; +class WrappedI420Buffer : public webrtc::VideoFrameBuffer { + public: + WrappedI420Buffer(int desired_width, + int desired_height, + int width, + int height, + const uint8_t* y_plane, + int y_stride, + const uint8_t* u_plane, + int u_stride, + const uint8_t* v_plane, + int v_stride, + const rtc::Callback0& no_longer_used); + int width() const override; + int height() const override; + + const uint8_t* data(PlaneType type) const override; + uint8_t* data(PlaneType type) override; + + int stride(PlaneType type) const override; + rtc::scoped_refptr native_handle() const override; + + private: + friend class rtc::RefCountedObject; + ~WrappedI420Buffer() override; + + int width_; + int height_; + const uint8_t* y_plane_; + const uint8_t* u_plane_; + const uint8_t* v_plane_; + const int y_stride_; + const int u_stride_; + const int v_stride_; + rtc::Callback0 no_longer_used_cb_; +}; + } // namespace webrtc #endif // WEBRTC_VIDEO_FRAME_BUFFER_H_ diff --git a/webrtc/common_video/video_frame_buffer.cc b/webrtc/common_video/video_frame_buffer.cc index 76e97b7e47..cca685b923 100644 --- a/webrtc/common_video/video_frame_buffer.cc +++ b/webrtc/common_video/video_frame_buffer.cc @@ -133,4 +133,86 @@ rtc::scoped_refptr TextureBuffer::native_handle() const { return native_handle_; } + +WrappedI420Buffer::WrappedI420Buffer(int desired_width, + int desired_height, + int width, + int height, + const uint8_t* y_plane, + int y_stride, + const uint8_t* u_plane, + int u_stride, + const uint8_t* v_plane, + int v_stride, + const rtc::Callback0& no_longer_used) + : width_(desired_width), + height_(desired_height), + y_plane_(y_plane), + u_plane_(u_plane), + v_plane_(v_plane), + y_stride_(y_stride), + u_stride_(u_stride), + v_stride_(v_stride), + no_longer_used_cb_(no_longer_used) { + CHECK(width >= desired_width && height >= desired_height); + + // Center crop to |desired_width| x |desired_height|. + // Make sure offset is even so that u/v plane becomes aligned. + const int offset_x = ((width - desired_width) / 2) & ~1; + const int offset_y = ((height - desired_height) / 2) & ~1; + y_plane_ += y_stride_ * offset_y + offset_x; + u_plane_ += u_stride_ * (offset_y / 2) + (offset_x / 2); + v_plane_ += v_stride_ * (offset_y / 2) + (offset_x / 2); +} + +WrappedI420Buffer::~WrappedI420Buffer() { + no_longer_used_cb_(); +} + + +int WrappedI420Buffer::width() const { + return width_; +} + +int WrappedI420Buffer::height() const { + return height_; +} + +const uint8_t* WrappedI420Buffer::data(PlaneType type) const { + switch (type) { + case kYPlane: + return y_plane_; + case kUPlane: + return u_plane_; + case kVPlane: + return v_plane_; + default: + RTC_NOTREACHED(); + return nullptr; + } +} + +uint8_t* WrappedI420Buffer::data(PlaneType type) { + RTC_NOTREACHED(); + return nullptr; +} + +int WrappedI420Buffer::stride(PlaneType type) const { + switch (type) { + case kYPlane: + return y_stride_; + case kUPlane: + return u_stride_; + case kVPlane: + return v_stride_; + default: + RTC_NOTREACHED(); + return 0; + } +} + +rtc::scoped_refptr WrappedI420Buffer::native_handle() const { + return nullptr; +} + } // namespace webrtc