diff --git a/talk/app/webrtc/androidvideocapturer.cc b/talk/app/webrtc/androidvideocapturer.cc index 8901188492..e227022822 100644 --- a/talk/app/webrtc/androidvideocapturer.cc +++ b/talk/app/webrtc/androidvideocapturer.cc @@ -27,69 +27,50 @@ #include "talk/app/webrtc/androidvideocapturer.h" #include "talk/media/webrtc/webrtcvideoframe.h" -#include "webrtc/base/bind.h" -#include "webrtc/base/callback.h" #include "webrtc/base/common.h" #include "webrtc/base/json.h" #include "webrtc/base/timeutils.h" -#include "webrtc/base/thread.h" -#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" namespace webrtc { -using cricket::WebRtcVideoFrame; -using rtc::scoped_ptr; -using rtc::scoped_refptr; - -// An implementation of cricket::VideoFrameFactory for frames that are not -// guaranteed to outlive the created cricket::VideoFrame. -// A frame is injected using UpdateCapturedFrame, and converted into a -// cricket::VideoFrame with -// CreateAliasedFrame. UpdateCapturedFrame should be called before -// CreateAliasedFrame for every frame. +// A hack for avoiding deep frame copies in +// cricket::VideoCapturer.SignalFrameCaptured() using a custom FrameFactory. +// A frame is injected using UpdateCapturedFrame(), and converted into a +// cricket::VideoFrame with CreateAliasedFrame(). UpdateCapturedFrame() should +// be called before CreateAliasedFrame() for every frame. +// TODO(magjed): Add an interface cricket::VideoCapturer::OnFrameCaptured() +// for ref counted I420 frames instead of this hack. class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory { public: - FrameFactory(int width, - int height, - const scoped_refptr& delegate) - : start_time_(rtc::TimeNanos()), delegate_(delegate) { + FrameFactory(const rtc::scoped_refptr& delegate) + : start_time_(rtc::TimeNanos()), delegate_(delegate) { // Create a CapturedFrame that only contains header information, not the // actual pixel data. - captured_frame_.width = width; - captured_frame_.height = height; captured_frame_.pixel_height = 1; captured_frame_.pixel_width = 1; - captured_frame_.rotation = 0; - captured_frame_.data = NULL; + captured_frame_.data = nullptr; captured_frame_.data_size = cricket::CapturedFrame::kUnknownDataSize; captured_frame_.fourcc = static_cast(cricket::FOURCC_ANY); } - void UpdateCapturedFrame(void* frame_data, - int length, - int width, - int height, - int rotation, - int64 time_stamp_in_ns) { - // Make sure we don't overwrite the previous frame. - CHECK(captured_frame_.data == nullptr); - captured_frame_.fourcc = static_cast(cricket::FOURCC_YV12); - captured_frame_.data = frame_data; - captured_frame_.width = width; - captured_frame_.height = height; + void UpdateCapturedFrame( + const rtc::scoped_refptr& buffer, + int rotation, + int64 time_stamp_in_ns) { + buffer_ = buffer; + captured_frame_.width = buffer->width(); + captured_frame_.height = buffer->height(); captured_frame_.elapsed_time = rtc::TimeNanos() - start_time_; captured_frame_.time_stamp = time_stamp_in_ns; captured_frame_.rotation = rotation; - captured_frame_.data_size = length; } - void ClearCapturedFrame() const { - captured_frame_.data = nullptr; + void ClearCapturedFrame() { + buffer_ = nullptr; captured_frame_.width = 0; captured_frame_.height = 0; captured_frame_.elapsed_time = 0; captured_frame_.time_stamp = 0; - captured_frame_.data_size = 0; } const cricket::CapturedFrame* GetCapturedFrame() const { @@ -100,64 +81,23 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory { const cricket::CapturedFrame* captured_frame, int dst_width, int dst_height) const override { - // This override of CreateAliasedFrame creates a copy of the frame since - // |captured_frame_.data| is only guaranteed to be valid during the scope - // of |AndroidVideoCapturer::OnIncomingFrame_w|. // Check that captured_frame is actually our frame. CHECK(captured_frame == &captured_frame_); - CHECK(captured_frame->data != nullptr); - - if (!apply_rotation_ || captured_frame->rotation == kVideoRotation_0) { - CHECK(captured_frame->fourcc == cricket::FOURCC_YV12); - const uint8_t* y_plane = static_cast(captured_frame_.data); - - // Android guarantees that the stride is a multiple of 16. - // http://developer.android.com/reference/android/hardware/Camera.Parameters.html#setPreviewFormat%28int%29 - int y_stride; - int uv_stride; - webrtc::Calc16ByteAlignedStride(captured_frame->width, &y_stride, - &uv_stride); - const uint8_t* v_plane = y_plane + y_stride * captured_frame->height; - const uint8_t* u_plane = - v_plane + uv_stride * webrtc::AlignInt(captured_frame->height, 2) / 2; - - // Create a WrappedI420Buffer and bind the |no_longer_used| callback - // to the static method ReturnFrame. The |delegate_| is bound as an - // argument which means that the callback will hold a reference to - // |delegate_|. - rtc::scoped_refptr buffer( - new rtc::RefCountedObject( - dst_width, dst_height, captured_frame->width, - captured_frame->height, y_plane, y_stride, u_plane, uv_stride, - v_plane, uv_stride, - rtc::Bind(&AndroidVideoCapturer::FrameFactory::ReturnFrame, - delegate_, - captured_frame->time_stamp))); - cricket::VideoFrame* cricket_frame = new WebRtcVideoFrame( - buffer, captured_frame->elapsed_time, - captured_frame->time_stamp, captured_frame->GetRotation()); - // |cricket_frame| is now responsible for returning the frame. Clear - // |captured_frame_| so the frame isn't returned twice. - ClearCapturedFrame(); - return cricket_frame; - } - - scoped_ptr frame(new WebRtcVideoFrame()); - frame->Init(captured_frame, dst_width, dst_height, apply_rotation_); - return frame.release(); - } - - static void ReturnFrame(scoped_refptr delegate, - int64 time_stamp) { - delegate->ReturnBuffer(time_stamp); + rtc::scoped_ptr frame(new cricket::WebRtcVideoFrame( + ShallowCenterCrop(buffer_, dst_width, dst_height), + captured_frame->elapsed_time, captured_frame->time_stamp, + captured_frame->GetRotation())); + // Caller takes ownership. + // TODO(magjed): Change CreateAliasedFrame() to return a rtc::scoped_ptr. + return apply_rotation_ ? frame->GetCopyWithRotationApplied()->Copy() + : frame.release(); } private: uint64 start_time_; - // |captured_frame_| is mutable as a hacky way to modify it inside - // CreateAliasedframe(). - mutable cricket::CapturedFrame captured_frame_; - scoped_refptr delegate_; + rtc::scoped_refptr buffer_; + cricket::CapturedFrame captured_frame_; + rtc::scoped_refptr delegate_; }; AndroidVideoCapturer::AndroidVideoCapturer( @@ -202,8 +142,7 @@ cricket::CaptureState AndroidVideoCapturer::Start( CHECK(thread_checker_.CalledOnValidThread()); CHECK(!running_); - frame_factory_ = new AndroidVideoCapturer::FrameFactory( - capture_format.width, capture_format.height, delegate_.get()); + frame_factory_ = new AndroidVideoCapturer::FrameFactory(delegate_.get()); set_frame_factory(frame_factory_); running_ = true; @@ -252,24 +191,14 @@ void AndroidVideoCapturer::OnCapturerStarted(bool success) { SignalStateChange(this, new_state); } -void AndroidVideoCapturer::OnIncomingFrame(void* frame_data, - int length, - int width, - int height, - int rotation, - int64 time_stamp) { +void AndroidVideoCapturer::OnIncomingFrame( + rtc::scoped_refptr buffer, + int rotation, + int64 time_stamp) { CHECK(thread_checker_.CalledOnValidThread()); - frame_factory_->UpdateCapturedFrame(frame_data, length, width, height, - rotation, time_stamp); + frame_factory_->UpdateCapturedFrame(buffer, rotation, time_stamp); SignalFrameCaptured(this, frame_factory_->GetCapturedFrame()); - if (frame_factory_->GetCapturedFrame()->data == nullptr) { - // Ownership has been passed to a WrappedI420Buffer. Do nothing. - } else { - // |captured_frame_| has either been copied or dropped, return it - // immediately. - delegate_->ReturnBuffer(time_stamp); - frame_factory_->ClearCapturedFrame(); - } + frame_factory_->ClearCapturedFrame(); } void AndroidVideoCapturer::OnOutputFormatRequest( diff --git a/talk/app/webrtc/androidvideocapturer.h b/talk/app/webrtc/androidvideocapturer.h index 84b1c06d65..78594f8c23 100644 --- a/talk/app/webrtc/androidvideocapturer.h +++ b/talk/app/webrtc/androidvideocapturer.h @@ -32,6 +32,7 @@ #include "talk/media/base/videocapturer.h" #include "webrtc/base/thread_checker.h" +#include "webrtc/common_video/interface/video_frame_buffer.h" namespace webrtc { @@ -49,10 +50,6 @@ class AndroidVideoCapturerDelegate : public rtc::RefCountInterface { // The delegate may not call into AndroidVideoCapturer after this call. virtual void Stop() = 0; - // Notify that a frame received in OnIncomingFrame with |time_stamp| has been - // processed and can be returned. May be called on an arbitrary thread. - virtual void ReturnBuffer(int64 time_stamp) = 0; - // Must returns a JSON string "{{width=xxx, height=xxx, framerate = xxx}}" virtual std::string GetSupportedFormats() = 0; }; @@ -69,10 +66,8 @@ class AndroidVideoCapturer : public cricket::VideoCapturer { void OnCapturerStarted(bool success); // Called from JNI when a new frame has been captured. - void OnIncomingFrame(void* video_frame, - int length, - int width, - int height, + // Argument |buffer| is intentionally by value, for use with rtc::Bind. + void OnIncomingFrame(rtc::scoped_refptr buffer, int rotation, int64 time_stamp); diff --git a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc index 0b7a2efb21..43a60c3cec 100644 --- a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc +++ b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc @@ -29,6 +29,7 @@ #include "talk/app/webrtc/java/jni/androidvideocapturer_jni.h" #include "talk/app/webrtc/java/jni/classreferenceholder.h" #include "webrtc/base/bind.h" +#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" namespace webrtc_jni { @@ -70,9 +71,7 @@ AndroidVideoCapturerJni::AndroidVideoCapturerJni(JNIEnv* jni, jni, FindClass(jni, "org/webrtc/VideoCapturerAndroid$NativeObserver")), - capturer_(nullptr), - thread_(nullptr), - valid_global_refs_(true) { + capturer_(nullptr) { LOG(LS_INFO) << "AndroidVideoCapturerJni ctor"; thread_checker_.DetachFromThread(); } @@ -88,30 +87,24 @@ bool AndroidVideoCapturerJni::Init(jstring device_name) { } AndroidVideoCapturerJni::~AndroidVideoCapturerJni() { - valid_global_refs_ = false; - if (thread_ != nullptr) { - LOG(LS_INFO) << "AndroidVideoCapturerJni dtor - flush invoker"; - invoker_.Flush(thread_); - } - LOG(LS_INFO) << "AndroidVideoCapturerJni dtor done"; + LOG(LS_INFO) << "AndroidVideoCapturerJni dtor"; } void AndroidVideoCapturerJni::Start(int width, int height, int framerate, webrtc::AndroidVideoCapturer* capturer) { LOG(LS_INFO) << "AndroidVideoCapturerJni start"; - CHECK(thread_checker_.CalledOnValidThread()); - CHECK(capturer_ == nullptr); - thread_ = rtc::Thread::Current(); - capturer_ = capturer; - - j_frame_observer_ = NewGlobalRef( - jni(), + DCHECK(thread_checker_.CalledOnValidThread()); + { + rtc::CritScope cs(&capturer_lock_); + CHECK(capturer_ == nullptr); + CHECK(invoker_.get() == nullptr); + capturer_ = capturer; + invoker_.reset(new rtc::GuardedAsyncInvoker()); + } + jobject j_frame_observer = jni()->NewObject(*j_observer_class_, - GetMethodID(jni(), - *j_observer_class_, - "", - "(J)V"), - jlongFromPointer(this))); + GetMethodID(jni(), *j_observer_class_, "", "(J)V"), + jlongFromPointer(this)); CHECK_EXCEPTION(jni()) << "error during NewObject"; jmethodID m = GetMethodID( @@ -122,33 +115,40 @@ void AndroidVideoCapturerJni::Start(int width, int height, int framerate, m, width, height, framerate, application_context_, - j_frame_observer_); + j_frame_observer); CHECK_EXCEPTION(jni()) << "error during VideoCapturerAndroid.startCapture"; } void AndroidVideoCapturerJni::Stop() { LOG(LS_INFO) << "AndroidVideoCapturerJni stop"; - CHECK(thread_checker_.CalledOnValidThread()); - capturer_ = nullptr; + DCHECK(thread_checker_.CalledOnValidThread()); + { + rtc::CritScope cs(&capturer_lock_); + // Destroying |invoker_| will cancel all pending calls to |capturer_|. + invoker_ = nullptr; + capturer_ = nullptr; + } jmethodID m = GetMethodID(jni(), *j_video_capturer_class_, "stopCapture", "()V"); jni()->CallVoidMethod(*j_capturer_global_, m); CHECK_EXCEPTION(jni()) << "error during VideoCapturerAndroid.stopCapture"; - DeleteGlobalRef(jni(), j_frame_observer_); LOG(LS_INFO) << "AndroidVideoCapturerJni stop done"; } -void AndroidVideoCapturerJni::ReturnBuffer(int64 time_stamp) { - invoker_.AsyncInvoke( - thread_, - rtc::Bind(&AndroidVideoCapturerJni::ReturnBuffer_w, this, time_stamp)); -} - -void AndroidVideoCapturerJni::ReturnBuffer_w(int64 time_stamp) { - if (!valid_global_refs_) { - LOG(LS_ERROR) << "ReturnBuffer_w is called for invalid global refs."; +template +void AndroidVideoCapturerJni::AsyncCapturerInvoke( + const char* method_name, + void (webrtc::AndroidVideoCapturer::*method)(Args...), + Args... args) { + rtc::CritScope cs(&capturer_lock_); + if (!invoker_) { + LOG(LS_WARNING) << method_name << "() called for closed capturer."; return; } + invoker_->AsyncInvoke(rtc::Bind(method, capturer_, args...)); +} + +void AndroidVideoCapturerJni::ReturnBuffer(int64 time_stamp) { jmethodID m = GetMethodID(jni(), *j_video_capturer_class_, "returnBuffer", "(J)V"); jni()->CallVoidMethod(*j_capturer_global_, m, time_stamp); @@ -166,10 +166,10 @@ std::string AndroidVideoCapturerJni::GetSupportedFormats() { } void AndroidVideoCapturerJni::OnCapturerStarted(bool success) { - LOG(LS_INFO) << "AndroidVideoCapturerJni capture started: " << success; - invoker_.AsyncInvoke( - thread_, - rtc::Bind(&AndroidVideoCapturerJni::OnCapturerStarted_w, this, success)); + LOG(LS_INFO) << "AndroidVideoCapturerJni capture started: " << success; + AsyncCapturerInvoke("OnCapturerStarted", + &webrtc::AndroidVideoCapturer::OnCapturerStarted, + success); } void AndroidVideoCapturerJni::OnIncomingFrame(void* video_frame, @@ -178,57 +178,34 @@ void AndroidVideoCapturerJni::OnIncomingFrame(void* video_frame, int height, int rotation, int64 time_stamp) { - invoker_.AsyncInvoke( - thread_, - rtc::Bind(&AndroidVideoCapturerJni::OnIncomingFrame_w, this, video_frame, - length, width, height, rotation, time_stamp)); + const uint8_t* y_plane = static_cast(video_frame); + // Android guarantees that the stride is a multiple of 16. + // http://developer.android.com/reference/android/hardware/Camera.Parameters.html#setPreviewFormat%28int%29 + int y_stride; + int uv_stride; + webrtc::Calc16ByteAlignedStride(width, &y_stride, &uv_stride); + const uint8_t* v_plane = y_plane + y_stride * height; + const uint8_t* u_plane = + v_plane + uv_stride * webrtc::AlignInt(height, 2) / 2; + + // Wrap the Java buffer, and call ReturnBuffer() in the wrapped + // VideoFrameBuffer destructor. + rtc::scoped_refptr buffer( + new rtc::RefCountedObject( + width, height, y_plane, y_stride, u_plane, uv_stride, v_plane, + uv_stride, + rtc::Bind(&AndroidVideoCapturerJni::ReturnBuffer, this, time_stamp))); + AsyncCapturerInvoke("OnIncomingFrame", + &webrtc::AndroidVideoCapturer::OnIncomingFrame, + buffer, rotation, time_stamp); } void AndroidVideoCapturerJni::OnOutputFormatRequest(int width, int height, int fps) { - invoker_.AsyncInvoke( - thread_, - rtc::Bind(&AndroidVideoCapturerJni::OnOutputFormatRequest_w, - this, width, height, fps)); -} - -void AndroidVideoCapturerJni::OnCapturerStarted_w(bool success) { - CHECK(thread_checker_.CalledOnValidThread()); - if (capturer_) { - capturer_->OnCapturerStarted(success); - } else { - LOG(LS_WARNING) << "OnCapturerStarted_w is called for closed capturer."; - } -} - -void AndroidVideoCapturerJni::OnIncomingFrame_w(void* video_frame, - int length, - int width, - int height, - int rotation, - int64 time_stamp) { - CHECK(thread_checker_.CalledOnValidThread()); - if (capturer_) { - capturer_->OnIncomingFrame(video_frame, length, width, height, rotation, - time_stamp); - } else { - LOG(LS_INFO) << - "Frame arrived after camera has been stopped: " << time_stamp << - ". Valid global refs: " << valid_global_refs_; - ReturnBuffer_w(time_stamp); - } -} - -void AndroidVideoCapturerJni::OnOutputFormatRequest_w(int width, - int height, - int fps) { - CHECK(thread_checker_.CalledOnValidThread()); - if (capturer_) { - capturer_->OnOutputFormatRequest(width, height, fps); - } else { - LOG(LS_WARNING) << "OnOutputFormatRequest_w is called for closed capturer."; - } + AsyncCapturerInvoke("OnOutputFormatRequest", + &webrtc::AndroidVideoCapturer::OnOutputFormatRequest, + width, height, fps); } JNIEnv* AndroidVideoCapturerJni::jni() { return AttachCurrentThreadIfNeeded(); } @@ -238,16 +215,14 @@ JOW(void, VideoCapturerAndroid_00024NativeObserver_nativeOnFrameCaptured) jint width, jint height, jint rotation, jlong ts) { jboolean is_copy = true; jbyte* bytes = jni->GetByteArrayElements(j_frame, &is_copy); - if (!is_copy) { - reinterpret_cast(j_capturer) - ->OnIncomingFrame(bytes, length, width, height, rotation, ts); - } else { - // If this is a copy of the original frame, it means that the memory - // is not direct memory and thus VideoCapturerAndroid does not guarantee - // that the memory is valid when we have released |j_frame|. - LOG(LS_ERROR) << "NativeObserver_nativeOnFrameCaptured: frame is a copy"; - CHECK(false) << "j_frame is a copy."; - } + // If this is a copy of the original frame, it means that the memory + // is not direct memory and thus VideoCapturerAndroid does not guarantee + // that the memory is valid when we have released |j_frame|. + // TODO(magjed): Move ReleaseByteArrayElements() into ReturnBuffer() and + // remove this check. + CHECK(!is_copy) << "NativeObserver_nativeOnFrameCaptured: frame is a copy"; + reinterpret_cast(j_capturer) + ->OnIncomingFrame(bytes, length, width, height, rotation, ts); jni->ReleaseByteArrayElements(j_frame, bytes, JNI_ABORT); } @@ -267,4 +242,3 @@ JOW(void, VideoCapturerAndroid_00024NativeObserver_nativeOnOutputFormatRequest) } } // namespace webrtc_jni - diff --git a/talk/app/webrtc/java/jni/androidvideocapturer_jni.h b/talk/app/webrtc/java/jni/androidvideocapturer_jni.h index ed3f8d2cc2..625486fadb 100644 --- a/talk/app/webrtc/java/jni/androidvideocapturer_jni.h +++ b/talk/app/webrtc/java/jni/androidvideocapturer_jni.h @@ -34,6 +34,7 @@ #include "talk/app/webrtc/androidvideocapturer.h" #include "talk/app/webrtc/java/jni/jni_helpers.h" #include "webrtc/base/asyncinvoker.h" +#include "webrtc/base/criticalsection.h" #include "webrtc/base/thread_checker.h" namespace webrtc_jni { @@ -56,8 +57,6 @@ class AndroidVideoCapturerJni : public webrtc::AndroidVideoCapturerDelegate { webrtc::AndroidVideoCapturer* capturer) override; void Stop() override; - void ReturnBuffer(int64 time_stamp) override; - std::string GetSupportedFormats() override; // Called from VideoCapturerAndroid::NativeObserver on a Java thread. @@ -75,34 +74,31 @@ protected: private: bool Init(jstring device_name); - - void OnCapturerStarted_w(bool success); - void OnCapturerStopped_w(); - void OnIncomingFrame_w(void* video_frame, - int length, - int width, - int height, - int rotation, - int64 time_stamp); - void OnOutputFormatRequest_w(int width, int height, int fps); - void ReturnBuffer_w(int64 time_stamp); - + void ReturnBuffer(int64 time_stamp); JNIEnv* jni(); + // Helper function to make safe asynchronous calls to |capturer_|. The calls + // are not guaranteed to be delivered. + template + void AsyncCapturerInvoke( + const char* method_name, + void (webrtc::AndroidVideoCapturer::*method)(Args...), + Args... args); + const ScopedGlobalRef j_capturer_global_; const ScopedGlobalRef j_video_capturer_class_; const ScopedGlobalRef j_observer_class_; - volatile bool valid_global_refs_; - jobject j_frame_observer_; rtc::ThreadChecker thread_checker_; - rtc::Thread* thread_; // The thread where Start is called on. // |capturer| is a guaranteed to be a valid pointer between a call to // AndroidVideoCapturerDelegate::Start // until AndroidVideoCapturerDelegate::Stop. - webrtc::AndroidVideoCapturer* capturer_; - rtc::AsyncInvoker invoker_; + rtc::CriticalSection capturer_lock_; + webrtc::AndroidVideoCapturer* capturer_ GUARDED_BY(capturer_lock_); + // |invoker_| is used to communicate with |capturer_| on the thread Start() is + // called on. + rtc::scoped_ptr invoker_ GUARDED_BY(capturer_lock_); static jobject application_context_; diff --git a/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java b/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java index dee414673c..ac42883e98 100644 --- a/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java +++ b/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java @@ -720,17 +720,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba frameObserver.OnOutputFormatRequest(width, height, fps); } - synchronized void returnBuffer(final long timeStamp) { - if (cameraThreadHandler == null) { - // The camera has been stopped. - videoBuffers.returnBuffer(timeStamp); - return; - } - cameraThreadHandler.post(new Runnable() { - @Override public void run() { - videoBuffers.returnBuffer(timeStamp); - } - }); + void returnBuffer(long timeStamp) { + videoBuffers.returnBuffer(timeStamp); } private int getDeviceOrientation() { @@ -854,7 +845,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } // Class used for allocating and bookkeeping video frames. All buffers are - // direct allocated so that they can be directly used from native code. + // direct allocated so that they can be directly used from native code. This class is + // synchronized and can be called from multiple threads. private static class FramePool { // Arbitrary queue depth. Higher number means more memory allocated & held, // lower number means more sensitivity to processing time in the client (and @@ -869,12 +861,12 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private int frameSize = 0; private Camera camera; - int numCaptureBuffersAvailable() { + synchronized int numCaptureBuffersAvailable() { return queuedBuffers.size(); } // Discards previous queued buffers and adds new callback buffers to camera. - void queueCameraBuffers(int frameSize, Camera camera) { + synchronized void queueCameraBuffers(int frameSize, Camera camera) { this.camera = camera; this.frameSize = frameSize; @@ -888,7 +880,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba + " buffers of size " + frameSize + "."); } - String pendingFramesTimeStamps() { + synchronized String pendingFramesTimeStamps() { List timeStampsMs = new ArrayList(); for (Long timeStampNs : pendingBuffers.keySet()) { timeStampsMs.add(TimeUnit.NANOSECONDS.toMillis(timeStampNs)); @@ -896,7 +888,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba return timeStampsMs.toString(); } - void stopReturnBuffersToCamera() { + synchronized void stopReturnBuffersToCamera() { this.camera = null; queuedBuffers.clear(); // Frames in |pendingBuffers| need to be kept alive until they are returned. @@ -906,7 +898,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba : " Pending buffers: " + pendingFramesTimeStamps() + ".")); } - boolean reserveByteBuffer(byte[] data, long timeStamp) { + synchronized boolean reserveByteBuffer(byte[] data, long timeStamp) { final ByteBuffer buffer = queuedBuffers.remove(data); if (buffer == null) { // Frames might be posted to |onPreviewFrame| with the previous format while changing @@ -930,7 +922,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba return true; } - void returnBuffer(long timeStamp) { + synchronized void returnBuffer(long timeStamp) { final ByteBuffer returnedFrame = pendingBuffers.remove(timeStamp); if (returnedFrame == null) { throw new RuntimeException("unknown data buffer with time stamp " diff --git a/webrtc/common_video/interface/video_frame_buffer.h b/webrtc/common_video/interface/video_frame_buffer.h index 4079019ccc..a7ac8d84d7 100644 --- a/webrtc/common_video/interface/video_frame_buffer.h +++ b/webrtc/common_video/interface/video_frame_buffer.h @@ -109,9 +109,7 @@ class NativeHandleBuffer : public VideoFrameBuffer { class WrappedI420Buffer : public webrtc::VideoFrameBuffer { public: - WrappedI420Buffer(int desired_width, - int desired_height, - int width, + WrappedI420Buffer(int width, int height, const uint8_t* y_plane, int y_stride, @@ -135,17 +133,24 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer { friend class rtc::RefCountedObject; ~WrappedI420Buffer() override; - int width_; - int height_; - const uint8_t* y_plane_; - const uint8_t* u_plane_; - const uint8_t* v_plane_; + const int width_; + const int height_; + const uint8_t* const y_plane_; + const uint8_t* const u_plane_; + const uint8_t* const v_plane_; const int y_stride_; const int u_stride_; const int v_stride_; rtc::Callback0 no_longer_used_cb_; }; +// Helper function to crop |buffer| without making a deep copy. May only be used +// for non-native frames. +rtc::scoped_refptr ShallowCenterCrop( + const rtc::scoped_refptr& buffer, + int cropped_width, + int cropped_height); + } // namespace webrtc #endif // WEBRTC_VIDEO_FRAME_BUFFER_H_ diff --git a/webrtc/common_video/video_frame_buffer.cc b/webrtc/common_video/video_frame_buffer.cc index 2cf427ce42..cad33e8e97 100644 --- a/webrtc/common_video/video_frame_buffer.cc +++ b/webrtc/common_video/video_frame_buffer.cc @@ -17,6 +17,12 @@ static const int kBufferAlignment = 64; namespace webrtc { +namespace { + +// Used in rtc::Bind to keep a buffer alive until destructor is called. +static void NoLongerUsedCallback(rtc::scoped_refptr dummy) {} + +} // anonymous namespace VideoFrameBuffer::~VideoFrameBuffer() {} @@ -135,9 +141,7 @@ void* NativeHandleBuffer::native_handle() const { return native_handle_; } -WrappedI420Buffer::WrappedI420Buffer(int desired_width, - int desired_height, - int width, +WrappedI420Buffer::WrappedI420Buffer(int width, int height, const uint8_t* y_plane, int y_stride, @@ -146,31 +150,21 @@ WrappedI420Buffer::WrappedI420Buffer(int desired_width, const uint8_t* v_plane, int v_stride, const rtc::Callback0& no_longer_used) - : width_(desired_width), - height_(desired_height), - y_plane_(y_plane), - u_plane_(u_plane), - v_plane_(v_plane), - y_stride_(y_stride), - u_stride_(u_stride), - v_stride_(v_stride), - no_longer_used_cb_(no_longer_used) { - CHECK(width >= desired_width && height >= desired_height); - - // Center crop to |desired_width| x |desired_height|. - // Make sure offset is even so that u/v plane becomes aligned. - const int offset_x = ((width - desired_width) / 2) & ~1; - const int offset_y = ((height - desired_height) / 2) & ~1; - y_plane_ += y_stride_ * offset_y + offset_x; - u_plane_ += u_stride_ * (offset_y / 2) + (offset_x / 2); - v_plane_ += v_stride_ * (offset_y / 2) + (offset_x / 2); + : width_(width), + height_(height), + y_plane_(y_plane), + u_plane_(u_plane), + v_plane_(v_plane), + y_stride_(y_stride), + u_stride_(u_stride), + v_stride_(v_stride), + no_longer_used_cb_(no_longer_used) { } WrappedI420Buffer::~WrappedI420Buffer() { no_longer_used_cb_(); } - int WrappedI420Buffer::width() const { return width_; } @@ -221,4 +215,37 @@ rtc::scoped_refptr WrappedI420Buffer::NativeToI420Buffer() { return nullptr; } +rtc::scoped_refptr ShallowCenterCrop( + const rtc::scoped_refptr& buffer, + int cropped_width, + int cropped_height) { + CHECK(buffer->native_handle() == nullptr); + CHECK_LE(cropped_width, buffer->width()); + CHECK_LE(cropped_height, buffer->height()); + if (buffer->width() == cropped_width && buffer->height() == cropped_height) + return buffer; + + // Center crop to |cropped_width| x |cropped_height|. + // Make sure offset is even so that u/v plane becomes aligned. + const int uv_offset_x = (buffer->width() - cropped_width) / 4; + const int uv_offset_y = (buffer->height() - cropped_height) / 4; + const int offset_x = uv_offset_x * 2; + const int offset_y = uv_offset_y * 2; + + // Const cast to call the correct const-version of data(). + const VideoFrameBuffer* const_buffer(buffer.get()); + const uint8_t* y_plane = const_buffer->data(kYPlane) + + buffer->stride(kYPlane) * offset_y + offset_x; + const uint8_t* u_plane = const_buffer->data(kUPlane) + + buffer->stride(kUPlane) * uv_offset_y + uv_offset_x; + const uint8_t* v_plane = const_buffer->data(kVPlane) + + buffer->stride(kVPlane) * uv_offset_y + uv_offset_x; + return new rtc::RefCountedObject( + cropped_width, cropped_height, + y_plane, buffer->stride(kYPlane), + u_plane, buffer->stride(kUPlane), + v_plane, buffer->stride(kVPlane), + rtc::Bind(&NoLongerUsedCallback, buffer)); +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 04bbd161ec..3e4367fb67 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -739,7 +739,6 @@ int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img, uint32_t timestamp) { // using a WrappedI420Buffer. rtc::scoped_refptr img_wrapped_buffer( new rtc::RefCountedObject( - img->d_w, img->d_h, img->d_w, img->d_h, img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],