From 39607c9e3433389db0256d1df4fcdf58a99fedf8 Mon Sep 17 00:00:00 2001 From: magjed Date: Thu, 14 Jul 2016 08:12:17 -0700 Subject: [PATCH] AVFoundationVideoCapturer: Output native frames instead of I420 frames BUG=webrtc:4081 Review-Url: https://codereview.webrtc.org/2135953002 Cr-Commit-Position: refs/heads/master@{#13478} --- webrtc/sdk/DEPS | 1 + .../Classes/avfoundationvideocapturer.h | 4 +- .../Classes/avfoundationvideocapturer.mm | 78 ++++++++----------- 3 files changed, 38 insertions(+), 45 deletions(-) diff --git a/webrtc/sdk/DEPS b/webrtc/sdk/DEPS index 1bb7a2fe8c..d6e53d504d 100644 --- a/webrtc/sdk/DEPS +++ b/webrtc/sdk/DEPS @@ -1,6 +1,7 @@ include_rules = [ "+WebRTC", "+webrtc/api", + "+webrtc/common_video/include", "+webrtc/media", "+webrtc/system_wrappers", ] diff --git a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.h b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.h index c523b527bb..37378e7da1 100644 --- a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.h +++ b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.h @@ -13,6 +13,7 @@ #import +#include "webrtc/common_video/include/i420_buffer_pool.h" #include "webrtc/media/base/videocapturer.h" #include "webrtc/video_frame.h" @@ -61,10 +62,11 @@ class AVFoundationVideoCapturer : public cricket::VideoCapturer, void OnMessage(rtc::Message *msg) override; private: - void OnFrameMessage(CVImageBufferRef image_buffer, int64_t capture_time); + void OnFrameMessage(CVImageBufferRef image_buffer, int64_t capture_time_ns); RTCAVFoundationVideoCapturerInternal *_capturer; rtc::Thread *_startThread; // Set in Start(), unset in Stop(). + webrtc::I420BufferPool _buffer_pool; }; // AVFoundationVideoCapturer } // namespace webrtc diff --git a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm index 778d70b897..9b1a7832d2 100644 --- a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm +++ b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm @@ -22,6 +22,7 @@ #include "webrtc/base/bind.h" #include "webrtc/base/checks.h" #include "webrtc/base/thread.h" +#include "webrtc/common_video/include/corevideo_frame_buffer.h" // TODO(tkchin): support other formats. static NSString *const kDefaultPreset = AVCaptureSessionPreset640x480; @@ -642,57 +643,46 @@ void AVFoundationVideoCapturer::OnMessage(rtc::Message *msg) { } void AVFoundationVideoCapturer::OnFrameMessage(CVImageBufferRef image_buffer, - int64_t capture_time) { + int64_t capture_time_ns) { RTC_DCHECK(_startThread->IsCurrent()); - // Base address must be unlocked to access frame data. - CVOptionFlags lock_flags = kCVPixelBufferLock_ReadOnly; - CVReturn ret = CVPixelBufferLockBaseAddress(image_buffer, lock_flags); - if (ret != kCVReturnSuccess) { + rtc::scoped_refptr buffer = + new rtc::RefCountedObject(image_buffer); + + const int captured_width = buffer->width(); + const int captured_height = buffer->height(); + + int adapted_width; + int adapted_height; + int crop_width; + int crop_height; + int crop_x; + int crop_y; + int64_t translated_camera_time_us; + + if (!AdaptFrame(captured_width, captured_height, + capture_time_ns / rtc::kNumNanosecsPerMicrosec, + rtc::TimeMicros(), &adapted_width, &adapted_height, + &crop_width, &crop_height, &crop_x, &crop_y, + &translated_camera_time_us)) { + CVBufferRelease(image_buffer); return; } - static size_t const kYPlaneIndex = 0; - static size_t const kUVPlaneIndex = 1; - uint8_t* y_plane_address = - static_cast(CVPixelBufferGetBaseAddressOfPlane(image_buffer, - kYPlaneIndex)); - size_t y_plane_height = - CVPixelBufferGetHeightOfPlane(image_buffer, kYPlaneIndex); - size_t y_plane_width = - CVPixelBufferGetWidthOfPlane(image_buffer, kYPlaneIndex); - size_t y_plane_bytes_per_row = - CVPixelBufferGetBytesPerRowOfPlane(image_buffer, kYPlaneIndex); - size_t uv_plane_height = - CVPixelBufferGetHeightOfPlane(image_buffer, kUVPlaneIndex); - size_t uv_plane_bytes_per_row = - CVPixelBufferGetBytesPerRowOfPlane(image_buffer, kUVPlaneIndex); - size_t frame_size = y_plane_bytes_per_row * y_plane_height + - uv_plane_bytes_per_row * uv_plane_height; + if (adapted_width != captured_width || crop_width != captured_width || + adapted_height != captured_height || crop_height != captured_height) { + // TODO(magjed): Avoid converting to I420. + rtc::scoped_refptr scaled_buffer( + _buffer_pool.CreateBuffer(adapted_width, adapted_height)); + scaled_buffer->CropAndScaleFrom(buffer->NativeToI420Buffer(), crop_x, + crop_y, crop_width, crop_height); + buffer = scaled_buffer; + } - // Sanity check assumption that planar bytes are contiguous. - uint8_t* uv_plane_address = - static_cast(CVPixelBufferGetBaseAddressOfPlane(image_buffer, - kUVPlaneIndex)); - RTC_DCHECK(uv_plane_address == - y_plane_address + y_plane_height * y_plane_bytes_per_row); + OnFrame(cricket::WebRtcVideoFrame(buffer, webrtc::kVideoRotation_0, + translated_camera_time_us), + captured_width, captured_height); - // Stuff data into a cricket::CapturedFrame. - cricket::CapturedFrame frame; - frame.width = y_plane_width; - frame.height = y_plane_height; - frame.pixel_width = 1; - frame.pixel_height = 1; - frame.fourcc = static_cast(cricket::FOURCC_NV12); - frame.time_stamp = capture_time; - frame.data = y_plane_address; - frame.data_size = frame_size; - - // This will call a superclass method that will perform the frame conversion - // to I420. - SignalFrameCaptured(this, &frame); - - CVPixelBufferUnlockBaseAddress(image_buffer, lock_flags); CVBufferRelease(image_buffer); }