From 4ea50c2b421ae3e40d1d02b8eb8c5802288b181e Mon Sep 17 00:00:00 2001 From: Anders Carlsson Date: Wed, 28 Mar 2018 16:18:04 +0200 Subject: [PATCH] Add unit tests for RTCCVPixelBuffer and ObjCVideoTrackSource. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL also fixes a couple of bugs found in the toI420 method for RTCCVPixelBuffers backed by RGB CVPixelBuffers. Bug: webrtc:9007 Change-Id: I19ab8177f4b124a503cfda9f0166bd960f668982 Reviewed-on: https://webrtc-review.googlesource.com/64940 Commit-Queue: Anders Carlsson Reviewed-by: Kári Helgason Cr-Commit-Position: refs/heads/master@{#22656} --- sdk/BUILD.gn | 17 + .../Classes/Video/RTCCVPixelBuffer.mm | 87 +++-- .../Framework/Classes/Video/RTCI420Buffer.mm | 4 + .../Headers/WebRTC/RTCVideoFrameBuffer.h | 5 +- .../UnitTests/ObjCVideoTrackSource_xctest.mm | 364 ++++++++++++++++++ .../UnitTests/RTCCVPixelBuffer_xctest.mm | 227 +++++++++++ .../UnitTests/frame_buffer_helpers.h | 22 ++ .../UnitTests/frame_buffer_helpers.mm | 122 ++++++ 8 files changed, 815 insertions(+), 33 deletions(-) create mode 100644 sdk/objc/Framework/UnitTests/ObjCVideoTrackSource_xctest.mm create mode 100644 sdk/objc/Framework/UnitTests/RTCCVPixelBuffer_xctest.mm create mode 100644 sdk/objc/Framework/UnitTests/frame_buffer_helpers.h create mode 100644 sdk/objc/Framework/UnitTests/frame_buffer_helpers.mm diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index bd22f42835..732d1e3ffc 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -669,22 +669,33 @@ if (is_ios || is_mac) { ] sources = [ + "objc/Framework/UnitTests/ObjCVideoTrackSource_xctest.mm", + "objc/Framework/UnitTests/RTCCVPixelBuffer_xctest.mm", "objc/Framework/UnitTests/RTCCallbackLogger_xctest.m", "objc/Framework/UnitTests/RTCDoNotPutCPlusPlusInFrameworkHeaders_xctest.m", "objc/Framework/UnitTests/RTCFileVideoCapturer_xctest.mm", + "objc/Framework/UnitTests/frame_buffer_helpers.h", + "objc/Framework/UnitTests/frame_buffer_helpers.mm", ] deps = [ ":common_objc", ":framework_objc", + ":native_api", + ":native_video", ":videocapture_objc", + ":videoframebuffer_objc", ":videosource_objc", ":videotoolbox_objc", "../../system_wrappers:system_wrappers_default", + "../api:video_frame_api_i420", + "../common_video:common_video", "../media:rtc_media_base", + "../media:rtc_media_tests_utils", "../modules:module_api", "../rtc_base:rtc_base", "../rtc_base:rtc_base_tests_utils", + "//third_party/libyuv", ] if (rtc_use_metal_rendering) { @@ -698,6 +709,12 @@ if (is_ios || is_mac) { ] include_dirs += [ "$root_out_dir/WebRTC.framework/Headers/" ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin + # (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } } bundle_data("sdk_unittests_bundle_data") { diff --git a/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm b/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm index 203896758e..8ca2106d70 100644 --- a/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm +++ b/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm @@ -28,6 +28,8 @@ @synthesize pixelBuffer = _pixelBuffer; @synthesize cropX = _cropX; @synthesize cropY = _cropY; +@synthesize cropWidth = _cropWidth; +@synthesize cropHeight = _cropHeight; + (NSSet*)supportedPixelFormats { return [NSSet setWithObjects:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), @@ -112,11 +114,15 @@ return 0; } -- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer { +- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer + withTempBuffer:(nullable uint8_t*)tmpBuffer { const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + RTC_DCHECK(srcPixelFormat == CVPixelBufferGetPixelFormatType(outputPixelBuffer)); + switch (srcPixelFormat) { case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { + RTC_DCHECK(tmpBuffer); [self cropAndScaleNV12To:outputPixelBuffer withTempBuffer:tmpBuffer]; break; } @@ -143,10 +149,10 @@ case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { const uint8_t* srcY = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); const uint8_t* srcUV = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); + static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); // Crop just by modifying pointers. @@ -173,32 +179,52 @@ } case kCVPixelFormatType_32BGRA: case kCVPixelFormatType_32ARGB: { - const uint8_t* src = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + CVPixelBufferRef scaledPixelBuffer = NULL; + CVPixelBufferRef sourcePixelBuffer = NULL; + if ([self requiresCropping] || + [self requiresScalingToWidth:i420Buffer.width height:i420Buffer.height]) { + CVPixelBufferCreate( + NULL, i420Buffer.width, i420Buffer.height, pixelFormat, NULL, &scaledPixelBuffer); + [self cropAndScaleTo:scaledPixelBuffer withTempBuffer:NULL]; + + CVPixelBufferLockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly); + sourcePixelBuffer = scaledPixelBuffer; + } else { + sourcePixelBuffer = _pixelBuffer; + } + const uint8_t* src = static_cast(CVPixelBufferGetBaseAddress(sourcePixelBuffer)); + const size_t bytesPerRow = CVPixelBufferGetBytesPerRow(sourcePixelBuffer); - uint32 libyuvPixelFormat = 0; if (pixelFormat == kCVPixelFormatType_32BGRA) { - libyuvPixelFormat = libyuv::FOURCC_ARGB; + // Corresponds to libyuv::FOURCC_ARGB + libyuv::ARGBToI420(src, + bytesPerRow, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); } else if (pixelFormat == kCVPixelFormatType_32ARGB) { - libyuvPixelFormat = libyuv::FOURCC_ABGR; + // Corresponds to libyuv::FOURCC_BGRA + libyuv::BGRAToI420(src, + bytesPerRow, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); } - libyuv::ConvertToI420(src, - 0, - i420Buffer.mutableDataY, - i420Buffer.strideY, - i420Buffer.mutableDataU, - i420Buffer.strideU, - i420Buffer.mutableDataV, - i420Buffer.strideV, - _cropX, - _cropY, - _cropWidth, - _cropHeight, - i420Buffer.width, - i420Buffer.height, - libyuv::kRotate0, - libyuvPixelFormat); + if (scaledPixelBuffer) { + CVPixelBufferUnlockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly); + CVBufferRelease(scaledPixelBuffer); + } break; } default: { RTC_NOTREACHED() << "Unsupported pixel format."; } @@ -226,11 +252,9 @@ // Prepare source pointers. CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); - const uint8_t* srcY = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + const uint8_t* srcY = static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); - const uint8_t* srcUV = - static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); + const uint8_t* srcUV = static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); // Crop just by modifying pointers. @@ -264,13 +288,12 @@ const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); - uint8_t* dst = - reinterpret_cast(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0)); - const int dstStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0); + uint8_t* dst = reinterpret_cast(CVPixelBufferGetBaseAddress(outputPixelBuffer)); + const int dstStride = CVPixelBufferGetBytesPerRow(outputPixelBuffer); // Prepare source pointers. CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); - const uint8_t* src = static_cast(CVPixelBufferGetBaseAddress(_pixelBuffer)); + const uint8_t* src = static_cast(CVPixelBufferGetBaseAddress(_pixelBuffer)); const int srcStride = CVPixelBufferGetBytesPerRow(_pixelBuffer); // Crop just by modifying pointers. diff --git a/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm b/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm index a428a8d6cb..2848f2b45e 100644 --- a/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm +++ b/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm @@ -89,6 +89,10 @@ return self; } +- (rtc::scoped_refptr)nativeI420Buffer { + return _i420Buffer; +} + @end @implementation RTCMutableI420Buffer diff --git a/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h b/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h index 4a683b0c92..d0f75b091f 100644 --- a/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h +++ b/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h @@ -71,6 +71,8 @@ RTC_EXPORT @property(nonatomic, readonly) CVPixelBufferRef pixelBuffer; @property(nonatomic, readonly) int cropX; @property(nonatomic, readonly) int cropY; +@property(nonatomic, readonly) int cropWidth; +@property(nonatomic, readonly) int cropHeight; + (NSSet *)supportedPixelFormats; @@ -89,7 +91,8 @@ RTC_EXPORT /** The minimum size of the |tmpBuffer| must be the number of bytes returned from the * bufferSizeForCroppingAndScalingToWidth:height: method. */ -- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t *)tmpBuffer; +- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer + withTempBuffer:(nullable uint8_t *)tmpBuffer; @end diff --git a/sdk/objc/Framework/UnitTests/ObjCVideoTrackSource_xctest.mm b/sdk/objc/Framework/UnitTests/ObjCVideoTrackSource_xctest.mm new file mode 100644 index 0000000000..d2ed398525 --- /dev/null +++ b/sdk/objc/Framework/UnitTests/ObjCVideoTrackSource_xctest.mm @@ -0,0 +1,364 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#include "sdk/objc/Framework/Native/src/objc_video_track_source.h" + +#import "Video/RTCI420Buffer+Private.h" +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "media/base/fakevideorenderer.h" +#include "rtc_base/refcountedobject.h" +#include "rtc_base/scoped_ref_ptr.h" +#include "sdk/objc/Framework/Native/api/video_frame.h" +#import "sdk/objc/Framework/UnitTests/frame_buffer_helpers.h" + +typedef void (^VideoSinkCallback)(RTCVideoFrame *); + +namespace { + +class ObjCCallbackVideoSink : public rtc::VideoSinkInterface { + public: + ObjCCallbackVideoSink(VideoSinkCallback callback) : callback_(callback) {} + + virtual void OnFrame(const webrtc::VideoFrame &frame) { + callback_(NativeToObjCVideoFrame(frame)); + } + + private: + VideoSinkCallback callback_; +}; + +} // namespace + +@interface ObjCVideoTrackSourceTests : XCTestCase +@end + +@implementation ObjCVideoTrackSourceTests { + rtc::scoped_refptr _video_source; +} + +- (void)setUp { + _video_source = new rtc::RefCountedObject(); +} + +- (void)tearDown { + _video_source = NULL; +} + +- (void)testOnCapturedFrameAdaptsFrame { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + cricket::FakeVideoRenderer *video_renderer = new cricket::FakeVideoRenderer(); + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(video_renderer, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + XCTAssertEqual(video_renderer->num_rendered_frames(), 1); + XCTAssertEqual(video_renderer->width(), 360); + XCTAssertEqual(video_renderer->height(), 640); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameWithoutAdaptation { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 360, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTCVideoFrame *outputFrame) { + XCTAssertEqual(frame.width, outputFrame.width); + XCTAssertEqual(frame.height, outputFrame.height); + + RTCCVPixelBuffer *outputBuffer = outputFrame.buffer; + XCTAssertEqual(buffer.cropX, outputBuffer.cropX); + XCTAssertEqual(buffer.cropY, outputBuffer.cropY); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameCVPixelBufferNeedsAdaptation { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTCVideoFrame *outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTCCVPixelBuffer *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 0); + XCTAssertEqual(outputBuffer.cropY, 0); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameCVPixelBufferNeedsCropping { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTCVideoFrame *outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTCCVPixelBuffer *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 10); + XCTAssertEqual(outputBuffer.cropY, 0); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFramePreAdaptedCVPixelBufferNeedsAdaptation { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:700 + adaptedHeight:700 + cropWidth:720 + cropHeight:1280 + cropX:0 + cropY:0]; + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTCVideoFrame *outputFrame) { + XCTAssertEqual(outputFrame.width, 297); + XCTAssertEqual(outputFrame.height, 525); + + RTCCVPixelBuffer *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 152); + XCTAssertEqual(outputBuffer.cropY, 0); + XCTAssertEqual(outputBuffer.cropWidth, 396); + XCTAssertEqual(outputBuffer.cropHeight, 700); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFramePreCroppedCVPixelBufferNeedsCropping { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:370 + adaptedHeight:640 + cropWidth:370 + cropHeight:640 + cropX:10 + cropY:0]; + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTCVideoFrame *outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTCCVPixelBuffer *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 14); + XCTAssertEqual(outputBuffer.cropY, 0); + XCTAssertEqual(outputBuffer.cropWidth, 360); + XCTAssertEqual(outputBuffer.cropHeight, 640); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameSmallerPreCroppedCVPixelBufferNeedsCropping { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:300 + adaptedHeight:640 + cropWidth:300 + cropHeight:640 + cropX:40 + cropY:0]; + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTCVideoFrame *outputFrame) { + XCTAssertEqual(outputFrame.width, 300); + XCTAssertEqual(outputFrame.height, 533); + + RTCCVPixelBuffer *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 40); + XCTAssertEqual(outputBuffer.cropY, 52); + XCTAssertEqual(outputBuffer.cropWidth, 300); + XCTAssertEqual(outputBuffer.cropHeight, 533); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameI420BufferNeedsAdaptation { + rtc::scoped_refptr i420Buffer = CreateI420Gradient(720, 1280); + RTCI420Buffer *buffer = [[RTCI420Buffer alloc] initWithFrameBuffer:i420Buffer]; + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTCVideoFrame *outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTCI420Buffer *outputBuffer = (RTCI420Buffer *)outputFrame.buffer; + + double psnr = I420PSNR(*[buffer nativeI420Buffer], *[outputBuffer nativeI420Buffer]); + XCTAssertEqual(psnr, webrtc::kPerfectPSNR); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +- (void)testOnCapturedFrameI420BufferNeedsCropping { + rtc::scoped_refptr i420Buffer = CreateI420Gradient(380, 640); + RTCI420Buffer *buffer = [[RTCI420Buffer alloc] initWithFrameBuffer:i420Buffer]; + RTCVideoFrame *frame = + [[RTCVideoFrame alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTCVideoFrame *outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTCI420Buffer *outputBuffer = (RTCI420Buffer *)outputFrame.buffer; + + double psnr = I420PSNR(*[buffer nativeI420Buffer], *[outputBuffer nativeI420Buffer]); + XCTAssertGreaterThanOrEqual(psnr, 40); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface *video_source_interface = _video_source; + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +@end diff --git a/sdk/objc/Framework/UnitTests/RTCCVPixelBuffer_xctest.mm b/sdk/objc/Framework/UnitTests/RTCCVPixelBuffer_xctest.mm new file mode 100644 index 0000000000..bb81fe50d1 --- /dev/null +++ b/sdk/objc/Framework/UnitTests/RTCCVPixelBuffer_xctest.mm @@ -0,0 +1,227 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#import "Video/RTCI420Buffer+Private.h" +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#import "sdk/objc/Framework/UnitTests/frame_buffer_helpers.h" +#include "third_party/libyuv/include/libyuv.h" + +@interface RTCCVPixelBufferTests : XCTestCase +@end + +@implementation RTCCVPixelBufferTests { +} + +- (void)testRequiresCroppingNoCrop { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + + XCTAssertFalse([buffer requiresCropping]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testRequiresCroppingWithCrop { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + RTCCVPixelBuffer *croppedBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:720 + adaptedHeight:1280 + cropWidth:360 + cropHeight:640 + cropX:100 + cropY:100]; + + XCTAssertTrue([croppedBuffer requiresCropping]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testRequiresScalingNoScale { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertFalse([buffer requiresScalingToWidth:720 height:1280]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testRequiresScalingWithScale { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertTrue([buffer requiresScalingToWidth:360 height:640]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testRequiresScalingWithScaleAndMatchingCrop { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:720 + adaptedHeight:1280 + cropWidth:360 + cropHeight:640 + cropX:100 + cropY:100]; + XCTAssertFalse([buffer requiresScalingToWidth:360 height:640]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testBufferSize_NV12 { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 576000); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testBufferSize_RGB { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 720, 1280, kCVPixelFormatType_32BGRA, NULL, &pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 0); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testCropAndScale_NV12 { + [self cropAndScaleTestWithNV12]; +} + +- (void)testCropAndScale_32BGRA { + [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32BGRA]; +} + +- (void)testCropAndScale_32ARGB { + [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB]; +} + +- (void)testToI420_NV12 { + [self toI420WithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; +} + +- (void)testToI420_32BGRA { + [self toI420WithPixelFormat:kCVPixelFormatType_32BGRA]; +} + +- (void)testToI420_32ARGB { + [self toI420WithPixelFormat:kCVPixelFormatType_32ARGB]; +} + +#pragma mark - Shared test code + +- (void)cropAndScaleTestWithNV12 { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + rtc::scoped_refptr i420Buffer = CreateI420Gradient(720, 1280); + CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertEqual(buffer.width, 720); + XCTAssertEqual(buffer.height, 1280); + + CVPixelBufferRef outputPixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 360, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &outputPixelBufferRef); + + std::vector frameScaleBuffer; + int size = [buffer bufferSizeForCroppingAndScalingToWidth:360 height:640]; + frameScaleBuffer.resize(size); + + [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:frameScaleBuffer.data()]; + + RTCCVPixelBuffer *scaledBuffer = + [[RTCCVPixelBuffer alloc] initWithPixelBuffer:outputPixelBufferRef]; + XCTAssertEqual(scaledBuffer.width, 360); + XCTAssertEqual(scaledBuffer.height, 640); + + RTCI420Buffer *originalBufferI420 = [buffer toI420]; + RTCI420Buffer *scaledBufferI420 = [scaledBuffer toI420]; + double psnr = + I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]); + XCTAssertEqual(psnr, webrtc::kPerfectPSNR); + + CVBufferRelease(pixelBufferRef); +} + +- (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 720, 1280, pixelFormat, NULL, &pixelBufferRef); + + DrawGradientInRGBPixelBuffer(pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertEqual(buffer.width, 720); + XCTAssertEqual(buffer.height, 1280); + + CVPixelBufferRef outputPixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &outputPixelBufferRef); + [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:NULL]; + + RTCCVPixelBuffer *scaledBuffer = + [[RTCCVPixelBuffer alloc] initWithPixelBuffer:outputPixelBufferRef]; + XCTAssertEqual(scaledBuffer.width, 360); + XCTAssertEqual(scaledBuffer.height, 640); + + RTCI420Buffer *originalBufferI420 = [buffer toI420]; + RTCI420Buffer *scaledBufferI420 = [scaledBuffer toI420]; + double psnr = + I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]); + XCTAssertEqual(psnr, webrtc::kPerfectPSNR); + + CVBufferRelease(pixelBufferRef); +} + +- (void)toI420WithPixelFormat:(OSType)pixelFormat { + rtc::scoped_refptr i420Buffer = CreateI420Gradient(360, 640); + + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &pixelBufferRef); + + CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); + + RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBufferRef]; + RTCI420Buffer *fromCVPixelBuffer = [buffer toI420]; + + double psnr = I420PSNR(*i420Buffer, *[fromCVPixelBuffer nativeI420Buffer]); + double target = webrtc::kPerfectPSNR; + if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { + // libyuv's I420ToRGB functions seem to lose some quality. + target = 19.0; + } + XCTAssertGreaterThanOrEqual(psnr, target); + + CVBufferRelease(pixelBufferRef); +} + +@end diff --git a/sdk/objc/Framework/UnitTests/frame_buffer_helpers.h b/sdk/objc/Framework/UnitTests/frame_buffer_helpers.h new file mode 100644 index 0000000000..76c0d15c7e --- /dev/null +++ b/sdk/objc/Framework/UnitTests/frame_buffer_helpers.h @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#include "api/video/i420_buffer.h" + +void DrawGradientInRGBPixelBuffer(CVPixelBufferRef pixelBuffer); + +rtc::scoped_refptr CreateI420Gradient(int width, + int height); + +void CopyI420BufferToCVPixelBuffer( + rtc::scoped_refptr i420Buffer, + CVPixelBufferRef pixelBuffer); diff --git a/sdk/objc/Framework/UnitTests/frame_buffer_helpers.mm b/sdk/objc/Framework/UnitTests/frame_buffer_helpers.mm new file mode 100644 index 0000000000..678732c5fe --- /dev/null +++ b/sdk/objc/Framework/UnitTests/frame_buffer_helpers.mm @@ -0,0 +1,122 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/Framework/UnitTests/frame_buffer_helpers.h" + +#include "third_party/libyuv/include/libyuv.h" + +void DrawGradientInRGBPixelBuffer(CVPixelBufferRef pixelBuffer) { + CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + void* baseAddr = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t width = CVPixelBufferGetWidth(pixelBuffer); + size_t height = CVPixelBufferGetHeight(pixelBuffer); + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef cgContext = CGBitmapContextCreate(baseAddr, + width, + height, + 8, + CVPixelBufferGetBytesPerRow(pixelBuffer), + colorSpace, + kCGImageAlphaNoneSkipLast); + + // Create a gradient + CGFloat colors[] = { + 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, + }; + CGGradientRef gradient = CGGradientCreateWithColorComponents(colorSpace, colors, NULL, 4); + + CGContextDrawLinearGradient( + cgContext, gradient, CGPointMake(0, 0), CGPointMake(width, height), 0); + CGGradientRelease(gradient); + + CGImageRef cgImage = CGBitmapContextCreateImage(cgContext); + CGContextRelease(cgContext); + CGImageRelease(cgImage); + CGColorSpaceRelease(colorSpace); + + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); +} + +rtc::scoped_refptr CreateI420Gradient(int width, int height) { + rtc::scoped_refptr buffer(webrtc::I420Buffer::Create(width, height)); + // Initialize with gradient, Y = 128(x/w + y/h), U = 256 x/w, V = 256 y/h + for (int x = 0; x < width; x++) { + for (int y = 0; y < height; y++) { + buffer->MutableDataY()[x + y * width] = 128 * (x * height + y * width) / (width * height); + } + } + int chroma_width = buffer->ChromaWidth(); + int chroma_height = buffer->ChromaHeight(); + for (int x = 0; x < chroma_width; x++) { + for (int y = 0; y < chroma_height; y++) { + buffer->MutableDataU()[x + y * chroma_width] = 255 * x / (chroma_width - 1); + buffer->MutableDataV()[x + y * chroma_width] = 255 * y / (chroma_height - 1); + } + } + return buffer; +} + +void CopyI420BufferToCVPixelBuffer(rtc::scoped_refptr i420Buffer, + CVPixelBufferRef pixelBuffer) { + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + + const OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { + // NV12 + uint8_t* dstY = static_cast(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)); + const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); + uint8_t* dstUV = static_cast(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)); + const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); + + libyuv::I420ToNV12(i420Buffer->DataY(), + i420Buffer->StrideY(), + i420Buffer->DataU(), + i420Buffer->StrideU(), + i420Buffer->DataV(), + i420Buffer->StrideV(), + dstY, + dstYStride, + dstUV, + dstUVStride, + i420Buffer->width(), + i420Buffer->height()); + } else { + uint8_t* dst = static_cast(CVPixelBufferGetBaseAddress(pixelBuffer)); + const int bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); + + if (pixelFormat == kCVPixelFormatType_32BGRA) { + // Corresponds to libyuv::FOURCC_ARGB + libyuv::I420ToARGB(i420Buffer->DataY(), + i420Buffer->StrideY(), + i420Buffer->DataU(), + i420Buffer->StrideU(), + i420Buffer->DataV(), + i420Buffer->StrideV(), + dst, + bytesPerRow, + i420Buffer->width(), + i420Buffer->height()); + } else if (pixelFormat == kCVPixelFormatType_32ARGB) { + // Corresponds to libyuv::FOURCC_BGRA + libyuv::I420ToBGRA(i420Buffer->DataY(), + i420Buffer->StrideY(), + i420Buffer->DataU(), + i420Buffer->StrideU(), + i420Buffer->DataV(), + i420Buffer->StrideV(), + dst, + bytesPerRow, + i420Buffer->width(), + i420Buffer->height()); + } + } + + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); +}