diff --git a/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm b/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm index f82f206e91..7aafd98f43 100644 --- a/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm +++ b/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm @@ -99,6 +99,22 @@ return _i420Buffer->DataV(); } +- (id)cropAndScaleWith:(int)offsetX + offsetY:(int)offsetY + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + scaleWidth:(int)scaleWidth + scaleHeight:(int)scaleHeight { + rtc::scoped_refptr scaled_buffer = + _i420Buffer->CropAndScale(offsetX, offsetY, cropWidth, cropHeight, scaleWidth, scaleHeight); + RTC_DCHECK_EQ(scaled_buffer->type(), webrtc::VideoFrameBuffer::Type::kI420); + // Calling ToI420() doesn't do any conversions. + rtc::scoped_refptr buffer = scaled_buffer->ToI420(); + RTC_OBJC_TYPE(RTCI420Buffer) *result = + [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:buffer]; + return result; +} + - (id)toI420 { return self; } diff --git a/sdk/objc/base/RTCVideoFrameBuffer.h b/sdk/objc/base/RTCVideoFrameBuffer.h index 82d057eea0..a36807951b 100644 --- a/sdk/objc/base/RTCVideoFrameBuffer.h +++ b/sdk/objc/base/RTCVideoFrameBuffer.h @@ -27,6 +27,14 @@ RTC_OBJC_EXPORT - (id)toI420; +@optional +- (id)cropAndScaleWith:(int)offsetX + offsetY:(int)offsetY + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + scaleWidth:(int)scaleWidth + scaleHeight:(int)scaleHeight; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm b/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm index 8e39eddc91..1a9b672d1a 100644 --- a/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm +++ b/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm @@ -153,6 +153,21 @@ return YES; } +- (id)cropAndScaleWith:(int)offsetX + offsetY:(int)offsetY + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + scaleWidth:(int)scaleWidth + scaleHeight:(int)scaleHeight { + return [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] + initWithPixelBuffer:_pixelBuffer + adaptedWidth:scaleWidth + adaptedHeight:scaleHeight + cropWidth:cropWidth * _cropWidth / _width + cropHeight:cropHeight * _cropHeight / _height + cropX:_cropX + offsetX * _cropWidth / _width + cropY:_cropY + offsetY * _cropHeight / _height]; +} - (id)toI420 { const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); diff --git a/sdk/objc/native/src/objc_frame_buffer.h b/sdk/objc/native/src/objc_frame_buffer.h index 9c1ff17876..944690c8bc 100644 --- a/sdk/objc/native/src/objc_frame_buffer.h +++ b/sdk/objc/native/src/objc_frame_buffer.h @@ -33,6 +33,12 @@ class ObjCFrameBuffer : public VideoFrameBuffer { int height() const override; rtc::scoped_refptr ToI420() override; + rtc::scoped_refptr CropAndScale(int offset_x, + int offset_y, + int crop_width, + int crop_height, + int scaled_width, + int scaled_height) override; id wrapped_frame_buffer() const; diff --git a/sdk/objc/native/src/objc_frame_buffer.mm b/sdk/objc/native/src/objc_frame_buffer.mm index 566733d692..00e4b4be85 100644 --- a/sdk/objc/native/src/objc_frame_buffer.mm +++ b/sdk/objc/native/src/objc_frame_buffer.mm @@ -70,6 +70,27 @@ rtc::scoped_refptr ObjCFrameBuffer::ToI420() { return rtc::make_ref_counted([frame_buffer_ toI420]); } +rtc::scoped_refptr ObjCFrameBuffer::CropAndScale(int offset_x, + int offset_y, + int crop_width, + int crop_height, + int scaled_width, + int scaled_height) { + if ([frame_buffer_ respondsToSelector:@selector + (cropAndScaleWith:offsetY:cropWidth:cropHeight:scaleWidth:scaleHeight:)]) { + return rtc::make_ref_counted([frame_buffer_ cropAndScaleWith:offset_x + offsetY:offset_y + cropWidth:crop_width + cropHeight:crop_height + scaleWidth:scaled_width + scaleHeight:scaled_height]); + } + + // Use the default implementation. + return VideoFrameBuffer::CropAndScale( + offset_x, offset_y, crop_width, crop_height, scaled_width, scaled_height); +} + id ObjCFrameBuffer::wrapped_frame_buffer() const { return frame_buffer_; } diff --git a/sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm b/sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm index 3a1ab24773..cf759c5243 100644 --- a/sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm +++ b/sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm @@ -21,6 +21,44 @@ #include "common_video/libyuv/include/webrtc_libyuv.h" #include "third_party/libyuv/include/libyuv.h" +namespace { + +struct ToI420WithCropAndScaleSetting { + int inputWidth; + int inputHeight; + int offsetX; + int offsetY; + int cropWidth; + int cropHeight; + int scaleWidth; + int scaleHeight; +}; + +constexpr const ToI420WithCropAndScaleSetting kToI420WithCropAndScaleSettings[] = { + ToI420WithCropAndScaleSetting{ + .inputWidth = 640, + .inputHeight = 360, + .offsetX = 0, + .offsetY = 0, + .cropWidth = 640, + .cropHeight = 360, + .scaleWidth = 320, + .scaleHeight = 180, + }, + ToI420WithCropAndScaleSetting{ + .inputWidth = 640, + .inputHeight = 360, + .offsetX = 160, + .offsetY = 90, + .cropWidth = 160, + .cropHeight = 90, + .scaleWidth = 320, + .scaleHeight = 180, + }, +}; + +} // namespace + @interface RTCCVPixelBufferTests : XCTestCase @end @@ -183,6 +221,76 @@ [self toI420WithPixelFormat:kCVPixelFormatType_32ARGB]; } +- (void)testToI420WithCropAndScale_NV12 { + for (const auto &setting : kToI420WithCropAndScaleSettings) { + [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + setting:setting]; + } +} + +- (void)testToI420WithCropAndScale_32BGRA { + for (const auto &setting : kToI420WithCropAndScaleSettings) { + [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32BGRA setting:setting]; + } +} + +- (void)testToI420WithCropAndScale_32ARGB { + for (const auto &setting : kToI420WithCropAndScaleSettings) { + [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32ARGB setting:setting]; + } +} + +- (void)testScaleBufferTest { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 1920, 1080, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + rtc::scoped_refptr i420Buffer = CreateI420Gradient(1920, 1080); + CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + + XCTAssertEqual(buffer.width, 1920); + XCTAssertEqual(buffer.height, 1080); + XCTAssertEqual(buffer.cropX, 0); + XCTAssertEqual(buffer.cropY, 0); + XCTAssertEqual(buffer.cropWidth, 1920); + XCTAssertEqual(buffer.cropHeight, 1080); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer2 = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer cropAndScaleWith:320 + offsetY:180 + cropWidth:1280 + cropHeight:720 + scaleWidth:960 + scaleHeight:540]; + + XCTAssertEqual(buffer2.width, 960); + XCTAssertEqual(buffer2.height, 540); + XCTAssertEqual(buffer2.cropX, 320); + XCTAssertEqual(buffer2.cropY, 180); + XCTAssertEqual(buffer2.cropWidth, 1280); + XCTAssertEqual(buffer2.cropHeight, 720); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer3 = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer2 cropAndScaleWith:240 + offsetY:135 + cropWidth:480 + cropHeight:270 + scaleWidth:320 + scaleHeight:180]; + + XCTAssertEqual(buffer3.width, 320); + XCTAssertEqual(buffer3.height, 180); + XCTAssertEqual(buffer3.cropX, 640); + XCTAssertEqual(buffer3.cropY, 360); + XCTAssertEqual(buffer3.cropWidth, 640); + XCTAssertEqual(buffer3.cropHeight, 360); + + CVBufferRelease(pixelBufferRef); +} + #pragma mark - Shared test code - (void)cropAndScaleTestWithNV12 { @@ -305,4 +413,49 @@ CVBufferRelease(pixelBufferRef); } +- (void)toI420WithCropAndScaleWithPixelFormat:(OSType)pixelFormat + setting:(const ToI420WithCropAndScaleSetting &)setting { + rtc::scoped_refptr i420Buffer = + CreateI420Gradient(setting.inputWidth, setting.inputHeight); + + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, setting.inputWidth, setting.inputHeight, pixelFormat, NULL, &pixelBufferRef); + + CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); + + RTC_OBJC_TYPE(RTCI420Buffer) *objcI420Buffer = + [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer]; + RTC_OBJC_TYPE(RTCI420Buffer) *scaledObjcI420Buffer = + (RTC_OBJC_TYPE(RTCI420Buffer) *)[objcI420Buffer cropAndScaleWith:setting.offsetX + offsetY:setting.offsetY + cropWidth:setting.cropWidth + cropHeight:setting.cropHeight + scaleWidth:setting.scaleWidth + scaleHeight:setting.scaleHeight]; + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + id scaledBuffer = + [buffer cropAndScaleWith:setting.offsetX + offsetY:setting.offsetY + cropWidth:setting.cropWidth + cropHeight:setting.cropHeight + scaleWidth:setting.scaleWidth + scaleHeight:setting.scaleHeight]; + XCTAssertTrue([scaledBuffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]); + + RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [scaledBuffer toI420]; + + double psnr = + I420PSNR(*[scaledObjcI420Buffer nativeI420Buffer], *[fromCVPixelBuffer nativeI420Buffer]); + double target = webrtc::kPerfectPSNR; + if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { + // libyuv's I420ToRGB functions seem to lose some quality. + target = 19.0; + } + XCTAssertGreaterThanOrEqual(psnr, target); + + CVBufferRelease(pixelBufferRef); +} + @end