From a3002db8d63aecd2a3ccf630afef9a868e7e1f78 Mon Sep 17 00:00:00 2001 From: Magnus Jedvert Date: Fri, 13 May 2016 12:51:04 +0200 Subject: [PATCH] Android: Add support for cropping textures BUG=b/28622232 R=glaznev@webrtc.org, nisse@webrtc.org Review URL: https://codereview.webrtc.org/1965953003 . Cr-Commit-Position: refs/heads/master@{#12720} --- webrtc/api/androidvideocapturer.cc | 10 ++-- .../api/java/jni/androidmediaencoder_jni.cc | 6 +-- webrtc/api/java/jni/native_handle_impl.cc | 51 ++++++++++++++++--- webrtc/api/java/jni/native_handle_impl.h | 7 ++- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/webrtc/api/androidvideocapturer.cc b/webrtc/api/androidvideocapturer.cc index ee5ef665d9..71a94fedb0 100644 --- a/webrtc/api/androidvideocapturer.cc +++ b/webrtc/api/androidvideocapturer.cc @@ -88,14 +88,12 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory { int output_width, int output_height) const override { if (buffer_->native_handle() != nullptr) { - // TODO(perkj) Implement cropping. - RTC_CHECK_EQ(cropped_input_width, buffer_->width()); - RTC_CHECK_EQ(cropped_input_height, buffer_->height()); rtc::scoped_refptr scaled_buffer( static_cast(buffer_.get()) - ->ScaleAndRotate(output_width, output_height, - apply_rotation_ ? input_frame->rotation : - webrtc::kVideoRotation_0)); + ->CropScaleAndRotate(cropped_input_width, cropped_input_height, + output_width, output_height, + apply_rotation_ ? input_frame->rotation + : webrtc::kVideoRotation_0)); return new cricket::WebRtcVideoFrame( scaled_buffer, input_frame->time_stamp, apply_rotation_ ? webrtc::kVideoRotation_0 : input_frame->rotation); diff --git a/webrtc/api/java/jni/androidmediaencoder_jni.cc b/webrtc/api/java/jni/androidmediaencoder_jni.cc index 5a817b6e84..8e9bfa3065 100644 --- a/webrtc/api/java/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/java/jni/androidmediaencoder_jni.cc @@ -683,9 +683,9 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( if (frame.video_frame_buffer()->native_handle() != nullptr) { rtc::scoped_refptr scaled_buffer( static_cast( - frame.video_frame_buffer().get())->ScaleAndRotate( - scaled_resolution.width, - scaled_resolution.height, + frame.video_frame_buffer().get())->CropScaleAndRotate( + frame.width(), frame.height(), + scaled_resolution.width, scaled_resolution.height, webrtc::kVideoRotation_0)); input_frame.set_video_frame_buffer(scaled_buffer); } else { diff --git a/webrtc/api/java/jni/native_handle_impl.cc b/webrtc/api/java/jni/native_handle_impl.cc index 1990828fa0..1f180ade9c 100644 --- a/webrtc/api/java/jni/native_handle_impl.cc +++ b/webrtc/api/java/jni/native_handle_impl.cc @@ -59,6 +59,37 @@ void RotateMatrix(float a[16], webrtc::VideoRotation rotation) { } } +// Calculates result = a * b, in column-major order. +void MultiplyMatrix(const float a[16], const float b[16], float result[16]) { + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + float sum = 0; + for (int k = 0; k < 4; ++k) { + sum += a[k * 4 + j] * b[i * 4 + k]; + } + result[i * 4 + j] = sum; + } + } +} + +// Center crop by keeping xFraction of the width and yFraction of the height, +// so e.g. cropping from 640x480 to 640x360 would use +// xFraction=1, yFraction=360/480. +void CropMatrix(float a[16], float xFraction, float yFraction) { + // Move cropped area to the center of the frame by offsetting half the + // removed area. + const float xOffset = (1 - xFraction) / 2; + const float yOffset = (1 - yFraction) / 2; + const float crop_matrix[16] = { + xFraction, 0, 0, 0, + 0, yFraction, 0, 0, + 0, 0, 1, 0, + xOffset, yOffset, 0, 1}; + float mul_result[16]; + MultiplyMatrix(crop_matrix, a, mul_result); + memcpy(a, mul_result, sizeof(mul_result)); +} + } // anonymouse namespace namespace webrtc_jni { @@ -146,15 +177,18 @@ AndroidTextureBuffer::NativeToI420Buffer() { } rtc::scoped_refptr -AndroidTextureBuffer::ScaleAndRotate(int dst_widht, - int dst_height, - webrtc::VideoRotation rotation) { - if (width() == dst_widht && height() == dst_height && +AndroidTextureBuffer::CropScaleAndRotate(int cropped_width, + int cropped_height, + int dst_width, + int dst_height, + webrtc::VideoRotation rotation) { + if (cropped_width == dst_width && cropped_height == dst_height && + width() == dst_width && height() == dst_height && rotation == webrtc::kVideoRotation_0) { return this; } - int rotated_width = (rotation % 180 == 0) ? dst_widht : dst_height; - int rotated_height = (rotation % 180 == 0) ? dst_height : dst_widht; + int rotated_width = (rotation % 180 == 0) ? dst_width : dst_height; + int rotated_height = (rotation % 180 == 0) ? dst_height : dst_width; // Here we use Bind magic to add a reference count to |this| until the newly // created AndroidTextureBuffer is destructed @@ -163,6 +197,11 @@ AndroidTextureBuffer::ScaleAndRotate(int dst_widht, rotated_width, rotated_height, native_handle_, surface_texture_helper_, rtc::KeepRefUntilDone(this))); + if (cropped_width != width() || cropped_height != height()) { + CropMatrix(buffer->native_handle_.sampling_matrix, + cropped_width / static_cast(width()), + cropped_height / static_cast(height())); + } RotateMatrix(buffer->native_handle_.sampling_matrix, rotation); return buffer; } diff --git a/webrtc/api/java/jni/native_handle_impl.h b/webrtc/api/java/jni/native_handle_impl.h index 859c3713b4..b781815718 100644 --- a/webrtc/api/java/jni/native_handle_impl.h +++ b/webrtc/api/java/jni/native_handle_impl.h @@ -38,8 +38,11 @@ class AndroidTextureBuffer : public webrtc::NativeHandleBuffer { ~AndroidTextureBuffer(); rtc::scoped_refptr NativeToI420Buffer() override; - rtc::scoped_refptr ScaleAndRotate( - int dst_widht, + // First crop, then scale to dst resolution, and then rotate. + rtc::scoped_refptr CropScaleAndRotate( + int cropped_width, + int cropped_height, + int dst_width, int dst_height, webrtc::VideoRotation rotation);