From 81774526986d350d751fbdef72d7e50da134e56d Mon Sep 17 00:00:00 2001 From: magjed Date: Sat, 20 Aug 2016 10:53:26 -0700 Subject: [PATCH] iOS H264VideoToolBoxEncoder: Stop scaling native CVPixelBuffers If the input to H264VideoToolBoxEncoder is a native CVPixelBuffer and the quality scaler requests scaling, we fall back to a slow path where the buffer is converted from NV12 to I420 on the CPU and then uploaded to a native CVPixelBuffer again. It turns out this scaling is not needed and that the H264VideoToolBoxEncoder can handle the scaling internally. BUG=b/30939444 Review-Url: https://codereview.webrtc.org/2258103003 Cr-Commit-Position: refs/heads/master@{#13835} --- .../codecs/h264/h264_video_toolbox_encoder.mm | 73 +++++++------------ 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm index 43c3fe575b..2d1e2a1416 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm @@ -155,12 +155,12 @@ bool CopyVideoFrameToPixelBuffer( const rtc::scoped_refptr& frame, CVPixelBufferRef pixel_buffer) { RTC_DCHECK(pixel_buffer); - RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) == - kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); - RTC_DCHECK(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0) == - static_cast(frame->height())); - RTC_DCHECK(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0) == - static_cast(frame->width())); + RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer), + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); + RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0), + static_cast(frame->height())); + RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0), + static_cast(frame->width())); CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0); if (cvRet != kCVReturnSuccess) { @@ -218,13 +218,7 @@ namespace webrtc { H264VideoToolboxEncoder::H264VideoToolboxEncoder() : callback_(nullptr), compression_session_(nullptr), - bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95), - enable_scaling_(true) { -#if defined(WEBRTC_IOS) - if ([UIDevice deviceType] == RTCDeviceTypeIPhone4S) { - enable_scaling_ = false; - } -#endif + bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95) { } H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { @@ -236,8 +230,6 @@ int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, size_t max_payload_size) { RTC_DCHECK(codec_settings); RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); - width_ = codec_settings->width; - height_ = codec_settings->height; { rtc::CritScope lock(&quality_scaler_crit_); quality_scaler_.Init(QualityScaler::kLowH264QpThreshold, @@ -247,10 +239,8 @@ int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); // TODO(tkchin): We may need to enforce width/height dimension restrictions // to match what the encoder supports. - if (enable_scaling_) { - width_ = res.width; - height_ = res.height; - } + width_ = res.width; + height_ = res.height; } // We can only set average bitrate on the HW encoder. target_bitrate_bps_ = codec_settings->startBitrate; @@ -262,24 +252,6 @@ int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, return ResetCompressionSession(); } -rtc::scoped_refptr -H264VideoToolboxEncoder::GetScaledBufferOnEncode( - const rtc::scoped_refptr& frame) { - rtc::CritScope lock(&quality_scaler_crit_); - quality_scaler_.OnEncodeFrame(frame->width(), frame->height()); - if (!frame->native_handle()) - return quality_scaler_.GetScaledBuffer(frame); - - // Handle native (CVImageRef) scaling. - const QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); - if (!enable_scaling_ || - (res.width == frame->width() && res.height == frame->height())) - return frame; - // TODO(magjed): Implement efficient CVImageRef -> CVImageRef scaling instead - // of doing it via I420. - return quality_scaler_.GetScaledBuffer(frame->NativeToI420Buffer()); -} - int H264VideoToolboxEncoder::Encode( const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, @@ -296,12 +268,14 @@ int H264VideoToolboxEncoder::Encode( } #endif bool is_keyframe_required = false; - rtc::scoped_refptr input_image( - GetScaledBufferOnEncode(frame.video_frame_buffer())); - if (input_image->width() != width_ || input_image->height() != height_) { - width_ = input_image->width(); - height_ = input_image->height(); + quality_scaler_.OnEncodeFrame(frame.width(), frame.height()); + const QualityScaler::Resolution scaled_res = + quality_scaler_.GetScaledResolution(); + + if (scaled_res.width != width_ || scaled_res.height != height_) { + width_ = scaled_res.width; + height_ = scaled_res.height; int ret = ResetCompressionSession(); if (ret < 0) return ret; @@ -325,9 +299,13 @@ int H264VideoToolboxEncoder::Encode( } #endif - CVPixelBufferRef pixel_buffer = - static_cast(input_image->native_handle()); + CVPixelBufferRef pixel_buffer = static_cast( + frame.video_frame_buffer()->native_handle()); if (pixel_buffer) { + // This pixel buffer might have a higher resolution than what the + // compression session is configured to. The compression session can handle + // that and will output encoded frames in the configured resolution + // regardless of the input pixel buffer resolution. CVBufferRetain(pixel_buffer); pixel_buffer_pool = nullptr; } else { @@ -344,7 +322,12 @@ int H264VideoToolboxEncoder::Encode( return WEBRTC_VIDEO_CODEC_ERROR; } RTC_DCHECK(pixel_buffer); - if (!internal::CopyVideoFrameToPixelBuffer(input_image, pixel_buffer)) { + // TODO(magjed): Optimize by merging scaling and NV12 pixel buffer + // conversion once libyuv::MergeUVPlanes is available. + rtc::scoped_refptr scaled_i420_buffer = + quality_scaler_.GetScaledBuffer(frame.video_frame_buffer()); + if (!internal::CopyVideoFrameToPixelBuffer(scaled_i420_buffer, + pixel_buffer)) { LOG(LS_ERROR) << "Failed to copy frame data."; CVBufferRelease(pixel_buffer); return WEBRTC_VIDEO_CODEC_ERROR;