From abd9e0f5c2ad547a255b3f05ab43f4d08d843330 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Wed, 25 Sep 2019 16:05:47 +0200 Subject: [PATCH] VideoStreamEncoder: Don't crop and scales frames for external encoders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: none Change-Id: I1edc0314450793c0ae3fb471aa8936a102773a96 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/154424 Reviewed-by: Erik Språng Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#29319} --- video/video_stream_encoder.cc | 120 ++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 55 deletions(-) diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index b22f326d6a..e4b244a52c 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1365,9 +1365,65 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, TraceFrameDropEnd(); + // Encoder metadata needs to be updated before encode complete callback. + VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo(); + if (info.implementation_name != encoder_info_.implementation_name) { + encoder_stats_observer_->OnEncoderImplementationChanged( + info.implementation_name); + if (bitrate_adjuster_) { + // Encoder implementation changed, reset overshoot detector states. + bitrate_adjuster_->Reset(); + } + } + + if (bitrate_adjuster_) { + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + if (info.fps_allocation[si] != encoder_info_.fps_allocation[si]) { + bitrate_adjuster_->OnEncoderInfo(info); + break; + } + } + } + encoder_info_ = info; + last_encode_info_ms_ = clock_->TimeInMilliseconds(); + VideoFrame out_frame(video_frame); + + const VideoFrameBuffer::Type buffer_type = + out_frame.video_frame_buffer()->type(); + const bool is_buffer_type_supported = + buffer_type == VideoFrameBuffer::Type::kI420 || + (buffer_type == VideoFrameBuffer::Type::kNative && + info.supports_native_handle); + + if (!is_buffer_type_supported) { + // This module only supports software encoding. + rtc::scoped_refptr converted_buffer( + out_frame.video_frame_buffer()->ToI420()); + + if (!converted_buffer) { + RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame."; + return; + } + + VideoFrame::UpdateRect update_rect = out_frame.update_rect(); + if (!update_rect.IsEmpty() && + out_frame.video_frame_buffer()->GetI420() == nullptr) { + // UpdatedRect is reset to full update if it's not empty, and buffer was + // converted, therefore we can't guarantee that pixels outside of + // UpdateRect didn't change comparing to the previous frame. + update_rect = + VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()}; + } + + out_frame.set_video_frame_buffer(converted_buffer); + out_frame.set_update_rect(update_rect); + } + // Crop frame if needed. - if (crop_width_ > 0 || crop_height_ > 0) { + if ((crop_width_ > 0 || crop_height_ > 0) && + out_frame.video_frame_buffer()->type() != + VideoFrameBuffer::Type::kNative) { // If the frame can't be converted to I420, drop it. auto i420_buffer = video_frame.video_frame_buffer()->ToI420(); if (!i420_buffer) { @@ -1424,60 +1480,14 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, overuse_detector_->FrameCaptured(out_frame, time_when_posted_us); - // Encoder metadata needs to be updated before encode complete callback. - VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo(); - if (info.implementation_name != encoder_info_.implementation_name) { - encoder_stats_observer_->OnEncoderImplementationChanged( - info.implementation_name); - if (bitrate_adjuster_) { - // Encoder implementation changed, reset overshoot detector states. - bitrate_adjuster_->Reset(); - } - } - - if (bitrate_adjuster_) { - for (size_t si = 0; si < kMaxSpatialLayers; ++si) { - if (info.fps_allocation[si] != encoder_info_.fps_allocation[si]) { - bitrate_adjuster_->OnEncoderInfo(info); - break; - } - } - } - - encoder_info_ = info; - last_encode_info_ms_ = clock_->TimeInMilliseconds(); - RTC_DCHECK_EQ(send_codec_.width, out_frame.width()); - RTC_DCHECK_EQ(send_codec_.height, out_frame.height()); - const VideoFrameBuffer::Type buffer_type = - out_frame.video_frame_buffer()->type(); - const bool is_buffer_type_supported = - buffer_type == VideoFrameBuffer::Type::kI420 || - (buffer_type == VideoFrameBuffer::Type::kNative && - info.supports_native_handle); - - if (!is_buffer_type_supported) { - // This module only supports software encoding. - rtc::scoped_refptr converted_buffer( - out_frame.video_frame_buffer()->ToI420()); - - if (!converted_buffer) { - RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame."; - return; - } - - VideoFrame::UpdateRect update_rect = out_frame.update_rect(); - if (!update_rect.IsEmpty() && - out_frame.video_frame_buffer()->GetI420() == nullptr) { - // UpdatedRect is reset to full update if it's not empty, and buffer was - // converted, therefore we can't guarantee that pixels outside of - // UpdateRect didn't change comparing to the previous frame. - update_rect = - VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()}; - } - - out_frame.set_video_frame_buffer(converted_buffer); - out_frame.set_update_rect(update_rect); - } + RTC_DCHECK_LE(send_codec_.width, out_frame.width()); + RTC_DCHECK_LE(send_codec_.height, out_frame.height()); + // Native frames should be scaled by the client. + // For internal encoders we scale everything in one place here. + RTC_DCHECK((out_frame.video_frame_buffer()->type() == + VideoFrameBuffer::Type::kNative) || + (send_codec_.width == out_frame.width() && + send_codec_.height == out_frame.height())); TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp", out_frame.timestamp());