From 182ea46facde45811faebec40ad4981fd8db56a1 Mon Sep 17 00:00:00 2001 From: "magjed@webrtc.org" Date: Fri, 23 Jan 2015 11:50:13 +0000 Subject: [PATCH] Remove frame copy in ViEExternalRendererImpl::RenderFrame Add new interface for delivering frames to ExternalRenderer. The purpose is to avoid having to extract a packed buffer from I420VideoFrame, which will cause a deep frame copy. BUG=1128 R=mflodman@webrtc.org Review URL: https://webrtc-codereview.appspot.com/36489004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@8136 4adac7df-926f-26a2-2b94-8c16560cd09d --- talk/media/webrtc/webrtcvideoengine.cc | 34 ++++++++++ talk/media/webrtc/webrtcvideoframe.cc | 23 ++++--- talk/media/webrtc/webrtcvideoframe.h | 5 ++ webrtc/video_engine/include/vie_render.h | 4 ++ .../auto_test/source/vie_autotest_render.cc | 7 +++ .../helpers/vie_to_file_renderer.cc | 62 +++++++++++++------ .../libvietest/include/vie_to_file_renderer.h | 2 + webrtc/video_engine/vie_renderer.cc | 17 +++-- 8 files changed, 119 insertions(+), 35 deletions(-) diff --git a/talk/media/webrtc/webrtcvideoengine.cc b/talk/media/webrtc/webrtcvideoengine.cc index 0c7ac04b1c..33e12e5a39 100644 --- a/talk/media/webrtc/webrtcvideoengine.cc +++ b/talk/media/webrtc/webrtcvideoengine.cc @@ -417,6 +417,40 @@ class WebRtcRenderAdapter : public webrtc::ExternalRenderer { } } + virtual int DeliverI420Frame(const webrtc::I420VideoFrame* webrtc_frame) { + rtc::CritScope cs(&crit_); + DCHECK(webrtc_frame); + if (capture_start_rtp_time_stamp_ < 0) + capture_start_rtp_time_stamp_ = webrtc_frame->timestamp(); + + const int kVideoCodecClockratekHz = cricket::kVideoCodecClockrate / 1000; + const int64 elapsed_time_ms = + (rtp_ts_wraparound_handler_.Unwrap(webrtc_frame->timestamp()) - + capture_start_rtp_time_stamp_) / + kVideoCodecClockratekHz; + if (webrtc_frame->ntp_time_ms() > 0) { + capture_start_ntp_time_ms_ = + webrtc_frame->ntp_time_ms() - elapsed_time_ms; + } + frame_rate_tracker_.Update(1); + if (!renderer_) + return 0; + + const int64 elapsed_time_ns = + elapsed_time_ms * rtc::kNumNanosecsPerMillisec; + const int64 render_time_ns = + webrtc_frame->render_time_ms() * rtc::kNumNanosecsPerMillisec; + + if (!webrtc_frame->native_handle()) { + WebRtcVideoRenderFrame cricket_frame(webrtc_frame, render_time_ns, + elapsed_time_ns); + return renderer_->RenderFrame(&cricket_frame) ? 0 : -1; + } else { + return DeliverTextureFrame(webrtc_frame->native_handle(), render_time_ns, + elapsed_time_ns); + } + } + virtual bool IsTextureSupported() { return true; } int DeliverBufferFrame(unsigned char* buffer, size_t buffer_size, diff --git a/talk/media/webrtc/webrtcvideoframe.cc b/talk/media/webrtc/webrtcvideoframe.cc index 7a548f315e..d45b58405f 100644 --- a/talk/media/webrtc/webrtcvideoframe.cc +++ b/talk/media/webrtc/webrtcvideoframe.cc @@ -361,7 +361,18 @@ void WebRtcVideoFrame::InitToEmptyBuffer(int w, int h, size_t pixel_width, WebRtcVideoRenderFrame::WebRtcVideoRenderFrame( const webrtc::I420VideoFrame* frame) - : frame_(frame) { + : frame_(frame), + // Convert millisecond render time to ns timestamp. + // Convert 90K rtp timestamp to ns timestamp. + timestamp_((frame_->timestamp() / 90) * rtc::kNumNanosecsPerMillisec), + elapsed_time_(frame_->render_time_ms() * rtc::kNumNanosecsPerMillisec) { +} + +WebRtcVideoRenderFrame::WebRtcVideoRenderFrame( + const webrtc::I420VideoFrame* frame, + int64_t timestamp, + int64_t elapsed_time) + : frame_(frame), timestamp_(timestamp), elapsed_time_(elapsed_time) { } bool WebRtcVideoRenderFrame::InitToBlack(int w, @@ -442,18 +453,16 @@ size_t WebRtcVideoRenderFrame::GetPixelHeight() const { } int64_t WebRtcVideoRenderFrame::GetElapsedTime() const { - // Convert millisecond render time to ns timestamp. - return frame_->render_time_ms() * rtc::kNumNanosecsPerMillisec; + return elapsed_time_; } int64_t WebRtcVideoRenderFrame::GetTimeStamp() const { - // Convert 90K rtp timestamp to ns timestamp. - return (frame_->timestamp() / 90) * rtc::kNumNanosecsPerMillisec; + return timestamp_; } void WebRtcVideoRenderFrame::SetElapsedTime(int64_t elapsed_time) { - UNIMPLEMENTED; + elapsed_time_ = elapsed_time; } void WebRtcVideoRenderFrame::SetTimeStamp(int64_t time_stamp) { - UNIMPLEMENTED; + timestamp_ = time_stamp; } int WebRtcVideoRenderFrame::GetRotation() const { diff --git a/talk/media/webrtc/webrtcvideoframe.h b/talk/media/webrtc/webrtcvideoframe.h index bc4622fdd1..583e335458 100644 --- a/talk/media/webrtc/webrtcvideoframe.h +++ b/talk/media/webrtc/webrtcvideoframe.h @@ -140,6 +140,9 @@ class WebRtcVideoFrame : public VideoFrame { class WebRtcVideoRenderFrame : public VideoFrame { public: explicit WebRtcVideoRenderFrame(const webrtc::I420VideoFrame* frame); + WebRtcVideoRenderFrame(const webrtc::I420VideoFrame* frame, + int64_t timestamp, + int64_t elapsed_time); virtual bool InitToBlack(int w, int h, @@ -192,6 +195,8 @@ class WebRtcVideoRenderFrame : public VideoFrame { private: const webrtc::I420VideoFrame* const frame_; + int64_t timestamp_; + int64_t elapsed_time_; }; } // namespace cricket diff --git a/webrtc/video_engine/include/vie_render.h b/webrtc/video_engine/include/vie_render.h index 5cceb723d8..eab69a2963 100644 --- a/webrtc/video_engine/include/vie_render.h +++ b/webrtc/video_engine/include/vie_render.h @@ -20,6 +20,7 @@ namespace webrtc { +class I420VideoFrame; class VideoEngine; class VideoRender; class VideoRenderCallback; @@ -47,6 +48,9 @@ class ExternalRenderer { // Handle of the underlying video frame. void* handle) = 0; + // Alternative interface for I420 frames. + virtual int DeliverI420Frame(const I420VideoFrame* webrtc_frame) = 0; + // Returns true if the renderer supports textures. DeliverFrame can be called // with NULL |buffer| and non-NULL |handle|. virtual bool IsTextureSupported() = 0; diff --git a/webrtc/video_engine/test/auto_test/source/vie_autotest_render.cc b/webrtc/video_engine/test/auto_test/source/vie_autotest_render.cc index 889e907cbb..3e5be49340 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_autotest_render.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_autotest_render.cc @@ -71,6 +71,13 @@ public: return 0; } + virtual int DeliverI420Frame(const webrtc::I420VideoFrame* webrtc_frame) { + EXPECT_TRUE(webrtc_frame); + EXPECT_EQ(webrtc_frame->width(), _width); + EXPECT_EQ(webrtc_frame->height(), _height); + return 0; + } + virtual bool IsTextureSupported() { return false; } public: diff --git a/webrtc/video_engine/test/libvietest/helpers/vie_to_file_renderer.cc b/webrtc/video_engine/test/libvietest/helpers/vie_to_file_renderer.cc index 33e88391b4..c3916fb60b 100644 --- a/webrtc/video_engine/test/libvietest/helpers/vie_to_file_renderer.cc +++ b/webrtc/video_engine/test/libvietest/helpers/vie_to_file_renderer.cc @@ -12,6 +12,7 @@ #include +#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/event_wrapper.h" #include "webrtc/system_wrappers/interface/thread_wrapper.h" @@ -19,16 +20,7 @@ namespace test { struct Frame { public: - Frame(unsigned char* buffer, - size_t buffer_size, - uint32_t timestamp, - int64_t render_time) - : buffer(new unsigned char[buffer_size]), - buffer_size(buffer_size), - timestamp(timestamp), - render_time(render_time) { - memcpy(this->buffer.get(), buffer, buffer_size); - } + Frame() : buffer(nullptr), buffer_size(0), timestamp(0), render_time(0) {} webrtc::scoped_ptr buffer; size_t buffer_size; @@ -129,19 +121,53 @@ int ViEToFileRenderer::DeliverFrame(unsigned char *buffer, webrtc::CriticalSectionScoped lock(frame_queue_cs_.get()); test::Frame* frame; if (free_frame_queue_.empty()) { - frame = new test::Frame(buffer, buffer_size, time_stamp, render_time); + frame = new test::Frame(); } else { // Reuse an already allocated frame. frame = free_frame_queue_.front(); free_frame_queue_.pop_front(); - if (frame->buffer_size < buffer_size) { - frame->buffer.reset(new unsigned char[buffer_size]); - } - memcpy(frame->buffer.get(), buffer, buffer_size); - frame->buffer_size = buffer_size; - frame->timestamp = time_stamp; - frame->render_time = render_time; } + if (frame->buffer_size < buffer_size) { + frame->buffer.reset(new unsigned char[buffer_size]); + frame->buffer_size = buffer_size; + } + memcpy(frame->buffer.get(), buffer, buffer_size); + frame->timestamp = time_stamp; + frame->render_time = render_time; + + render_queue_.push_back(frame); + // Signal that a frame is ready to be written to file. + frame_render_event_->Set(); + return 0; +} + +int ViEToFileRenderer::DeliverI420Frame( + const webrtc::I420VideoFrame* input_frame) { + assert(input_frame); + const size_t buffer_size = CalcBufferSize(webrtc::kI420, input_frame->width(), + input_frame->height()); + + webrtc::CriticalSectionScoped lock(frame_queue_cs_.get()); + test::Frame* frame; + if (free_frame_queue_.empty()) { + frame = new test::Frame(); + } else { + // Reuse an already allocated frame. + frame = free_frame_queue_.front(); + free_frame_queue_.pop_front(); + } + if (frame->buffer_size < buffer_size) { + frame->buffer.reset(new unsigned char[buffer_size]); + frame->buffer_size = buffer_size; + } + const int length = + ExtractBuffer(*input_frame, frame->buffer_size, frame->buffer.get()); + assert(static_cast(length) == buffer_size); + if (length < 0) + return -1; + frame->timestamp = input_frame->timestamp(); + frame->render_time = input_frame->render_time_ms(); + render_queue_.push_back(frame); // Signal that a frame is ready to be written to file. frame_render_event_->Set(); diff --git a/webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h b/webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h index dccf262487..76e0f48e75 100644 --- a/webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h +++ b/webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h @@ -64,6 +64,8 @@ class ViEToFileRenderer: public webrtc::ExternalRenderer { int64_t render_time, void* handle) OVERRIDE; + int DeliverI420Frame(const webrtc::I420VideoFrame* webrtc_frame) OVERRIDE; + bool IsTextureSupported() OVERRIDE; const std::string GetFullOutputPath() const; diff --git a/webrtc/video_engine/vie_renderer.cc b/webrtc/video_engine/vie_renderer.cc index 5a2b7e28d2..976cef6730 100644 --- a/webrtc/video_engine/vie_renderer.cc +++ b/webrtc/video_engine/vie_renderer.cc @@ -189,6 +189,13 @@ int32_t ViEExternalRendererImpl::RenderFrame( return 0; } + // Fast path for I420 without frame copy. + if (external_renderer_format_ == kVideoI420) { + NotifyFrameSizeChange(stream_id, video_frame); + external_renderer_->DeliverI420Frame(&video_frame); + return 0; + } + VideoFrame* out_frame = converted_frame_.get(); // Convert to requested format. @@ -204,16 +211,6 @@ int32_t ViEExternalRendererImpl::RenderFrame( converted_frame_->VerifyAndAllocate(buffer_size); switch (external_renderer_format_) { - case kVideoI420: { - // TODO(mikhal): need to copy the buffer as is. - // can the output here be a I420 frame? - int length = ExtractBuffer(video_frame, out_frame->Size(), - out_frame->Buffer()); - if (length < 0) - return -1; - out_frame->SetLength(length); - break; - } case kVideoYV12: case kVideoYUY2: case kVideoUYVY: