From 712338eed24a8d68699b331acad12edf340ac929 Mon Sep 17 00:00:00 2001 From: magjed Date: Thu, 11 May 2017 05:11:57 -0700 Subject: [PATCH] Add support for I444 in VideoFrameBuffer VideoFrameBuffer is currently hard coded to be either I420 or Native. This CL makes VideoFrameBuffer more generic by moving the I420 specific functions into their own class, and adds an enum tag that represents the format and storage type of the buffer. Each buffer type is then represented as a subclass. See webrtc/api/video/video_frame_buffer.h for more info. This CL also adds support for representing I444 in VideoFrameBuffer using the new interface. Possible future buffer type candidates are RGB and NV12. BUG=webrtc:7632 TBR=stefan@webrtc.org Review-Url: https://codereview.webrtc.org/2847383002 Cr-Commit-Position: refs/heads/master@{#18098} --- webrtc/api/BUILD.gn | 1 + webrtc/api/video/i420_buffer.cc | 12 +- webrtc/api/video/i420_buffer.h | 6 +- webrtc/api/video/video_frame_buffer.cc | 153 ++++++++++++++++++ webrtc/api/video/video_frame_buffer.h | 88 ++++++++-- .../common_video/include/video_frame_buffer.h | 9 +- webrtc/common_video/video_frame_buffer.cc | 17 +- 7 files changed, 247 insertions(+), 39 deletions(-) create mode 100644 webrtc/api/video/video_frame_buffer.cc diff --git a/webrtc/api/BUILD.gn b/webrtc/api/BUILD.gn index 2b1ca34576..d47483b81e 100644 --- a/webrtc/api/BUILD.gn +++ b/webrtc/api/BUILD.gn @@ -170,6 +170,7 @@ rtc_source_set("video_frame_api") { "video/i420_buffer.h", "video/video_frame.cc", "video/video_frame.h", + "video/video_frame_buffer.cc", "video/video_frame_buffer.h", "video/video_rotation.h", ] diff --git a/webrtc/api/video/i420_buffer.cc b/webrtc/api/video/i420_buffer.cc index 32c73eac67..51d83c9df5 100644 --- a/webrtc/api/video/i420_buffer.cc +++ b/webrtc/api/video/i420_buffer.cc @@ -136,6 +136,10 @@ void I420Buffer::InitializeData() { I420DataSize(height_, stride_y_, stride_u_, stride_v_)); } +VideoFrameBuffer::Type I420Buffer::type() const { + return Type::kI420; +} + int I420Buffer::width() const { return width_; } @@ -164,14 +168,6 @@ int I420Buffer::StrideV() const { return stride_v_; } -void* I420Buffer::native_handle() const { - return nullptr; -} - -rtc::scoped_refptr I420Buffer::NativeToI420Buffer() { - return this; -} - uint8_t* I420Buffer::MutableDataY() { return const_cast(DataY()); } diff --git a/webrtc/api/video/i420_buffer.h b/webrtc/api/video/i420_buffer.h index 507bd0571b..20a84a1a53 100644 --- a/webrtc/api/video/i420_buffer.h +++ b/webrtc/api/video/i420_buffer.h @@ -20,7 +20,7 @@ namespace webrtc { // Plain I420 buffer in standard memory. -class I420Buffer : public VideoFrameBuffer { +class I420Buffer : public PlanarYuvBuffer { public: static rtc::scoped_refptr Create(int width, int height); static rtc::scoped_refptr Create(int width, @@ -53,6 +53,7 @@ class I420Buffer : public VideoFrameBuffer { // are resolved in a better way. Or in the mean time, use SetBlack. void InitializeData(); + Type type() const override; int width() const override; int height() const override; const uint8_t* DataY() const override; @@ -63,9 +64,6 @@ class I420Buffer : public VideoFrameBuffer { int StrideU() const override; int StrideV() const override; - void* native_handle() const override; - rtc::scoped_refptr NativeToI420Buffer() override; - uint8_t* MutableDataY(); uint8_t* MutableDataU(); uint8_t* MutableDataV(); diff --git a/webrtc/api/video/video_frame_buffer.cc b/webrtc/api/video/video_frame_buffer.cc new file mode 100644 index 0000000000..b4b30a1268 --- /dev/null +++ b/webrtc/api/video/video_frame_buffer.cc @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/api/video/video_frame_buffer.h" + +#include "libyuv/convert_from.h" +#include "webrtc/api/video/i420_buffer.h" +#include "webrtc/base/checks.h" + +namespace webrtc { + +namespace { + +// TODO(magjed): Remove this class. It is only used for providing a default +// implementation of ToI420() until external clients are updated. ToI420() will +// then be made pure virtual. This adapter adapts a VideoFrameBuffer (which is +// expected to be in I420 format) to the PlanarYuvBuffer interface. The reason +// this is needed is because of the return type mismatch in NativeToI420Buffer +// (returns VideoFrameBuffer) vs ToI420 (returns PlanarYuvBuffer). +class PlanarYuvBufferAdapter : public PlanarYuvBuffer { + public: + explicit PlanarYuvBufferAdapter(rtc::scoped_refptr buffer) + : buffer_(buffer) {} + + Type type() const override { return Type::kI420; } + + int width() const override { return buffer_->width(); } + int height() const override { return buffer_->height(); } + + const uint8_t* DataY() const override { return buffer_->DataY(); } + const uint8_t* DataU() const override { return buffer_->DataU(); } + const uint8_t* DataV() const override { return buffer_->DataV(); } + + int StrideY() const override { return buffer_->StrideY(); } + int StrideU() const override { return buffer_->StrideU(); } + int StrideV() const override { return buffer_->StrideV(); } + + private: + rtc::scoped_refptr buffer_; +}; + +} // namespace + +// TODO(magjed): The default implementations in VideoFrameBuffer are provided in +// order to support the deprecated interface until external clients are updated. +// Remove once done. +VideoFrameBuffer::Type VideoFrameBuffer::type() const { + return native_handle() ? Type::kNative : Type::kI420; +} + +const uint8_t* VideoFrameBuffer::DataY() const { + return const_cast(this)->GetI420()->DataY(); +} + +const uint8_t* VideoFrameBuffer::DataU() const { + return const_cast(this)->GetI420()->DataU(); +} + +const uint8_t* VideoFrameBuffer::DataV() const { + return const_cast(this)->GetI420()->DataV(); +} + +// Returns the number of bytes between successive rows for a given plane. +int VideoFrameBuffer::StrideY() const { + return const_cast(this)->GetI420()->StrideY(); +} + +int VideoFrameBuffer::StrideU() const { + return const_cast(this)->GetI420()->StrideU(); +} + +int VideoFrameBuffer::StrideV() const { + return const_cast(this)->GetI420()->StrideV(); +} + +void* VideoFrameBuffer::native_handle() const { + RTC_DCHECK(type() != Type::kNative); + return nullptr; +} + +rtc::scoped_refptr VideoFrameBuffer::NativeToI420Buffer() { + return ToI420(); +} + +rtc::scoped_refptr VideoFrameBuffer::ToI420() { + return new rtc::RefCountedObject( + NativeToI420Buffer()); +} + +rtc::scoped_refptr VideoFrameBuffer::GetI420() { + RTC_CHECK(type() == Type::kI420); + // TODO(magjed): static_cast to PlanarYuvBuffer instead once external clients + // are updated. + return new rtc::RefCountedObject(this); +} + +rtc::scoped_refptr VideoFrameBuffer::GetI444() { + RTC_CHECK(type() == Type::kI444); + return static_cast(this); +} + +rtc::scoped_refptr PlanarYuvBuffer::ToI420() { + switch (type()) { + case Type::kI420: + return this; + case Type::kI444: { + rtc::scoped_refptr i420_buffer = + I420Buffer::Create(width(), height()); + libyuv::I420ToI444(DataY(), StrideY(), DataU(), StrideU(), DataV(), + StrideV(), i420_buffer->MutableDataY(), + i420_buffer->StrideY(), i420_buffer->MutableDataU(), + i420_buffer->StrideU(), i420_buffer->MutableDataV(), + i420_buffer->StrideV(), width(), height()); + return i420_buffer; + } + default: + RTC_NOTREACHED(); + return nullptr; + } +} + +int PlanarYuvBuffer::ChromaWidth() const { + switch (type()) { + case Type::kI420: + return (width() + 1) / 2; + case Type::kI444: + return width(); + default: + RTC_NOTREACHED(); + return 0; + } +} + +int PlanarYuvBuffer::ChromaHeight() const { + switch (type()) { + case Type::kI420: + return (height() + 1) / 2; + case Type::kI444: + return height(); + default: + RTC_NOTREACHED(); + return 0; + } +} + +} // namespace webrtc diff --git a/webrtc/api/video/video_frame_buffer.h b/webrtc/api/video/video_frame_buffer.h index c8c2e5d5d4..6d0c99107c 100644 --- a/webrtc/api/video/video_frame_buffer.h +++ b/webrtc/api/video/video_frame_buffer.h @@ -18,38 +18,100 @@ namespace webrtc { -// Interface of a simple frame buffer containing pixel data. This interface does -// not contain any frame metadata such as rotation, timestamp, pixel_width, etc. +class PlanarYuvBuffer; + +// Base class for frame buffers of different types of pixel format and storage. +// The tag in type() indicates how the data is represented, and each type is +// implemented as a subclass. To access the pixel data, call the appropriate +// GetXXX() function, where XXX represents the type. There is also a function +// ToI420() that returns a frame buffer in I420 format, converting from the +// underlying representation if necessary. I420 is the most widely accepted +// format and serves as a fallback for video sinks that can only handle I420, +// e.g. the internal WebRTC software encoders. A special enum value 'kNative' is +// provided for external clients to implement their own frame buffer +// representations, e.g. as textures. The external client can produce such +// native frame buffers from custom video sources, and then cast it back to the +// correct subclass in custom video sinks. The purpose of this is to improve +// performance by providing an optimized path without intermediate conversions. +// Frame metadata such as rotation and timestamp are stored in +// webrtc::VideoFrame, and not here. class VideoFrameBuffer : public rtc::RefCountInterface { public: + // New frame buffer types will be added conservatively when there is an + // opportunity to optimize the path between some pair of video source and + // video sink. + enum class Type { + kNative, + kI420, + kI444, + }; + + // This function specifies in what pixel format the data is stored in. + virtual Type type() const; + // The resolution of the frame in pixels. For formats where some planes are // subsampled, this is the highest-resolution plane. virtual int width() const = 0; virtual int height() const = 0; + // Returns a memory-backed frame buffer in I420 format. If the pixel data is + // in another format, a conversion will take place. All implementations must + // provide a fallback to I420 for compatibility with e.g. the internal WebRTC + // software encoders. + virtual rtc::scoped_refptr ToI420(); + + // These functions should only be called if type() is of the correct type. + // Calling with a different type will result in a crash. + rtc::scoped_refptr GetI420(); + rtc::scoped_refptr GetI444(); + + // Deprecated - use ToI420() first instead. // Returns pointer to the pixel data for a given plane. The memory is owned by // the VideoFrameBuffer object and must not be freed by the caller. - virtual const uint8_t* DataY() const = 0; - virtual const uint8_t* DataU() const = 0; - virtual const uint8_t* DataV() const = 0; - + virtual const uint8_t* DataY() const; + virtual const uint8_t* DataU() const; + virtual const uint8_t* DataV() const; // Returns the number of bytes between successive rows for a given plane. - virtual int StrideY() const = 0; - virtual int StrideU() const = 0; - virtual int StrideV() const = 0; + virtual int StrideY() const; + virtual int StrideU() const; + virtual int StrideV() const; + // Deprecated - use type() to determine if the stored data is kNative, and + // then cast into the appropriate type. // Return the handle of the underlying video frame. This is used when the // frame is backed by a texture. - virtual void* native_handle() const = 0; + virtual void* native_handle() const; - // Returns a new memory-backed frame buffer converted from this buffer's - // native handle. - virtual rtc::scoped_refptr NativeToI420Buffer() = 0; + // Deprecated - use ToI420() instead. + virtual rtc::scoped_refptr NativeToI420Buffer(); protected: ~VideoFrameBuffer() override {} }; +// This interface represents Type::kI420 and Type::kI444. +class PlanarYuvBuffer : public VideoFrameBuffer { + public: + int ChromaWidth() const; + int ChromaHeight() const; + + // Returns pointer to the pixel data for a given plane. The memory is owned by + // the VideoFrameBuffer object and must not be freed by the caller. + const uint8_t* DataY() const override = 0; + const uint8_t* DataU() const override = 0; + const uint8_t* DataV() const override = 0; + + // Returns the number of bytes between successive rows for a given plane. + int StrideY() const override = 0; + int StrideU() const override = 0; + int StrideV() const override = 0; + + rtc::scoped_refptr ToI420() override; + + protected: + ~PlanarYuvBuffer() override {} +}; + } // namespace webrtc #endif // WEBRTC_API_VIDEO_VIDEO_FRAME_BUFFER_H_ diff --git a/webrtc/common_video/include/video_frame_buffer.h b/webrtc/common_video/include/video_frame_buffer.h index e6c1f147a2..7e2fcf99e1 100644 --- a/webrtc/common_video/include/video_frame_buffer.h +++ b/webrtc/common_video/include/video_frame_buffer.h @@ -27,6 +27,7 @@ class NativeHandleBuffer : public VideoFrameBuffer { public: NativeHandleBuffer(void* native_handle, int width, int height); + Type type() const override; int width() const override; int height() const override; const uint8_t* DataY() const override; @@ -44,7 +45,7 @@ class NativeHandleBuffer : public VideoFrameBuffer { const int height_; }; -class WrappedI420Buffer : public webrtc::VideoFrameBuffer { +class WrappedI420Buffer : public PlanarYuvBuffer { public: WrappedI420Buffer(int width, int height, @@ -55,6 +56,8 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer { const uint8_t* v_plane, int v_stride, const rtc::Callback0& no_longer_used); + Type type() const override; + int width() const override; int height() const override; @@ -65,10 +68,6 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer { int StrideU() const override; int StrideV() const override; - void* native_handle() const override; - - rtc::scoped_refptr NativeToI420Buffer() override; - private: friend class rtc::RefCountedObject; ~WrappedI420Buffer() override; diff --git a/webrtc/common_video/video_frame_buffer.cc b/webrtc/common_video/video_frame_buffer.cc index 4646bf491f..0322dd1fd5 100644 --- a/webrtc/common_video/video_frame_buffer.cc +++ b/webrtc/common_video/video_frame_buffer.cc @@ -30,6 +30,10 @@ NativeHandleBuffer::NativeHandleBuffer(void* native_handle, RTC_DCHECK_GT(height, 0); } +VideoFrameBuffer::Type NativeHandleBuffer::type() const { + return Type::kNative; +} + int NativeHandleBuffer::width() const { return width_; } @@ -92,6 +96,10 @@ WrappedI420Buffer::~WrappedI420Buffer() { no_longer_used_cb_(); } +VideoFrameBuffer::Type WrappedI420Buffer::type() const { + return Type::kI420; +} + int WrappedI420Buffer::width() const { return width_; } @@ -120,13 +128,4 @@ int WrappedI420Buffer::StrideV() const { return v_stride_; } -void* WrappedI420Buffer::native_handle() const { - return nullptr; -} - -rtc::scoped_refptr WrappedI420Buffer::NativeToI420Buffer() { - RTC_NOTREACHED(); - return nullptr; -} - } // namespace webrtc