diff --git a/webrtc/api/BUILD.gn b/webrtc/api/BUILD.gn index 2b1ca34576..d47483b81e 100644 --- a/webrtc/api/BUILD.gn +++ b/webrtc/api/BUILD.gn @@ -170,6 +170,7 @@ rtc_source_set("video_frame_api") { "video/i420_buffer.h", "video/video_frame.cc", "video/video_frame.h", + "video/video_frame_buffer.cc", "video/video_frame_buffer.h", "video/video_rotation.h", ] diff --git a/webrtc/api/video/i420_buffer.cc b/webrtc/api/video/i420_buffer.cc index 32c73eac67..51d83c9df5 100644 --- a/webrtc/api/video/i420_buffer.cc +++ b/webrtc/api/video/i420_buffer.cc @@ -136,6 +136,10 @@ void I420Buffer::InitializeData() { I420DataSize(height_, stride_y_, stride_u_, stride_v_)); } +VideoFrameBuffer::Type I420Buffer::type() const { + return Type::kI420; +} + int I420Buffer::width() const { return width_; } @@ -164,14 +168,6 @@ int I420Buffer::StrideV() const { return stride_v_; } -void* I420Buffer::native_handle() const { - return nullptr; -} - -rtc::scoped_refptr I420Buffer::NativeToI420Buffer() { - return this; -} - uint8_t* I420Buffer::MutableDataY() { return const_cast(DataY()); } diff --git a/webrtc/api/video/i420_buffer.h b/webrtc/api/video/i420_buffer.h index 507bd0571b..20a84a1a53 100644 --- a/webrtc/api/video/i420_buffer.h +++ b/webrtc/api/video/i420_buffer.h @@ -20,7 +20,7 @@ namespace webrtc { // Plain I420 buffer in standard memory. -class I420Buffer : public VideoFrameBuffer { +class I420Buffer : public PlanarYuvBuffer { public: static rtc::scoped_refptr Create(int width, int height); static rtc::scoped_refptr Create(int width, @@ -53,6 +53,7 @@ class I420Buffer : public VideoFrameBuffer { // are resolved in a better way. Or in the mean time, use SetBlack. void InitializeData(); + Type type() const override; int width() const override; int height() const override; const uint8_t* DataY() const override; @@ -63,9 +64,6 @@ class I420Buffer : public VideoFrameBuffer { int StrideU() const override; int StrideV() const override; - void* native_handle() const override; - rtc::scoped_refptr NativeToI420Buffer() override; - uint8_t* MutableDataY(); uint8_t* MutableDataU(); uint8_t* MutableDataV(); diff --git a/webrtc/api/video/video_frame_buffer.cc b/webrtc/api/video/video_frame_buffer.cc new file mode 100644 index 0000000000..b4b30a1268 --- /dev/null +++ b/webrtc/api/video/video_frame_buffer.cc @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/api/video/video_frame_buffer.h" + +#include "libyuv/convert_from.h" +#include "webrtc/api/video/i420_buffer.h" +#include "webrtc/base/checks.h" + +namespace webrtc { + +namespace { + +// TODO(magjed): Remove this class. It is only used for providing a default +// implementation of ToI420() until external clients are updated. ToI420() will +// then be made pure virtual. This adapter adapts a VideoFrameBuffer (which is +// expected to be in I420 format) to the PlanarYuvBuffer interface. The reason +// this is needed is because of the return type mismatch in NativeToI420Buffer +// (returns VideoFrameBuffer) vs ToI420 (returns PlanarYuvBuffer). +class PlanarYuvBufferAdapter : public PlanarYuvBuffer { + public: + explicit PlanarYuvBufferAdapter(rtc::scoped_refptr buffer) + : buffer_(buffer) {} + + Type type() const override { return Type::kI420; } + + int width() const override { return buffer_->width(); } + int height() const override { return buffer_->height(); } + + const uint8_t* DataY() const override { return buffer_->DataY(); } + const uint8_t* DataU() const override { return buffer_->DataU(); } + const uint8_t* DataV() const override { return buffer_->DataV(); } + + int StrideY() const override { return buffer_->StrideY(); } + int StrideU() const override { return buffer_->StrideU(); } + int StrideV() const override { return buffer_->StrideV(); } + + private: + rtc::scoped_refptr buffer_; +}; + +} // namespace + +// TODO(magjed): The default implementations in VideoFrameBuffer are provided in +// order to support the deprecated interface until external clients are updated. +// Remove once done. +VideoFrameBuffer::Type VideoFrameBuffer::type() const { + return native_handle() ? Type::kNative : Type::kI420; +} + +const uint8_t* VideoFrameBuffer::DataY() const { + return const_cast(this)->GetI420()->DataY(); +} + +const uint8_t* VideoFrameBuffer::DataU() const { + return const_cast(this)->GetI420()->DataU(); +} + +const uint8_t* VideoFrameBuffer::DataV() const { + return const_cast(this)->GetI420()->DataV(); +} + +// Returns the number of bytes between successive rows for a given plane. +int VideoFrameBuffer::StrideY() const { + return const_cast(this)->GetI420()->StrideY(); +} + +int VideoFrameBuffer::StrideU() const { + return const_cast(this)->GetI420()->StrideU(); +} + +int VideoFrameBuffer::StrideV() const { + return const_cast(this)->GetI420()->StrideV(); +} + +void* VideoFrameBuffer::native_handle() const { + RTC_DCHECK(type() != Type::kNative); + return nullptr; +} + +rtc::scoped_refptr VideoFrameBuffer::NativeToI420Buffer() { + return ToI420(); +} + +rtc::scoped_refptr VideoFrameBuffer::ToI420() { + return new rtc::RefCountedObject( + NativeToI420Buffer()); +} + +rtc::scoped_refptr VideoFrameBuffer::GetI420() { + RTC_CHECK(type() == Type::kI420); + // TODO(magjed): static_cast to PlanarYuvBuffer instead once external clients + // are updated. + return new rtc::RefCountedObject(this); +} + +rtc::scoped_refptr VideoFrameBuffer::GetI444() { + RTC_CHECK(type() == Type::kI444); + return static_cast(this); +} + +rtc::scoped_refptr PlanarYuvBuffer::ToI420() { + switch (type()) { + case Type::kI420: + return this; + case Type::kI444: { + rtc::scoped_refptr i420_buffer = + I420Buffer::Create(width(), height()); + libyuv::I420ToI444(DataY(), StrideY(), DataU(), StrideU(), DataV(), + StrideV(), i420_buffer->MutableDataY(), + i420_buffer->StrideY(), i420_buffer->MutableDataU(), + i420_buffer->StrideU(), i420_buffer->MutableDataV(), + i420_buffer->StrideV(), width(), height()); + return i420_buffer; + } + default: + RTC_NOTREACHED(); + return nullptr; + } +} + +int PlanarYuvBuffer::ChromaWidth() const { + switch (type()) { + case Type::kI420: + return (width() + 1) / 2; + case Type::kI444: + return width(); + default: + RTC_NOTREACHED(); + return 0; + } +} + +int PlanarYuvBuffer::ChromaHeight() const { + switch (type()) { + case Type::kI420: + return (height() + 1) / 2; + case Type::kI444: + return height(); + default: + RTC_NOTREACHED(); + return 0; + } +} + +} // namespace webrtc diff --git a/webrtc/api/video/video_frame_buffer.h b/webrtc/api/video/video_frame_buffer.h index c8c2e5d5d4..6d0c99107c 100644 --- a/webrtc/api/video/video_frame_buffer.h +++ b/webrtc/api/video/video_frame_buffer.h @@ -18,38 +18,100 @@ namespace webrtc { -// Interface of a simple frame buffer containing pixel data. This interface does -// not contain any frame metadata such as rotation, timestamp, pixel_width, etc. +class PlanarYuvBuffer; + +// Base class for frame buffers of different types of pixel format and storage. +// The tag in type() indicates how the data is represented, and each type is +// implemented as a subclass. To access the pixel data, call the appropriate +// GetXXX() function, where XXX represents the type. There is also a function +// ToI420() that returns a frame buffer in I420 format, converting from the +// underlying representation if necessary. I420 is the most widely accepted +// format and serves as a fallback for video sinks that can only handle I420, +// e.g. the internal WebRTC software encoders. A special enum value 'kNative' is +// provided for external clients to implement their own frame buffer +// representations, e.g. as textures. The external client can produce such +// native frame buffers from custom video sources, and then cast it back to the +// correct subclass in custom video sinks. The purpose of this is to improve +// performance by providing an optimized path without intermediate conversions. +// Frame metadata such as rotation and timestamp are stored in +// webrtc::VideoFrame, and not here. class VideoFrameBuffer : public rtc::RefCountInterface { public: + // New frame buffer types will be added conservatively when there is an + // opportunity to optimize the path between some pair of video source and + // video sink. + enum class Type { + kNative, + kI420, + kI444, + }; + + // This function specifies in what pixel format the data is stored in. + virtual Type type() const; + // The resolution of the frame in pixels. For formats where some planes are // subsampled, this is the highest-resolution plane. virtual int width() const = 0; virtual int height() const = 0; + // Returns a memory-backed frame buffer in I420 format. If the pixel data is + // in another format, a conversion will take place. All implementations must + // provide a fallback to I420 for compatibility with e.g. the internal WebRTC + // software encoders. + virtual rtc::scoped_refptr ToI420(); + + // These functions should only be called if type() is of the correct type. + // Calling with a different type will result in a crash. + rtc::scoped_refptr GetI420(); + rtc::scoped_refptr GetI444(); + + // Deprecated - use ToI420() first instead. // Returns pointer to the pixel data for a given plane. The memory is owned by // the VideoFrameBuffer object and must not be freed by the caller. - virtual const uint8_t* DataY() const = 0; - virtual const uint8_t* DataU() const = 0; - virtual const uint8_t* DataV() const = 0; - + virtual const uint8_t* DataY() const; + virtual const uint8_t* DataU() const; + virtual const uint8_t* DataV() const; // Returns the number of bytes between successive rows for a given plane. - virtual int StrideY() const = 0; - virtual int StrideU() const = 0; - virtual int StrideV() const = 0; + virtual int StrideY() const; + virtual int StrideU() const; + virtual int StrideV() const; + // Deprecated - use type() to determine if the stored data is kNative, and + // then cast into the appropriate type. // Return the handle of the underlying video frame. This is used when the // frame is backed by a texture. - virtual void* native_handle() const = 0; + virtual void* native_handle() const; - // Returns a new memory-backed frame buffer converted from this buffer's - // native handle. - virtual rtc::scoped_refptr NativeToI420Buffer() = 0; + // Deprecated - use ToI420() instead. + virtual rtc::scoped_refptr NativeToI420Buffer(); protected: ~VideoFrameBuffer() override {} }; +// This interface represents Type::kI420 and Type::kI444. +class PlanarYuvBuffer : public VideoFrameBuffer { + public: + int ChromaWidth() const; + int ChromaHeight() const; + + // Returns pointer to the pixel data for a given plane. The memory is owned by + // the VideoFrameBuffer object and must not be freed by the caller. + const uint8_t* DataY() const override = 0; + const uint8_t* DataU() const override = 0; + const uint8_t* DataV() const override = 0; + + // Returns the number of bytes between successive rows for a given plane. + int StrideY() const override = 0; + int StrideU() const override = 0; + int StrideV() const override = 0; + + rtc::scoped_refptr ToI420() override; + + protected: + ~PlanarYuvBuffer() override {} +}; + } // namespace webrtc #endif // WEBRTC_API_VIDEO_VIDEO_FRAME_BUFFER_H_ diff --git a/webrtc/common_video/include/video_frame_buffer.h b/webrtc/common_video/include/video_frame_buffer.h index e6c1f147a2..7e2fcf99e1 100644 --- a/webrtc/common_video/include/video_frame_buffer.h +++ b/webrtc/common_video/include/video_frame_buffer.h @@ -27,6 +27,7 @@ class NativeHandleBuffer : public VideoFrameBuffer { public: NativeHandleBuffer(void* native_handle, int width, int height); + Type type() const override; int width() const override; int height() const override; const uint8_t* DataY() const override; @@ -44,7 +45,7 @@ class NativeHandleBuffer : public VideoFrameBuffer { const int height_; }; -class WrappedI420Buffer : public webrtc::VideoFrameBuffer { +class WrappedI420Buffer : public PlanarYuvBuffer { public: WrappedI420Buffer(int width, int height, @@ -55,6 +56,8 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer { const uint8_t* v_plane, int v_stride, const rtc::Callback0& no_longer_used); + Type type() const override; + int width() const override; int height() const override; @@ -65,10 +68,6 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer { int StrideU() const override; int StrideV() const override; - void* native_handle() const override; - - rtc::scoped_refptr NativeToI420Buffer() override; - private: friend class rtc::RefCountedObject; ~WrappedI420Buffer() override; diff --git a/webrtc/common_video/video_frame_buffer.cc b/webrtc/common_video/video_frame_buffer.cc index 4646bf491f..0322dd1fd5 100644 --- a/webrtc/common_video/video_frame_buffer.cc +++ b/webrtc/common_video/video_frame_buffer.cc @@ -30,6 +30,10 @@ NativeHandleBuffer::NativeHandleBuffer(void* native_handle, RTC_DCHECK_GT(height, 0); } +VideoFrameBuffer::Type NativeHandleBuffer::type() const { + return Type::kNative; +} + int NativeHandleBuffer::width() const { return width_; } @@ -92,6 +96,10 @@ WrappedI420Buffer::~WrappedI420Buffer() { no_longer_used_cb_(); } +VideoFrameBuffer::Type WrappedI420Buffer::type() const { + return Type::kI420; +} + int WrappedI420Buffer::width() const { return width_; } @@ -120,13 +128,4 @@ int WrappedI420Buffer::StrideV() const { return v_stride_; } -void* WrappedI420Buffer::native_handle() const { - return nullptr; -} - -rtc::scoped_refptr WrappedI420Buffer::NativeToI420Buffer() { - RTC_NOTREACHED(); - return nullptr; -} - } // namespace webrtc