From 84995439fdfaa3ddee2d4120e32889f988a534e0 Mon Sep 17 00:00:00 2001 From: Evan Shrubsole Date: Wed, 9 Sep 2020 16:14:19 +0200 Subject: [PATCH] Add NV12 video buffer type This will allow incoming NV12 frames to be encodable by libvpx without requiring a conversion to I420 before encoding. NV12 is supported in libvpx https://chromium.googlesource.com/webm/libvpx/+/master/CHANGELOG Bug: webrtc:11916 Change-Id: I30e9c42c0607bee07691930c0248921bba09134c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/183720 Reviewed-by: Niels Moller Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Evan Shrubsole Cr-Commit-Position: refs/heads/master@{#32061} --- api/video/BUILD.gn | 18 ++++ api/video/DEPS | 4 + api/video/nv12_buffer.cc | 108 ++++++++++++++++++++++ api/video/nv12_buffer.h | 67 ++++++++++++++ api/video/test/BUILD.gn | 4 + api/video/test/nv12_buffer_unittest.cc | 119 +++++++++++++++++++++++++ api/video/video_frame_buffer.cc | 17 ++++ api/video/video_frame_buffer.h | 39 ++++++++ 8 files changed, 376 insertions(+) create mode 100644 api/video/nv12_buffer.cc create mode 100644 api/video/nv12_buffer.h create mode 100644 api/video/test/nv12_buffer_unittest.cc diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index b85c2b6232..6de518cd19 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -121,6 +121,24 @@ rtc_library("video_frame_i010") { ] } +rtc_library("video_frame_nv12") { + visibility = [ "*" ] + sources = [ + "nv12_buffer.cc", + "nv12_buffer.h", + ] + deps = [ + ":video_frame", + ":video_frame_i420", + "..:scoped_refptr", + "../../rtc_base", + "../../rtc_base:checks", + "../../rtc_base/memory:aligned_malloc", + "../../rtc_base/system:rtc_export", + "//third_party/libyuv", + ] +} + rtc_library("encoded_image") { visibility = [ "*" ] sources = [ diff --git a/api/video/DEPS b/api/video/DEPS index 3af594cd8a..1cb8ad83cb 100644 --- a/api/video/DEPS +++ b/api/video/DEPS @@ -18,6 +18,10 @@ specific_include_rules = { "+rtc_base/memory/aligned_malloc.h", ], + "nv12_buffer\.h": [ + "+rtc_base/memory/aligned_malloc.h", + ], + "recordable_encoded_frame\.h": [ "+rtc_base/ref_count.h", ], diff --git a/api/video/nv12_buffer.cc b/api/video/nv12_buffer.cc new file mode 100644 index 0000000000..54c27a8d78 --- /dev/null +++ b/api/video/nv12_buffer.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/nv12_buffer.h" + +#include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/ref_counted_object.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +namespace webrtc { + +namespace { + +static const int kBufferAlignment = 64; + +int NV12DataSize(int height, int stride_y, int stride_uv) { + return stride_y * height + stride_uv * ((height + 1) / 2); +} + +} // namespace + +NV12Buffer::NV12Buffer(int width, int height) + : NV12Buffer(width, height, width, width + width % 2) {} + +NV12Buffer::NV12Buffer(int width, int height, int stride_y, int stride_uv) + : width_(width), + height_(height), + stride_y_(stride_y), + stride_uv_(stride_uv), + data_(static_cast( + AlignedMalloc(NV12DataSize(height_, stride_y_, stride_uv), + kBufferAlignment))) { + RTC_DCHECK_GT(width, 0); + RTC_DCHECK_GT(height, 0); + RTC_DCHECK_GE(stride_y, width); + RTC_DCHECK_GE(stride_uv, (width + width % 2)); +} + +NV12Buffer::~NV12Buffer() = default; + +// static +rtc::scoped_refptr NV12Buffer::Create(int width, int height) { + return new rtc::RefCountedObject(width, height); +} + +// static +rtc::scoped_refptr NV12Buffer::Create(int width, + int height, + int stride_y, + int stride_uv) { + return new rtc::RefCountedObject(width, height, stride_y, + stride_uv); +} + +rtc::scoped_refptr NV12Buffer::ToI420() { + rtc::scoped_refptr i420_buffer = + I420Buffer::Create(width(), height()); + libyuv::NV12ToI420(DataY(), StrideY(), DataUV(), StrideUV(), + i420_buffer->MutableDataY(), i420_buffer->StrideY(), + i420_buffer->MutableDataU(), i420_buffer->StrideU(), + i420_buffer->MutableDataV(), i420_buffer->StrideV(), + width(), height()); + return i420_buffer; +} + +int NV12Buffer::width() const { + return width_; +} +int NV12Buffer::height() const { + return height_; +} + +int NV12Buffer::StrideY() const { + return stride_y_; +} +int NV12Buffer::StrideUV() const { + return stride_uv_; +} + +const uint8_t* NV12Buffer::DataY() const { + return data_.get(); +} + +const uint8_t* NV12Buffer::DataUV() const { + return data_.get() + UVOffset(); +} + +uint8_t* NV12Buffer::MutableDataY() { + return data_.get(); +} + +uint8_t* NV12Buffer::MutableDataUV() { + return data_.get() + UVOffset(); +} + +size_t NV12Buffer::UVOffset() const { + return stride_y_ * height_; +} + +} // namespace webrtc diff --git a/api/video/nv12_buffer.h b/api/video/nv12_buffer.h new file mode 100644 index 0000000000..8eb484c069 --- /dev/null +++ b/api/video/nv12_buffer.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_NV12_BUFFER_H_ +#define API_VIDEO_NV12_BUFFER_H_ + +#include +#include + +#include "api/scoped_refptr.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_base/memory/aligned_malloc.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// NV12 is a biplanar encoding format, with full-resolution Y and +// half-resolution interleved UV. More information can be found at +// http://msdn.microsoft.com/library/windows/desktop/dd206750.aspx#nv12. +class RTC_EXPORT NV12Buffer : public NV12BufferInterface { + public: + static rtc::scoped_refptr Create(int width, int height); + static rtc::scoped_refptr Create(int width, + int height, + int stride_y, + int stride_uv); + + rtc::scoped_refptr ToI420() override; + + int width() const override; + int height() const override; + + int StrideY() const override; + int StrideUV() const override; + + const uint8_t* DataY() const override; + const uint8_t* DataUV() const override; + + uint8_t* MutableDataY(); + uint8_t* MutableDataUV(); + + protected: + NV12Buffer(int width, int height); + NV12Buffer(int width, int height, int stride_y, int stride_uv); + + ~NV12Buffer() override; + + private: + size_t UVOffset() const; + + const int width_; + const int height_; + const int stride_y_; + const int stride_uv_; + const std::unique_ptr data_; +}; + +} // namespace webrtc + +#endif // API_VIDEO_NV12_BUFFER_H_ diff --git a/api/video/test/BUILD.gn b/api/video/test/BUILD.gn index 5633371102..d46b48e618 100644 --- a/api/video/test/BUILD.gn +++ b/api/video/test/BUILD.gn @@ -12,6 +12,7 @@ rtc_library("rtc_api_video_unittests") { testonly = true sources = [ "color_space_unittest.cc", + "nv12_buffer_unittest.cc", "video_adaptation_counters_unittest.cc", "video_bitrate_allocation_unittest.cc", ] @@ -19,7 +20,10 @@ rtc_library("rtc_api_video_unittests") { "..:video_adaptation", "..:video_bitrate_allocation", "..:video_frame", + "..:video_frame_i420", + "..:video_frame_nv12", "..:video_rtp_headers", + "../../../test:frame_utils", "../../../test:test_support", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] diff --git a/api/video/test/nv12_buffer_unittest.cc b/api/video/test/nv12_buffer_unittest.cc new file mode 100644 index 0000000000..d84adb5bf5 --- /dev/null +++ b/api/video/test/nv12_buffer_unittest.cc @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/nv12_buffer.h" + +#include "api/video/i420_buffer.h" +#include "test/frame_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +int GetY(rtc::scoped_refptr buf, int col, int row) { + return buf->DataY()[row * buf->StrideY() + col]; +} + +int GetU(rtc::scoped_refptr buf, int col, int row) { + return buf->DataUV()[(row / 2) * buf->StrideUV() + (col / 2) * 2]; +} + +int GetV(rtc::scoped_refptr buf, int col, int row) { + return buf->DataUV()[(row / 2) * buf->StrideUV() + (col / 2) * 2 + 1]; +} + +void FillNV12Buffer(rtc::scoped_refptr buf) { + const uint8_t Y = 1; + const uint8_t U = 2; + const uint8_t V = 3; + for (int row = 0; row < buf->height(); ++row) { + for (int col = 0; col < buf->width(); ++col) { + buf->MutableDataY()[row * buf->StrideY() + col] = Y; + } + } + // Fill interleaving UV values. + for (int row = 0; row < buf->ChromaHeight(); row++) { + for (int col = 0; col < buf->StrideUV(); col += 2) { + int uv_index = row * buf->StrideUV() + col; + buf->MutableDataUV()[uv_index] = U; + buf->MutableDataUV()[uv_index + 1] = V; + } + } +} + +} // namespace + +TEST(NV12BufferTest, InitialData) { + constexpr int stride_y = 3; + constexpr int stride_uv = 4; + constexpr int width = 3; + constexpr int height = 3; + + rtc::scoped_refptr nv12_buffer(NV12Buffer::Create(width, height)); + EXPECT_EQ(width, nv12_buffer->width()); + EXPECT_EQ(height, nv12_buffer->height()); + EXPECT_EQ(stride_y, nv12_buffer->StrideY()); + EXPECT_EQ(stride_uv, nv12_buffer->StrideUV()); + EXPECT_EQ(2, nv12_buffer->ChromaWidth()); + EXPECT_EQ(2, nv12_buffer->ChromaHeight()); +} + +TEST(NV12BufferTest, ReadPixels) { + constexpr int width = 3; + constexpr int height = 3; + + rtc::scoped_refptr nv12_buffer(NV12Buffer::Create(width, height)); + // Y = 1, U = 2, V = 3. + FillNV12Buffer(nv12_buffer); + for (int row = 0; row < height; row++) { + for (int col = 0; col < width; col++) { + EXPECT_EQ(1, GetY(nv12_buffer, col, row)); + EXPECT_EQ(2, GetU(nv12_buffer, col, row)); + EXPECT_EQ(3, GetV(nv12_buffer, col, row)); + } + } +} + +TEST(NV12BufferTest, ToI420) { + constexpr int width = 3; + constexpr int height = 3; + constexpr int size_y = width * height; + constexpr int size_u = (width + 1) / 2 * (height + 1) / 2; + constexpr int size_v = (width + 1) / 2 * (height + 1) / 2; + rtc::scoped_refptr reference(I420Buffer::Create(width, height)); + memset(reference->MutableDataY(), 8, size_y); + memset(reference->MutableDataU(), 4, size_u); + memset(reference->MutableDataV(), 2, size_v); + + rtc::scoped_refptr nv12_buffer(NV12Buffer::Create(width, height)); + // Convert the reference buffer to NV12. + memset(nv12_buffer->MutableDataY(), 8, size_y); + // Interleaving u/v values. + for (int i = 0; i < size_u + size_v; i += 2) { + nv12_buffer->MutableDataUV()[i] = 4; + nv12_buffer->MutableDataUV()[i + 1] = 2; + } + // Confirm YUV values are as expected. + for (int row = 0; row < height; row++) { + for (int col = 0; col < width; col++) { + EXPECT_EQ(8, GetY(nv12_buffer, col, row)); + EXPECT_EQ(4, GetU(nv12_buffer, col, row)); + EXPECT_EQ(2, GetV(nv12_buffer, col, row)); + } + } + + rtc::scoped_refptr i420_buffer(nv12_buffer->ToI420()); + EXPECT_EQ(height, i420_buffer->height()); + EXPECT_EQ(width, i420_buffer->width()); + EXPECT_TRUE(test::FrameBufsEqual(reference, i420_buffer)); +} + +} // namespace webrtc diff --git a/api/video/video_frame_buffer.cc b/api/video/video_frame_buffer.cc index b9fd9cd92a..aa4e69f15a 100644 --- a/api/video/video_frame_buffer.cc +++ b/api/video/video_frame_buffer.cc @@ -35,6 +35,11 @@ const I010BufferInterface* VideoFrameBuffer::GetI010() const { return static_cast(this); } +const NV12BufferInterface* VideoFrameBuffer::GetNV12() const { + RTC_CHECK(type() == Type::kNV12); + return static_cast(this); +} + VideoFrameBuffer::Type I420BufferInterface::type() const { return Type::kI420; } @@ -83,4 +88,16 @@ int I010BufferInterface::ChromaHeight() const { return (height() + 1) / 2; } +VideoFrameBuffer::Type NV12BufferInterface::type() const { + return Type::kNV12; +} + +int NV12BufferInterface::ChromaWidth() const { + return (width() + 1) / 2; +} + +int NV12BufferInterface::ChromaHeight() const { + return (height() + 1) / 2; +} + } // namespace webrtc diff --git a/api/video/video_frame_buffer.h b/api/video/video_frame_buffer.h index d87a4230a4..fa72ad69e3 100644 --- a/api/video/video_frame_buffer.h +++ b/api/video/video_frame_buffer.h @@ -23,6 +23,7 @@ class I420BufferInterface; class I420ABufferInterface; class I444BufferInterface; class I010BufferInterface; +class NV12BufferInterface; // Base class for frame buffers of different types of pixel format and storage. // The tag in type() indicates how the data is represented, and each type is @@ -50,6 +51,7 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface { kI420A, kI444, kI010, + kNV12, }; // This function specifies in what pixel format the data is stored in. @@ -79,6 +81,7 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface { const I420ABufferInterface* GetI420A() const; const I444BufferInterface* GetI444() const; const I010BufferInterface* GetI010() const; + const NV12BufferInterface* GetNV12() const; protected: ~VideoFrameBuffer() override {} @@ -175,6 +178,42 @@ class I010BufferInterface : public PlanarYuv16BBuffer { ~I010BufferInterface() override {} }; +class BiplanarYuvBuffer : public VideoFrameBuffer { + public: + virtual int ChromaWidth() const = 0; + virtual int ChromaHeight() const = 0; + + // Returns the number of steps(in terms of Data*() return type) between + // successive rows for a given plane. + virtual int StrideY() const = 0; + virtual int StrideUV() const = 0; + + protected: + ~BiplanarYuvBuffer() override {} +}; + +class BiplanarYuv8Buffer : public BiplanarYuvBuffer { + public: + virtual const uint8_t* DataY() const = 0; + virtual const uint8_t* DataUV() const = 0; + + protected: + ~BiplanarYuv8Buffer() override {} +}; + +// Represents Type::kNV12. NV12 is full resolution Y and half-resolution +// interleved UV. +class NV12BufferInterface : public BiplanarYuv8Buffer { + public: + Type type() const override; + + int ChromaWidth() const final; + int ChromaHeight() const final; + + protected: + ~NV12BufferInterface() override {} +}; + } // namespace webrtc #endif // API_VIDEO_VIDEO_FRAME_BUFFER_H_