Add NV12 video buffer type

This will allow incoming NV12 frames to be encodable by libvpx without
requiring a conversion to I420 before encoding. NV12 is supported in
libvpx https://chromium.googlesource.com/webm/libvpx/+/master/CHANGELOG

Bug: webrtc:11916
Change-Id: I30e9c42c0607bee07691930c0248921bba09134c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/183720
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Evan Shrubsole <eshr@google.com>
Cr-Commit-Position: refs/heads/master@{#32061}

api/video/BUILD.gn

@@ -121,6 +121,24 @@ rtc_library("video_frame_i010") {
   ]
 }
 
+rtc_library("video_frame_nv12") {
+  visibility = [ "*" ]
+  sources = [
+    "nv12_buffer.cc",
+    "nv12_buffer.h",
+  ]
+  deps = [
+    ":video_frame",
+    ":video_frame_i420",
+    "..:scoped_refptr",
+    "../../rtc_base",
+    "../../rtc_base:checks",
+    "../../rtc_base/memory:aligned_malloc",
+    "../../rtc_base/system:rtc_export",
+    "//third_party/libyuv",
+  ]
+}
+
 rtc_library("encoded_image") {
   visibility = [ "*" ]
   sources = [

api/video/DEPS

@@ -18,6 +18,10 @@ specific_include_rules = {
     "+rtc_base/memory/aligned_malloc.h",
   ],
 
+  "nv12_buffer\.h": [
+    "+rtc_base/memory/aligned_malloc.h",
+  ],
+
   "recordable_encoded_frame\.h": [
     "+rtc_base/ref_count.h",
   ],

api/video/nv12_buffer.cc

@@ -0,0 +1,108 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/nv12_buffer.h"
+
+#include "api/video/i420_buffer.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/ref_counted_object.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+
+namespace webrtc {
+
+namespace {
+
+static const int kBufferAlignment = 64;
+
+int NV12DataSize(int height, int stride_y, int stride_uv) {
+  return stride_y * height + stride_uv * ((height + 1) / 2);
+}
+
+}  // namespace
+
+NV12Buffer::NV12Buffer(int width, int height)
+    : NV12Buffer(width, height, width, width + width % 2) {}
+
+NV12Buffer::NV12Buffer(int width, int height, int stride_y, int stride_uv)
+    : width_(width),
+      height_(height),
+      stride_y_(stride_y),
+      stride_uv_(stride_uv),
+      data_(static_cast<uint8_t*>(
+          AlignedMalloc(NV12DataSize(height_, stride_y_, stride_uv),
+                        kBufferAlignment))) {
+  RTC_DCHECK_GT(width, 0);
+  RTC_DCHECK_GT(height, 0);
+  RTC_DCHECK_GE(stride_y, width);
+  RTC_DCHECK_GE(stride_uv, (width + width % 2));
+}
+
+NV12Buffer::~NV12Buffer() = default;
+
+// static
+rtc::scoped_refptr<NV12Buffer> NV12Buffer::Create(int width, int height) {
+  return new rtc::RefCountedObject<NV12Buffer>(width, height);
+}
+
+// static
+rtc::scoped_refptr<NV12Buffer> NV12Buffer::Create(int width,
+                                                  int height,
+                                                  int stride_y,
+                                                  int stride_uv) {
+  return new rtc::RefCountedObject<NV12Buffer>(width, height, stride_y,
+                                               stride_uv);
+}
+
+rtc::scoped_refptr<I420BufferInterface> NV12Buffer::ToI420() {
+  rtc::scoped_refptr<I420Buffer> i420_buffer =
+      I420Buffer::Create(width(), height());
+  libyuv::NV12ToI420(DataY(), StrideY(), DataUV(), StrideUV(),
+                     i420_buffer->MutableDataY(), i420_buffer->StrideY(),
+                     i420_buffer->MutableDataU(), i420_buffer->StrideU(),
+                     i420_buffer->MutableDataV(), i420_buffer->StrideV(),
+                     width(), height());
+  return i420_buffer;
+}
+
+int NV12Buffer::width() const {
+  return width_;
+}
+int NV12Buffer::height() const {
+  return height_;
+}
+
+int NV12Buffer::StrideY() const {
+  return stride_y_;
+}
+int NV12Buffer::StrideUV() const {
+  return stride_uv_;
+}
+
+const uint8_t* NV12Buffer::DataY() const {
+  return data_.get();
+}
+
+const uint8_t* NV12Buffer::DataUV() const {
+  return data_.get() + UVOffset();
+}
+
+uint8_t* NV12Buffer::MutableDataY() {
+  return data_.get();
+}
+
+uint8_t* NV12Buffer::MutableDataUV() {
+  return data_.get() + UVOffset();
+}
+
+size_t NV12Buffer::UVOffset() const {
+  return stride_y_ * height_;
+}
+
+}  // namespace webrtc

api/video/nv12_buffer.h

@@ -0,0 +1,67 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_NV12_BUFFER_H_
+#define API_VIDEO_NV12_BUFFER_H_
+
+#include <memory>
+#include <utility>
+
+#include "api/scoped_refptr.h"
+#include "api/video/video_frame_buffer.h"
+#include "rtc_base/memory/aligned_malloc.h"
+#include "rtc_base/system/rtc_export.h"
+
+namespace webrtc {
+
+// NV12 is a biplanar encoding format, with full-resolution Y and
+// half-resolution interleved UV. More information can be found at
+// http://msdn.microsoft.com/library/windows/desktop/dd206750.aspx#nv12.
+class RTC_EXPORT NV12Buffer : public NV12BufferInterface {
+ public:
+  static rtc::scoped_refptr<NV12Buffer> Create(int width, int height);
+  static rtc::scoped_refptr<NV12Buffer> Create(int width,
+                                               int height,
+                                               int stride_y,
+                                               int stride_uv);
+
+  rtc::scoped_refptr<I420BufferInterface> ToI420() override;
+
+  int width() const override;
+  int height() const override;
+
+  int StrideY() const override;
+  int StrideUV() const override;
+
+  const uint8_t* DataY() const override;
+  const uint8_t* DataUV() const override;
+
+  uint8_t* MutableDataY();
+  uint8_t* MutableDataUV();
+
+ protected:
+  NV12Buffer(int width, int height);
+  NV12Buffer(int width, int height, int stride_y, int stride_uv);
+
+  ~NV12Buffer() override;
+
+ private:
+  size_t UVOffset() const;
+
+  const int width_;
+  const int height_;
+  const int stride_y_;
+  const int stride_uv_;
+  const std::unique_ptr<uint8_t, AlignedFreeDeleter> data_;
+};
+
+}  // namespace webrtc
+
+#endif  // API_VIDEO_NV12_BUFFER_H_

api/video/test/BUILD.gn

@@ -12,6 +12,7 @@ rtc_library("rtc_api_video_unittests") {
   testonly = true
   sources = [
     "color_space_unittest.cc",
+    "nv12_buffer_unittest.cc",
     "video_adaptation_counters_unittest.cc",
     "video_bitrate_allocation_unittest.cc",
   ]
@@ -19,7 +20,10 @@ rtc_library("rtc_api_video_unittests") {
     "..:video_adaptation",
     "..:video_bitrate_allocation",
     "..:video_frame",
+    "..:video_frame_i420",
+    "..:video_frame_nv12",
     "..:video_rtp_headers",
+    "../../../test:frame_utils",
     "../../../test:test_support",
   ]
   absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]

api/video/test/nv12_buffer_unittest.cc

@@ -0,0 +1,119 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/nv12_buffer.h"
+
+#include "api/video/i420_buffer.h"
+#include "test/frame_utils.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+namespace {
+int GetY(rtc::scoped_refptr<NV12BufferInterface> buf, int col, int row) {
+  return buf->DataY()[row * buf->StrideY() + col];
+}
+
+int GetU(rtc::scoped_refptr<NV12BufferInterface> buf, int col, int row) {
+  return buf->DataUV()[(row / 2) * buf->StrideUV() + (col / 2) * 2];
+}
+
+int GetV(rtc::scoped_refptr<NV12BufferInterface> buf, int col, int row) {
+  return buf->DataUV()[(row / 2) * buf->StrideUV() + (col / 2) * 2 + 1];
+}
+
+void FillNV12Buffer(rtc::scoped_refptr<NV12Buffer> buf) {
+  const uint8_t Y = 1;
+  const uint8_t U = 2;
+  const uint8_t V = 3;
+  for (int row = 0; row < buf->height(); ++row) {
+    for (int col = 0; col < buf->width(); ++col) {
+      buf->MutableDataY()[row * buf->StrideY() + col] = Y;
+    }
+  }
+  // Fill interleaving UV values.
+  for (int row = 0; row < buf->ChromaHeight(); row++) {
+    for (int col = 0; col < buf->StrideUV(); col += 2) {
+      int uv_index = row * buf->StrideUV() + col;
+      buf->MutableDataUV()[uv_index] = U;
+      buf->MutableDataUV()[uv_index + 1] = V;
+    }
+  }
+}
+
+}  // namespace
+
+TEST(NV12BufferTest, InitialData) {
+  constexpr int stride_y = 3;
+  constexpr int stride_uv = 4;
+  constexpr int width = 3;
+  constexpr int height = 3;
+
+  rtc::scoped_refptr<NV12Buffer> nv12_buffer(NV12Buffer::Create(width, height));
+  EXPECT_EQ(width, nv12_buffer->width());
+  EXPECT_EQ(height, nv12_buffer->height());
+  EXPECT_EQ(stride_y, nv12_buffer->StrideY());
+  EXPECT_EQ(stride_uv, nv12_buffer->StrideUV());
+  EXPECT_EQ(2, nv12_buffer->ChromaWidth());
+  EXPECT_EQ(2, nv12_buffer->ChromaHeight());
+}
+
+TEST(NV12BufferTest, ReadPixels) {
+  constexpr int width = 3;
+  constexpr int height = 3;
+
+  rtc::scoped_refptr<NV12Buffer> nv12_buffer(NV12Buffer::Create(width, height));
+  // Y = 1, U = 2, V = 3.
+  FillNV12Buffer(nv12_buffer);
+  for (int row = 0; row < height; row++) {
+    for (int col = 0; col < width; col++) {
+      EXPECT_EQ(1, GetY(nv12_buffer, col, row));
+      EXPECT_EQ(2, GetU(nv12_buffer, col, row));
+      EXPECT_EQ(3, GetV(nv12_buffer, col, row));
+    }
+  }
+}
+
+TEST(NV12BufferTest, ToI420) {
+  constexpr int width = 3;
+  constexpr int height = 3;
+  constexpr int size_y = width * height;
+  constexpr int size_u = (width + 1) / 2 * (height + 1) / 2;
+  constexpr int size_v = (width + 1) / 2 * (height + 1) / 2;
+  rtc::scoped_refptr<I420Buffer> reference(I420Buffer::Create(width, height));
+  memset(reference->MutableDataY(), 8, size_y);
+  memset(reference->MutableDataU(), 4, size_u);
+  memset(reference->MutableDataV(), 2, size_v);
+
+  rtc::scoped_refptr<NV12Buffer> nv12_buffer(NV12Buffer::Create(width, height));
+  // Convert the reference buffer to NV12.
+  memset(nv12_buffer->MutableDataY(), 8, size_y);
+  // Interleaving u/v values.
+  for (int i = 0; i < size_u + size_v; i += 2) {
+    nv12_buffer->MutableDataUV()[i] = 4;
+    nv12_buffer->MutableDataUV()[i + 1] = 2;
+  }
+  // Confirm YUV values are as expected.
+  for (int row = 0; row < height; row++) {
+    for (int col = 0; col < width; col++) {
+      EXPECT_EQ(8, GetY(nv12_buffer, col, row));
+      EXPECT_EQ(4, GetU(nv12_buffer, col, row));
+      EXPECT_EQ(2, GetV(nv12_buffer, col, row));
+    }
+  }
+
+  rtc::scoped_refptr<I420BufferInterface> i420_buffer(nv12_buffer->ToI420());
+  EXPECT_EQ(height, i420_buffer->height());
+  EXPECT_EQ(width, i420_buffer->width());
+  EXPECT_TRUE(test::FrameBufsEqual(reference, i420_buffer));
+}
+
+}  // namespace webrtc

api/video/video_frame_buffer.cc

@@ -35,6 +35,11 @@ const I010BufferInterface* VideoFrameBuffer::GetI010() const {
   return static_cast<const I010BufferInterface*>(this);
 }
 
+const NV12BufferInterface* VideoFrameBuffer::GetNV12() const {
+  RTC_CHECK(type() == Type::kNV12);
+  return static_cast<const NV12BufferInterface*>(this);
+}
+
 VideoFrameBuffer::Type I420BufferInterface::type() const {
   return Type::kI420;
 }
@@ -83,4 +88,16 @@ int I010BufferInterface::ChromaHeight() const {
   return (height() + 1) / 2;
 }
 
+VideoFrameBuffer::Type NV12BufferInterface::type() const {
+  return Type::kNV12;
+}
+
+int NV12BufferInterface::ChromaWidth() const {
+  return (width() + 1) / 2;
+}
+
+int NV12BufferInterface::ChromaHeight() const {
+  return (height() + 1) / 2;
+}
+
 }  // namespace webrtc

api/video/video_frame_buffer.h

@@ -23,6 +23,7 @@ class I420BufferInterface;
 class I420ABufferInterface;
 class I444BufferInterface;
 class I010BufferInterface;
+class NV12BufferInterface;
 
 // Base class for frame buffers of different types of pixel format and storage.
 // The tag in type() indicates how the data is represented, and each type is
@@ -50,6 +51,7 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface {
     kI420A,
     kI444,
     kI010,
+    kNV12,
   };
 
   // This function specifies in what pixel format the data is stored in.
@@ -79,6 +81,7 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface {
   const I420ABufferInterface* GetI420A() const;
   const I444BufferInterface* GetI444() const;
   const I010BufferInterface* GetI010() const;
+  const NV12BufferInterface* GetNV12() const;
 
  protected:
   ~VideoFrameBuffer() override {}
@@ -175,6 +178,42 @@ class I010BufferInterface : public PlanarYuv16BBuffer {
   ~I010BufferInterface() override {}
 };
 
+class BiplanarYuvBuffer : public VideoFrameBuffer {
+ public:
+  virtual int ChromaWidth() const = 0;
+  virtual int ChromaHeight() const = 0;
+
+  // Returns the number of steps(in terms of Data*() return type) between
+  // successive rows for a given plane.
+  virtual int StrideY() const = 0;
+  virtual int StrideUV() const = 0;
+
+ protected:
+  ~BiplanarYuvBuffer() override {}
+};
+
+class BiplanarYuv8Buffer : public BiplanarYuvBuffer {
+ public:
+  virtual const uint8_t* DataY() const = 0;
+  virtual const uint8_t* DataUV() const = 0;
+
+ protected:
+  ~BiplanarYuv8Buffer() override {}
+};
+
+// Represents Type::kNV12. NV12 is full resolution Y and half-resolution
+// interleved UV.
+class NV12BufferInterface : public BiplanarYuv8Buffer {
+ public:
+  Type type() const override;
+
+  int ChromaWidth() const final;
+  int ChromaHeight() const final;
+
+ protected:
+  ~NV12BufferInterface() override {}
+};
+
 }  // namespace webrtc
 
 #endif  // API_VIDEO_VIDEO_FRAME_BUFFER_H_