diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index fe824f20ef..5d1aa2a1a3 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -92,6 +92,8 @@ rtc_library("video_frame_i010") { "i010_buffer.h", "i210_buffer.cc", "i210_buffer.h", + "i410_buffer.cc", + "i410_buffer.h", ] deps = [ ":video_frame", diff --git a/api/video/DEPS b/api/video/DEPS index d1d39121b4..c84299f943 100644 --- a/api/video/DEPS +++ b/api/video/DEPS @@ -18,6 +18,10 @@ specific_include_rules = { "+rtc_base/memory/aligned_malloc.h", ], + "i410_buffer\.h": [ + "+rtc_base/memory/aligned_malloc.h", + ], + "i420_buffer\.h": [ "+rtc_base/memory/aligned_malloc.h", ], diff --git a/api/video/i410_buffer.cc b/api/video/i410_buffer.cc new file mode 100644 index 0000000000..1b0d4fdb5c --- /dev/null +++ b/api/video/i410_buffer.cc @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/video/i410_buffer.h" + +#include + +#include +#include + +#include "api/make_ref_counted.h" +#include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" +#include "third_party/libyuv/include/libyuv/convert.h" +#include "third_party/libyuv/include/libyuv/planar_functions.h" +#include "third_party/libyuv/include/libyuv/scale.h" + +// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD. +static const int kBufferAlignment = 64; +static const int kBytesPerPixel = 2; + +namespace webrtc { + +namespace { + +int I410DataSize(int height, int stride_y, int stride_u, int stride_v) { + return kBytesPerPixel * + (stride_y * height + stride_u * height + stride_v * height); +} + +} // namespace + +I410Buffer::I410Buffer(int width, int height) + : I410Buffer(width, height, width, width, width) {} + +I410Buffer::I410Buffer(int width, + int height, + int stride_y, + int stride_u, + int stride_v) + : width_(width), + height_(height), + stride_y_(stride_y), + stride_u_(stride_u), + stride_v_(stride_v), + data_(static_cast( + AlignedMalloc(I410DataSize(height, stride_y, stride_u, stride_v), + kBufferAlignment))) { + RTC_DCHECK_GT(width, 0); + RTC_DCHECK_GT(height, 0); + RTC_DCHECK_GE(stride_y, width); + RTC_DCHECK_GE(stride_u, width); + RTC_DCHECK_GE(stride_v, width); +} + +I410Buffer::~I410Buffer() {} + +// static +rtc::scoped_refptr I410Buffer::Create(int width, int height) { + return rtc::make_ref_counted(width, height); +} + +// static +rtc::scoped_refptr I410Buffer::Create(int width, + int height, + int stride_y, + int stride_u, + int stride_v) { + return rtc::make_ref_counted(width, height, stride_y, stride_u, + stride_v); +} + +// static +rtc::scoped_refptr I410Buffer::Copy( + const I410BufferInterface& source) { + return Copy(source.width(), source.height(), source.DataY(), source.StrideY(), + source.DataU(), source.StrideU(), source.DataV(), + source.StrideV()); +} + +// static +rtc::scoped_refptr I410Buffer::Copy(int width, + int height, + const uint16_t* data_y, + int stride_y, + const uint16_t* data_u, + int stride_u, + const uint16_t* data_v, + int stride_v) { + // Note: May use different strides than the input data. + rtc::scoped_refptr buffer = Create(width, height); + int res = libyuv::I410Copy(data_y, stride_y, data_u, stride_u, data_v, + stride_v, buffer->MutableDataY(), + buffer->StrideY(), buffer->MutableDataU(), + buffer->StrideU(), buffer->MutableDataV(), + buffer->StrideV(), width, height); + RTC_DCHECK_EQ(res, 0); + + return buffer; +} + +// static +rtc::scoped_refptr I410Buffer::Rotate( + const I410BufferInterface& src, + VideoRotation rotation) { + RTC_CHECK(src.DataY()); + RTC_CHECK(src.DataU()); + RTC_CHECK(src.DataV()); + + int rotated_width = src.width(); + int rotated_height = src.height(); + if (rotation == webrtc::kVideoRotation_90 || + rotation == webrtc::kVideoRotation_270) { + std::swap(rotated_width, rotated_height); + } + + rtc::scoped_refptr buffer = + I410Buffer::Create(rotated_width, rotated_height); + + int res = libyuv::I410Rotate( + src.DataY(), src.StrideY(), src.DataU(), src.StrideU(), src.DataV(), + src.StrideV(), buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), buffer->MutableDataV(), + buffer->StrideV(), src.width(), src.height(), + static_cast(rotation)); + RTC_DCHECK_EQ(res, 0); + + return buffer; +} + +rtc::scoped_refptr I410Buffer::ToI420() { + rtc::scoped_refptr i420_buffer = + I420Buffer::Create(width(), height()); + int res = libyuv::I410ToI420( + DataY(), StrideY(), DataU(), StrideU(), DataV(), StrideV(), + i420_buffer->MutableDataY(), i420_buffer->StrideY(), + i420_buffer->MutableDataU(), i420_buffer->StrideU(), + i420_buffer->MutableDataV(), i420_buffer->StrideV(), width(), height()); + RTC_DCHECK_EQ(res, 0); + + return i420_buffer; +} + +void I410Buffer::InitializeData() { + memset(data_.get(), 0, + I410DataSize(height_, stride_y_, stride_u_, stride_v_)); +} + +int I410Buffer::width() const { + return width_; +} + +int I410Buffer::height() const { + return height_; +} + +const uint16_t* I410Buffer::DataY() const { + return data_.get(); +} +const uint16_t* I410Buffer::DataU() const { + return data_.get() + stride_y_ * height_; +} +const uint16_t* I410Buffer::DataV() const { + return data_.get() + stride_y_ * height_ + stride_u_ * height_; +} + +int I410Buffer::StrideY() const { + return stride_y_; +} +int I410Buffer::StrideU() const { + return stride_u_; +} +int I410Buffer::StrideV() const { + return stride_v_; +} + +uint16_t* I410Buffer::MutableDataY() { + return const_cast(DataY()); +} +uint16_t* I410Buffer::MutableDataU() { + return const_cast(DataU()); +} +uint16_t* I410Buffer::MutableDataV() { + return const_cast(DataV()); +} + +void I410Buffer::CropAndScaleFrom(const I410BufferInterface& src, + int offset_x, + int offset_y, + int crop_width, + int crop_height) { + RTC_CHECK_LE(crop_width, src.width()); + RTC_CHECK_LE(crop_height, src.height()); + RTC_CHECK_LE(crop_width + offset_x, src.width()); + RTC_CHECK_LE(crop_height + offset_y, src.height()); + RTC_CHECK_GE(offset_x, 0); + RTC_CHECK_GE(offset_y, 0); + + const uint16_t* y_plane = src.DataY() + src.StrideY() * offset_y + offset_x; + const uint16_t* u_plane = src.DataU() + src.StrideU() * offset_y + offset_x; + const uint16_t* v_plane = src.DataV() + src.StrideV() * offset_y + offset_x; + int res = libyuv::I444Scale_16( + y_plane, src.StrideY(), u_plane, src.StrideU(), v_plane, src.StrideV(), + crop_width, crop_height, MutableDataY(), StrideY(), MutableDataU(), + StrideU(), MutableDataV(), StrideV(), width(), height(), + libyuv::kFilterBox); + + RTC_DCHECK_EQ(res, 0); +} + +void I410Buffer::ScaleFrom(const I410BufferInterface& src) { + CropAndScaleFrom(src, 0, 0, src.width(), src.height()); +} + +} // namespace webrtc diff --git a/api/video/i410_buffer.h b/api/video/i410_buffer.h new file mode 100644 index 0000000000..1c0cd86c12 --- /dev/null +++ b/api/video/i410_buffer.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_I410_BUFFER_H_ +#define API_VIDEO_I410_BUFFER_H_ + +#include + +#include + +#include "api/scoped_refptr.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_rotation.h" +#include "rtc_base/memory/aligned_malloc.h" + +namespace webrtc { + +// Plain I410 (yuv 444 planar 10 bits) buffer in standard memory. +class RTC_EXPORT I410Buffer : public I410BufferInterface { + public: + static rtc::scoped_refptr Create(int width, int height); + static rtc::scoped_refptr Create(int width, + int height, + int stride_y, + int stride_u, + int stride_v); + + // Create a new buffer and copy the pixel data. + static rtc::scoped_refptr Copy(const I410BufferInterface& buffer); + + static rtc::scoped_refptr Copy(int width, + int height, + const uint16_t* data_y, + int stride_y, + const uint16_t* data_u, + int stride_u, + const uint16_t* data_v, + int stride_v); + + // Returns a rotated copy of |src|. + static rtc::scoped_refptr Rotate(const I410BufferInterface& src, + VideoRotation rotation); + + rtc::scoped_refptr ToI420() final; + const I420BufferInterface* GetI420() const final { return nullptr; } + + // Sets all three planes to all zeros. Used to work around for + // quirks in memory checkers + // (https://bugs.chromium.org/p/libyuv/issues/detail?id=377) and + // ffmpeg (http://crbug.com/390941). + // TODO(https://crbug.com/390941): Deprecated. Should be deleted if/when those + // issues are resolved in a better way. Or in the mean time, use SetBlack. + void InitializeData(); + + int width() const override; + int height() const override; + const uint16_t* DataY() const override; + const uint16_t* DataU() const override; + const uint16_t* DataV() const override; + + int StrideY() const override; + int StrideU() const override; + int StrideV() const override; + + uint16_t* MutableDataY(); + uint16_t* MutableDataU(); + uint16_t* MutableDataV(); + + // Scale the cropped area of |src| to the size of |this| buffer, and + // write the result into |this|. + void CropAndScaleFrom(const I410BufferInterface& src, + int offset_x, + int offset_y, + int crop_width, + int crop_height); + + // Scale all of `src` to the size of `this` buffer, with no cropping. + void ScaleFrom(const I410BufferInterface& src); + + protected: + I410Buffer(int width, int height); + I410Buffer(int width, int height, int stride_y, int stride_u, int stride_v); + + ~I410Buffer() override; + + private: + const int width_; + const int height_; + const int stride_y_; + const int stride_u_; + const int stride_v_; + const std::unique_ptr data_; +}; + +} // namespace webrtc + +#endif // API_VIDEO_I410_BUFFER_H_ diff --git a/api/video/test/BUILD.gn b/api/video/test/BUILD.gn index 3f8f3a0723..60ec4b852f 100644 --- a/api/video/test/BUILD.gn +++ b/api/video/test/BUILD.gn @@ -13,6 +13,7 @@ rtc_library("rtc_api_video_unittests") { sources = [ "color_space_unittest.cc", "i210_buffer_unittest.cc", + "i410_buffer_unittest.cc", "i422_buffer_unittest.cc", "i444_buffer_unittest.cc", "nv12_buffer_unittest.cc", diff --git a/api/video/test/i410_buffer_unittest.cc b/api/video/test/i410_buffer_unittest.cc new file mode 100644 index 0000000000..c5d2d5bf2d --- /dev/null +++ b/api/video/test/i410_buffer_unittest.cc @@ -0,0 +1,120 @@ + +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/i410_buffer.h" + +#include "api/video/i420_buffer.h" +#include "test/frame_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr uint16_t kYValue = 4; +constexpr uint16_t kUValue = 8; +constexpr uint16_t kVValue = 16; + +int GetY(rtc::scoped_refptr buf, int col, int row) { + return buf->DataY()[row * buf->StrideY() + col]; +} + +int GetU(rtc::scoped_refptr buf, int col, int row) { + return buf->DataU()[row * buf->StrideU() + col]; +} + +int GetV(rtc::scoped_refptr buf, int col, int row) { + return buf->DataV()[row * buf->StrideV() + col]; +} + +void FillI410Buffer(rtc::scoped_refptr buf) { + for (int row = 0; row < buf->height(); ++row) { + for (int col = 0; col < buf->width(); ++col) { + buf->MutableDataY()[row * buf->StrideY() + col] = kYValue; + buf->MutableDataU()[row * buf->StrideU() + col] = kUValue; + buf->MutableDataV()[row * buf->StrideV() + col] = kVValue; + } + } +} + +} // namespace + +TEST(I410BufferTest, InitialData) { + constexpr int stride = 3; + constexpr int width = 3; + constexpr int height = 3; + + rtc::scoped_refptr i410_buffer(I410Buffer::Create(width, height)); + EXPECT_EQ(width, i410_buffer->width()); + EXPECT_EQ(height, i410_buffer->height()); + EXPECT_EQ(stride, i410_buffer->StrideY()); + EXPECT_EQ(stride, i410_buffer->StrideU()); + EXPECT_EQ(stride, i410_buffer->StrideV()); + EXPECT_EQ(3, i410_buffer->ChromaWidth()); + EXPECT_EQ(3, i410_buffer->ChromaHeight()); +} + +TEST(I410BufferTest, ReadPixels) { + constexpr int width = 3; + constexpr int height = 3; + + rtc::scoped_refptr i410_buffer(I410Buffer::Create(width, height)); + FillI410Buffer(i410_buffer); + for (int row = 0; row < height; row++) { + for (int col = 0; col < width; col++) { + EXPECT_EQ(kYValue, GetY(i410_buffer, col, row)); + EXPECT_EQ(kUValue, GetU(i410_buffer, col, row)); + EXPECT_EQ(kVValue, GetV(i410_buffer, col, row)); + } + } +} + +TEST(I410BufferTest, ToI420) { + // libyuv I410ToI420 only handles correctly even sizes and skips last row/col + // if odd. + constexpr int width = 4; + constexpr int height = 4; + constexpr int size_y = width * height; + constexpr int size_u = (width + 1) / 2 * (height + 1) / 2; + constexpr int size_v = (width + 1) / 2 * (height + 1) / 2; + rtc::scoped_refptr reference(I420Buffer::Create(width, height)); + // I410 is 10-bit while I420 is 8 bit, so last 2 bits would be discarded. + memset(reference->MutableDataY(), kYValue >> 2, size_y); + memset(reference->MutableDataU(), kUValue >> 2, size_u); + memset(reference->MutableDataV(), kVValue >> 2, size_v); + + rtc::scoped_refptr i410_buffer(I410Buffer::Create(width, height)); + FillI410Buffer(i410_buffer); + + // Confirm YUV values are as expected. + for (int row = 0; row < height; row++) { + for (int col = 0; col < width; col++) { + EXPECT_EQ(kYValue, GetY(i410_buffer, col, row)); + EXPECT_EQ(kUValue, GetU(i410_buffer, col, row)); + EXPECT_EQ(kVValue, GetV(i410_buffer, col, row)); + } + } + + rtc::scoped_refptr i420_buffer(i410_buffer->ToI420()); + + // Confirm YUV values are as expected. + for (int row = 0; row < height; row++) { + for (int col = 0; col < width; col++) { + EXPECT_EQ(1, i420_buffer->DataY()[row * i420_buffer->StrideY() + col]); + } + } + + EXPECT_EQ(height, i420_buffer->height()); + EXPECT_EQ(width, i420_buffer->width()); + EXPECT_TRUE(test::FrameBufsEqual(reference, i420_buffer)); +} + +} // namespace webrtc diff --git a/api/video/video_frame_buffer.cc b/api/video/video_frame_buffer.cc index 398e30b606..374b438adc 100644 --- a/api/video/video_frame_buffer.cc +++ b/api/video/video_frame_buffer.cc @@ -63,6 +63,11 @@ const I210BufferInterface* VideoFrameBuffer::GetI210() const { return static_cast(this); } +const I410BufferInterface* VideoFrameBuffer::GetI410() const { + RTC_CHECK(type() == Type::kI410); + return static_cast(this); +} + const NV12BufferInterface* VideoFrameBuffer::GetNV12() const { RTC_CHECK(type() == Type::kNV12); return static_cast(this); @@ -94,6 +99,8 @@ const char* VideoFrameBufferTypeToString(VideoFrameBuffer::Type type) { return "kI010"; case VideoFrameBuffer::Type::kI210: return "kI210"; + case VideoFrameBuffer::Type::kI410: + return "kI410"; case VideoFrameBuffer::Type::kNV12: return "kNV12"; default: @@ -195,6 +202,18 @@ int I210BufferInterface::ChromaHeight() const { return height(); } +VideoFrameBuffer::Type I410BufferInterface::type() const { + return Type::kI410; +} + +int I410BufferInterface::ChromaWidth() const { + return width(); +} + +int I410BufferInterface::ChromaHeight() const { + return height(); +} + VideoFrameBuffer::Type NV12BufferInterface::type() const { return Type::kNV12; } diff --git a/api/video/video_frame_buffer.h b/api/video/video_frame_buffer.h index cf90ff22c1..aaf786699f 100644 --- a/api/video/video_frame_buffer.h +++ b/api/video/video_frame_buffer.h @@ -26,6 +26,7 @@ class I422BufferInterface; class I444BufferInterface; class I010BufferInterface; class I210BufferInterface; +class I410BufferInterface; class NV12BufferInterface; // Base class for frame buffers of different types of pixel format and storage. @@ -58,6 +59,7 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface { kI444, kI010, kI210, + kI410, kNV12, }; @@ -112,6 +114,7 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface { const I444BufferInterface* GetI444() const; const I010BufferInterface* GetI010() const; const I210BufferInterface* GetI210() const; + const I410BufferInterface* GetI410() const; const NV12BufferInterface* GetNV12() const; // From a kNative frame, returns a VideoFrameBuffer with a pixel format in @@ -261,6 +264,19 @@ class I210BufferInterface : public PlanarYuv16BBuffer { ~I210BufferInterface() override {} }; +// Represents Type::kI410, allocates 16 bits per pixel and fills 10 least +// significant bits with color information. +class I410BufferInterface : public PlanarYuv16BBuffer { + public: + Type type() const override; + + int ChromaWidth() const final; + int ChromaHeight() const final; + + protected: + ~I410BufferInterface() override {} +}; + class BiplanarYuvBuffer : public VideoFrameBuffer { public: virtual int ChromaWidth() const = 0; diff --git a/common_video/include/video_frame_buffer.h b/common_video/include/video_frame_buffer.h index 34a9bb5a37..1f6331b94d 100644 --- a/common_video/include/video_frame_buffer.h +++ b/common_video/include/video_frame_buffer.h @@ -99,6 +99,17 @@ rtc::scoped_refptr WrapI210Buffer( const uint16_t* v_plane, int v_stride, std::function no_longer_used); + +rtc::scoped_refptr WrapI410Buffer( + int width, + int height, + const uint16_t* y_plane, + int y_stride, + const uint16_t* u_plane, + int u_stride, + const uint16_t* v_plane, + int v_stride, + std::function no_longer_used); } // namespace webrtc #endif // COMMON_VIDEO_INCLUDE_VIDEO_FRAME_BUFFER_H_ diff --git a/common_video/include/video_frame_buffer_pool.h b/common_video/include/video_frame_buffer_pool.h index fd1bd164ec..3d94bc5669 100644 --- a/common_video/include/video_frame_buffer_pool.h +++ b/common_video/include/video_frame_buffer_pool.h @@ -18,6 +18,7 @@ #include "api/scoped_refptr.h" #include "api/video/i010_buffer.h" #include "api/video/i210_buffer.h" +#include "api/video/i410_buffer.h" #include "api/video/i420_buffer.h" #include "api/video/i422_buffer.h" #include "api/video/i444_buffer.h" @@ -50,6 +51,7 @@ class VideoFrameBufferPool { rtc::scoped_refptr CreateI444Buffer(int width, int height); rtc::scoped_refptr CreateI010Buffer(int width, int height); rtc::scoped_refptr CreateI210Buffer(int width, int height); + rtc::scoped_refptr CreateI410Buffer(int width, int height); rtc::scoped_refptr CreateNV12Buffer(int width, int height); // Changes the max amount of buffers in the pool to the new value. diff --git a/common_video/video_frame_buffer.cc b/common_video/video_frame_buffer.cc index d57330c652..ca2916e580 100644 --- a/common_video/video_frame_buffer.cc +++ b/common_video/video_frame_buffer.cc @@ -227,6 +227,22 @@ rtc::scoped_refptr I210BufferBase::ToI420() { return i420_buffer; } +class I410BufferBase : public I410BufferInterface { + public: + rtc::scoped_refptr ToI420() final; +}; + +rtc::scoped_refptr I410BufferBase::ToI420() { + rtc::scoped_refptr i420_buffer = + I420Buffer::Create(width(), height()); + libyuv::I410ToI420(DataY(), StrideY(), DataU(), StrideU(), DataV(), StrideV(), + i420_buffer->MutableDataY(), i420_buffer->StrideY(), + i420_buffer->MutableDataU(), i420_buffer->StrideU(), + i420_buffer->MutableDataV(), i420_buffer->StrideV(), + width(), height()); + return i420_buffer; +} + } // namespace rtc::scoped_refptr WrapI420Buffer( @@ -353,4 +369,20 @@ rtc::scoped_refptr WrapI210Buffer( v_stride, no_longer_used)); } +rtc::scoped_refptr WrapI410Buffer( + int width, + int height, + const uint16_t* y_plane, + int y_stride, + const uint16_t* u_plane, + int u_stride, + const uint16_t* v_plane, + int v_stride, + std::function no_longer_used) { + return rtc::scoped_refptr( + rtc::make_ref_counted>( + width, height, y_plane, y_stride, u_plane, u_stride, v_plane, + v_stride, no_longer_used)); +} + } // namespace webrtc diff --git a/common_video/video_frame_buffer_pool.cc b/common_video/video_frame_buffer_pool.cc index 7f695814f9..c0215110fd 100644 --- a/common_video/video_frame_buffer_pool.cc +++ b/common_video/video_frame_buffer_pool.cc @@ -44,6 +44,10 @@ bool HasOneRef(const rtc::scoped_refptr& buffer) { return static_cast*>(buffer.get()) ->HasOneRef(); } + case VideoFrameBuffer::Type::kI410: { + return static_cast*>(buffer.get()) + ->HasOneRef(); + } case VideoFrameBuffer::Type::kNV12: { return static_cast*>(buffer.get()) ->HasOneRef(); @@ -281,6 +285,33 @@ rtc::scoped_refptr VideoFrameBufferPool::CreateI210Buffer( return buffer; } +rtc::scoped_refptr VideoFrameBufferPool::CreateI410Buffer( + int width, + int height) { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + + rtc::scoped_refptr existing_buffer = + GetExistingBuffer(width, height, VideoFrameBuffer::Type::kI410); + if (existing_buffer) { + // Cast is safe because the only way kI410 buffer is created is + // in the same function below, where |RefCountedObject| + // is created. + rtc::RefCountedObject* raw_buffer = + static_cast*>(existing_buffer.get()); + // Creates a new scoped_refptr, which is also pointing to the same + // RefCountedObject as buffer, increasing ref count. + return rtc::scoped_refptr(raw_buffer); + } + + if (buffers_.size() >= max_number_of_buffers_) + return nullptr; + // Allocate new buffer. + rtc::scoped_refptr buffer = I410Buffer::Create(width, height); + + buffers_.push_back(buffer); + return buffer; +} + rtc::scoped_refptr VideoFrameBufferPool::GetExistingBuffer( int width, int height, diff --git a/common_video/video_frame_unittest.cc b/common_video/video_frame_unittest.cc index 15f07a9401..ae8e54e7d3 100644 --- a/common_video/video_frame_unittest.cc +++ b/common_video/video_frame_unittest.cc @@ -15,6 +15,7 @@ #include "api/video/i010_buffer.h" #include "api/video/i210_buffer.h" +#include "api/video/i410_buffer.h" #include "api/video/i420_buffer.h" #include "api/video/i422_buffer.h" #include "api/video/i444_buffer.h" @@ -47,6 +48,8 @@ SubSampling SubSamplingForType(VideoFrameBuffer::Type type) { return {.x = 2, .y = 2}; case VideoFrameBuffer::Type::kI210: return {.x = 2, .y = 1}; + case VideoFrameBuffer::Type::kI410: + return {.x = 1, .y = 1}; default: return {}; } @@ -299,7 +302,8 @@ rtc::scoped_refptr CreateAndFillBuffer() { auto buf = T::Create(20, 10); memset(buf->MutableDataY(), 1, 200); - if (buf->type() == VideoFrameBuffer::Type::kI444) { + if (buf->type() == VideoFrameBuffer::Type::kI444 || + buf->type() == VideoFrameBuffer::Type::kI410) { memset(buf->MutableDataU(), 2, 200); memset(buf->MutableDataV(), 3, 200); } else if (buf->type() == VideoFrameBuffer::Type::kI422 || @@ -384,8 +388,12 @@ REGISTER_TYPED_TEST_SUITE_P(TestPlanarYuvBuffer, CropYNotCenter, CropAndScale16x9); -using TestTypesAll = ::testing:: - Types; +using TestTypesAll = ::testing::Types; INSTANTIATE_TYPED_TEST_SUITE_P(All, TestPlanarYuvBuffer, TestTypesAll); template @@ -403,7 +411,8 @@ TYPED_TEST_P(TestPlanarYuvBufferScale, Scale) { REGISTER_TYPED_TEST_SUITE_P(TestPlanarYuvBufferScale, Scale); -using TestTypesScale = ::testing::Types; +using TestTypesScale = + ::testing::Types; INSTANTIATE_TYPED_TEST_SUITE_P(All, TestPlanarYuvBufferScale, TestTypesScale); template diff --git a/modules/video_coding/codecs/h264/h264_decoder_impl.cc b/modules/video_coding/codecs/h264/h264_decoder_impl.cc index c96a1c80c0..f67718cb23 100644 --- a/modules/video_coding/codecs/h264/h264_decoder_impl.cc +++ b/modules/video_coding/codecs/h264/h264_decoder_impl.cc @@ -39,10 +39,10 @@ namespace webrtc { namespace { -constexpr std::array kPixelFormatsSupported = { - AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, - AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P, - AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_YUV422P10LE}; +constexpr std::array kPixelFormatsSupported = { + AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, + AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P, + AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_YUV422P10LE, AV_PIX_FMT_YUV444P10LE}; const size_t kYPlaneIndex = 0; const size_t kUPlaneIndex = 1; const size_t kVPlaneIndex = 2; @@ -120,6 +120,7 @@ int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context, rtc::scoped_refptr i422_buffer; rtc::scoped_refptr i010_buffer; rtc::scoped_refptr i210_buffer; + rtc::scoped_refptr i410_buffer; int bytes_per_pixel = 1; switch (context->pix_fmt) { case AV_PIX_FMT_YUV420P: @@ -194,6 +195,22 @@ int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context, frame_buffer = i210_buffer; bytes_per_pixel = 2; break; + case AV_PIX_FMT_YUV444P10LE: + i410_buffer = + decoder->ffmpeg_buffer_pool_.CreateI410Buffer(width, height); + // Set `av_frame` members as required by FFmpeg. + av_frame->data[kYPlaneIndex] = + reinterpret_cast(i410_buffer->MutableDataY()); + av_frame->linesize[kYPlaneIndex] = i410_buffer->StrideY() * 2; + av_frame->data[kUPlaneIndex] = + reinterpret_cast(i410_buffer->MutableDataU()); + av_frame->linesize[kUPlaneIndex] = i410_buffer->StrideU() * 2; + av_frame->data[kVPlaneIndex] = + reinterpret_cast(i410_buffer->MutableDataV()); + av_frame->linesize[kVPlaneIndex] = i410_buffer->StrideV() * 2; + frame_buffer = i410_buffer; + bytes_per_pixel = 2; + break; default: RTC_LOG(LS_ERROR) << "Unsupported buffer type " << context->pix_fmt << ". Check supported supported pixel formats!"; @@ -424,6 +441,11 @@ int32_t H264DecoderImpl::Decode(const EncodedImage& input_image, planar_yuv16_buffer = reinterpret_cast( planar_yuv_buffer); break; + case VideoFrameBuffer::Type::kI410: + planar_yuv_buffer = frame_buffer->GetI410(); + planar_yuv16_buffer = reinterpret_cast( + planar_yuv_buffer); + break; default: // If this code is changed to allow other video frame buffer type, // make sure that the code below which wraps I420/I422/I444 buffer and @@ -469,7 +491,8 @@ int32_t H264DecoderImpl::Decode(const EncodedImage& input_image, break; } case VideoFrameBuffer::Type::kI010: - case VideoFrameBuffer::Type::kI210: { + case VideoFrameBuffer::Type::kI210: + case VideoFrameBuffer::Type::kI410: { RTC_DCHECK_GE( av_frame_->data[kYPlaneIndex], reinterpret_cast(planar_yuv16_buffer->DataY())); @@ -562,6 +585,18 @@ int32_t H264DecoderImpl::Decode(const EncodedImage& input_image, // To keep reference alive. [frame_buffer] {}); break; + case VideoFrameBuffer::Type::kI410: + cropped_buffer = WrapI410Buffer( + av_frame_->width, av_frame_->height, + reinterpret_cast(av_frame_->data[kYPlaneIndex]), + av_frame_->linesize[kYPlaneIndex] / 2, + reinterpret_cast(av_frame_->data[kUPlaneIndex]), + av_frame_->linesize[kUPlaneIndex] / 2, + reinterpret_cast(av_frame_->data[kVPlaneIndex]), + av_frame_->linesize[kVPlaneIndex] / 2, + // To keep reference alive. + [frame_buffer] {}); + break; default: RTC_LOG(LS_ERROR) << "frame_buffer type: " << static_cast(video_frame_buffer_type) diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.cc b/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.cc index 997139df06..a981f259cf 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.cc +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.cc @@ -328,6 +328,16 @@ int LibvpxVp9Decoder::ReturnFrame( reinterpret_cast(img->planes[VPX_PLANE_V]), img->stride[VPX_PLANE_V] / 2, [img_buffer] {}); break; + case VPX_IMG_FMT_I44416: + img_wrapped_buffer = WrapI410Buffer( + img->d_w, img->d_h, + reinterpret_cast(img->planes[VPX_PLANE_Y]), + img->stride[VPX_PLANE_Y] / 2, + reinterpret_cast(img->planes[VPX_PLANE_U]), + img->stride[VPX_PLANE_U] / 2, + reinterpret_cast(img->planes[VPX_PLANE_V]), + img->stride[VPX_PLANE_V] / 2, [img_buffer] {}); + break; default: RTC_LOG(LS_ERROR) << "Unsupported pixel format produced by the decoder: " << static_cast(img->fmt);