diff --git a/api/video/video_frame_buffer.cc b/api/video/video_frame_buffer.cc index dd869f7747..b231745e11 100644 --- a/api/video/video_frame_buffer.cc +++ b/api/video/video_frame_buffer.cc @@ -27,6 +27,16 @@ rtc::scoped_refptr VideoFrameBuffer::GetI420() return static_cast(this); } +I420ABufferInterface* VideoFrameBuffer::GetI420A() { + RTC_CHECK(type() == Type::kI420A); + return static_cast(this); +} + +const I420ABufferInterface* VideoFrameBuffer::GetI420A() const { + RTC_CHECK(type() == Type::kI420A); + return static_cast(this); +} + I444BufferInterface* VideoFrameBuffer::GetI444() { RTC_CHECK(type() == Type::kI444); return static_cast(this); @@ -53,6 +63,10 @@ rtc::scoped_refptr I420BufferInterface::ToI420() { return this; } +VideoFrameBuffer::Type I420ABufferInterface::type() const { + return Type::kI420A; +} + VideoFrameBuffer::Type I444BufferInterface::type() const { return Type::kI444; } diff --git a/api/video/video_frame_buffer.h b/api/video/video_frame_buffer.h index c8d02a153f..e656edd7e6 100644 --- a/api/video/video_frame_buffer.h +++ b/api/video/video_frame_buffer.h @@ -19,6 +19,7 @@ namespace webrtc { class I420BufferInterface; +class I420ABufferInterface; class I444BufferInterface; // Base class for frame buffers of different types of pixel format and storage. @@ -44,6 +45,7 @@ class VideoFrameBuffer : public rtc::RefCountInterface { enum class Type { kNative, kI420, + kI420A, kI444, }; @@ -67,6 +69,8 @@ class VideoFrameBuffer : public rtc::RefCountInterface { // removed. rtc::scoped_refptr GetI420(); rtc::scoped_refptr GetI420() const; + I420ABufferInterface* GetI420A(); + const I420ABufferInterface* GetI420A() const; I444BufferInterface* GetI444(); const I444BufferInterface* GetI444() const; @@ -97,7 +101,7 @@ class PlanarYuvBuffer : public VideoFrameBuffer { class I420BufferInterface : public PlanarYuvBuffer { public: - Type type() const final; + Type type() const override; int ChromaWidth() const final; int ChromaHeight() const final; @@ -108,6 +112,16 @@ class I420BufferInterface : public PlanarYuvBuffer { ~I420BufferInterface() override {} }; +class I420ABufferInterface : public I420BufferInterface { + public: + Type type() const final; + virtual const uint8_t* DataA() const = 0; + virtual int StrideA() const = 0; + + protected: + ~I420ABufferInterface() override {} +}; + class I444BufferInterface : public PlanarYuvBuffer { public: Type type() const final; diff --git a/common_video/include/video_frame_buffer.h b/common_video/include/video_frame_buffer.h index bc9d4f94a4..312979b05d 100644 --- a/common_video/include/video_frame_buffer.h +++ b/common_video/include/video_frame_buffer.h @@ -78,6 +78,19 @@ rtc::scoped_refptr WrapI444Buffer( int v_stride, const rtc::Callback0& no_longer_used); +rtc::scoped_refptr WrapI420ABuffer( + int width, + int height, + const uint8_t* y_plane, + int y_stride, + const uint8_t* u_plane, + int u_stride, + const uint8_t* v_plane, + int v_stride, + const uint8_t* a_plane, + int a_stride, + const rtc::Callback0& no_longer_used); + rtc::scoped_refptr WrapYuvBuffer( VideoFrameBuffer::Type type, int width, diff --git a/common_video/video_frame_buffer.cc b/common_video/video_frame_buffer.cc index 19649cf68f..9c8955cbd7 100644 --- a/common_video/video_frame_buffer.cc +++ b/common_video/video_frame_buffer.cc @@ -96,6 +96,8 @@ class WrappedYuvBuffer : public Base { v_stride_(v_stride), no_longer_used_cb_(no_longer_used) {} + ~WrappedYuvBuffer() override { no_longer_used_cb_(); } + int width() const override { return width_; } int height() const override { return height_; } @@ -115,8 +117,6 @@ class WrappedYuvBuffer : public Base { private: friend class rtc::RefCountedObject; - ~WrappedYuvBuffer() override { no_longer_used_cb_(); } - const int width_; const int height_; const uint8_t* const y_plane_; @@ -128,6 +128,41 @@ class WrappedYuvBuffer : public Base { rtc::Callback0 no_longer_used_cb_; }; +// Template to implement a wrapped buffer for a I4??BufferInterface. +template +class WrappedYuvaBuffer : public WrappedYuvBuffer { + public: + WrappedYuvaBuffer(int width, + int height, + const uint8_t* y_plane, + int y_stride, + const uint8_t* u_plane, + int u_stride, + const uint8_t* v_plane, + int v_stride, + const uint8_t* a_plane, + int a_stride, + const rtc::Callback0& no_longer_used) + : WrappedYuvBuffer(width, + height, + y_plane, + y_stride, + u_plane, + u_stride, + v_plane, + v_stride, + no_longer_used), + a_plane_(a_plane), + a_stride_(a_stride) {} + + const uint8_t* DataA() const override { return a_plane_; } + int StrideA() const override { return a_stride_; } + + private: + const uint8_t* const a_plane_; + const int a_stride_; +}; + rtc::scoped_refptr WrapI420Buffer( int width, int height, @@ -144,6 +179,24 @@ rtc::scoped_refptr WrapI420Buffer( v_stride, no_longer_used)); } +rtc::scoped_refptr WrapI420ABuffer( + int width, + int height, + const uint8_t* y_plane, + int y_stride, + const uint8_t* u_plane, + int u_stride, + const uint8_t* v_plane, + int v_stride, + const uint8_t* a_plane, + int a_stride, + const rtc::Callback0& no_longer_used) { + return rtc::scoped_refptr( + new rtc::RefCountedObject>( + width, height, y_plane, y_stride, u_plane, u_stride, v_plane, + v_stride, a_plane, a_stride, no_longer_used)); +} + rtc::scoped_refptr WrapI444Buffer( int width, int height, diff --git a/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc b/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc index 5f8b4c5989..82e87d4cf3 100644 --- a/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc +++ b/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc @@ -11,6 +11,7 @@ #include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h" #include "api/video/i420_buffer.h" +#include "api/video/video_frame_buffer.h" #include "api/video_codecs/sdp_video_format.h" #include "common_video/include/video_frame.h" #include "common_video/include/video_frame_buffer.h" @@ -18,6 +19,11 @@ #include "rtc_base/keep_ref_until_done.h" #include "rtc_base/logging.h" +namespace { +void KeepBufferRefs(rtc::scoped_refptr, + rtc::scoped_refptr) {} +} // anonymous namespace + namespace webrtc { class StereoDecoderAdapter::AdapterDecodedImageCallback @@ -173,11 +179,22 @@ void StereoDecoderAdapter::MergeAlphaImages( VideoFrame* alpha_decodedImage, const rtc::Optional& alpha_decode_time_ms, const rtc::Optional& alpha_qp) { - // TODO(emircan): Merge the output and put in a VideoFrame container that can - // transport I420A. - decoded_complete_callback_->Decoded(*decodedImage, decode_time_ms, qp); - decoded_complete_callback_->Decoded(*alpha_decodedImage, alpha_decode_time_ms, - alpha_qp); + rtc::scoped_refptr yuv_buffer = + decodedImage->video_frame_buffer()->ToI420(); + rtc::scoped_refptr alpha_buffer = + alpha_decodedImage->video_frame_buffer()->ToI420(); + RTC_DCHECK_EQ(yuv_buffer->width(), alpha_buffer->width()); + RTC_DCHECK_EQ(yuv_buffer->height(), alpha_buffer->height()); + rtc::scoped_refptr merged_buffer = WrapI420ABuffer( + yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(), + yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(), + yuv_buffer->DataV(), yuv_buffer->StrideV(), alpha_buffer->DataY(), + alpha_buffer->StrideY(), + rtc::Bind(&KeepBufferRefs, yuv_buffer, alpha_buffer)); + + VideoFrame merged_image(merged_buffer, decodedImage->timestamp(), + 0 /* render_time_ms */, decodedImage->rotation()); + decoded_complete_callback_->Decoded(merged_image, decode_time_ms, qp); } } // namespace webrtc diff --git a/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc b/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc index 283273d21e..b449c6826e 100644 --- a/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc +++ b/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc @@ -83,28 +83,30 @@ int StereoEncoderAdapter::Encode(const VideoFrame& input_image, if (!encoded_complete_callback_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - - // TODO(emircan): Extract alpha and create an alpha frame with dummy planes. - // Since we don't have a way of transporting alpha yet, put a dummy output for - // alpha consisting of YXX. - - // Encode AXX - rtc::scoped_refptr yuva_buffer = - input_image.video_frame_buffer()->ToI420(); - rtc::scoped_refptr alpha_buffer( - new rtc::RefCountedObject( - input_image.width(), input_image.height(), yuva_buffer->DataY(), - yuva_buffer->StrideY(), stereo_dummy_planes_.data(), - yuva_buffer->StrideU(), stereo_dummy_planes_.data(), - yuva_buffer->StrideV(), - rtc::KeepRefUntilDone(input_image.video_frame_buffer()))); - VideoFrame alpha_image(alpha_buffer, input_image.timestamp(), - input_image.render_time_ms(), input_image.rotation()); - encoders_[kAXXStream]->Encode(alpha_image, codec_specific_info, frame_types); - // Encode YUV int rv = encoders_[kYUVStream]->Encode(input_image, codec_specific_info, frame_types); + if (rv) + return rv; + + const bool has_alpha = input_image.video_frame_buffer()->type() == + VideoFrameBuffer::Type::kI420A; + if (!has_alpha) + return rv; + + // Encode AXX + const I420ABufferInterface* yuva_buffer = + input_image.video_frame_buffer()->GetI420A(); + rtc::scoped_refptr alpha_buffer = + WrapI420Buffer(input_image.width(), input_image.height(), + yuva_buffer->DataA(), yuva_buffer->StrideA(), + stereo_dummy_planes_.data(), yuva_buffer->StrideU(), + stereo_dummy_planes_.data(), yuva_buffer->StrideV(), + rtc::KeepRefUntilDone(input_image.video_frame_buffer())); + VideoFrame alpha_image(alpha_buffer, input_image.timestamp(), + input_image.render_time_ms(), input_image.rotation()); + rv = encoders_[kAXXStream]->Encode(alpha_image, codec_specific_info, + frame_types); return rv; } @@ -158,6 +160,9 @@ EncodedImageCallback::Result StereoEncoderAdapter::OnEncodedImage( const EncodedImage& encodedImage, const CodecSpecificInfo* codecSpecificInfo, const RTPFragmentationHeader* fragmentation) { + if (stream_idx == kAXXStream) + return EncodedImageCallback::Result(EncodedImageCallback::Result::OK); + // TODO(emircan): Fill |codec_specific_info| with stereo parameters. encoded_complete_callback_->OnEncodedImage(encodedImage, codecSpecificInfo, fragmentation); diff --git a/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc b/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc index a8913e7e52..7c16620f1c 100644 --- a/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc +++ b/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc @@ -10,11 +10,14 @@ #include "api/test/mock_video_decoder_factory.h" #include "api/test/mock_video_encoder_factory.h" +#include "common_video/include/video_frame_buffer.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h" #include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h" #include "modules/video_coding/codecs/test/video_codec_test.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "rtc_base/keep_ref_until_done.h" +#include "rtc_base/ptr_util.h" using testing::_; using testing::Return; @@ -44,6 +47,18 @@ class TestStereoAdapter : public VideoCodecTest { return codec_settings; } + std::unique_ptr CreateI420AInputFrame() { + rtc::scoped_refptr yuv_buffer = + input_frame_->video_frame_buffer()->ToI420(); + rtc::scoped_refptr yuva_buffer = WrapI420ABuffer( + yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(), + yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(), + yuv_buffer->DataV(), yuv_buffer->StrideV(), yuv_buffer->DataY(), + yuv_buffer->StrideY(), rtc::KeepRefUntilDone(yuv_buffer)); + return rtc::WrapUnique( + new VideoFrame(yuva_buffer, kVideoRotation_0, 0)); + } + private: void SetUp() override { EXPECT_CALL(*decoder_factory_, Die()); @@ -93,4 +108,20 @@ TEST_F(TestStereoAdapter, EncodeDecodeI420Frame) { EXPECT_GT(I420PSNR(input_frame_.get(), decoded_frame.get()), 36); } +TEST_F(TestStereoAdapter, EncodeDecodeI420AFrame) { + std::unique_ptr yuva_frame = CreateI420AInputFrame(); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(*yuva_frame, nullptr, nullptr)); + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + decoder_->Decode(encoded_frame, false, nullptr)); + std::unique_ptr decoded_frame; + rtc::Optional decoded_qp; + ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); + ASSERT_TRUE(decoded_frame); + EXPECT_GT(I420PSNR(yuva_frame.get(), decoded_frame.get()), 36); +} + } // namespace webrtc