From e6eded31e642b3b986fef478315603b5f398c227 Mon Sep 17 00:00:00 2001 From: Markus Handell Date: Fri, 15 Nov 2019 16:18:21 +0100 Subject: [PATCH] VideoFrame: Store a reference to an encoded frame Enable webrtc::VideoFrame to store a reference to an encoded frame. Bug: chromium:1013590 Change-Id: Id5a06f1c7249f104dfd328f08677cf8001958f0d Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/158788 Reviewed-by: Stefan Holmer Reviewed-by: Philip Eliasson Reviewed-by: Ilya Nikolaevskiy Reviewed-by: Niels Moller Commit-Queue: Markus Handell Cr-Commit-Position: refs/heads/master@{#29809} --- api/video/BUILD.gn | 1 + api/video/DEPS | 1 + api/video/video_frame.cc | 40 +++++++++++++++++++++------- api/video/video_frame.h | 37 +++++++++++++++++++++++-- common_video/BUILD.gn | 1 + common_video/video_frame_unittest.cc | 33 +++++++++++++++++++++++ 6 files changed, 101 insertions(+), 12 deletions(-) diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 1473229247..41698f089b 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -47,6 +47,7 @@ rtc_library("video_frame") { deps = [ ":video_rtp_headers", "..:array_view", + "..:refcountedbase", "..:rtp_packet_info", "..:scoped_refptr", "../../rtc_base:checks", diff --git a/api/video/DEPS b/api/video/DEPS index 3f5df957f8..555f7e148a 100644 --- a/api/video/DEPS +++ b/api/video/DEPS @@ -19,6 +19,7 @@ specific_include_rules = { ], "video_frame\.h": [ + "+rtc_base/ref_count.h", ], "video_frame_buffer\.h": [ diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc index 0e6a611dd8..63902af3d5 100644 --- a/api/video/video_frame.cc +++ b/api/video/video_frame.cc @@ -68,7 +68,7 @@ VideoFrame VideoFrame::Builder::build() { RTC_CHECK(video_frame_buffer_ != nullptr); return VideoFrame(id_, video_frame_buffer_, timestamp_us_, timestamp_rtp_, ntp_time_ms_, rotation_, color_space_, update_rect_, - packet_infos_); + packet_infos_, encoded_frame_buffer_); } VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer( @@ -135,6 +135,13 @@ VideoFrame::Builder& VideoFrame::Builder::set_packet_infos( return *this; } +VideoFrame::Builder& VideoFrame::Builder::set_encoded_video_frame_buffer( + rtc::scoped_refptr + encoded_frame_buffer) { + encoded_frame_buffer_ = std::move(encoded_frame_buffer); + return *this; +} + VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, webrtc::VideoRotation rotation, int64_t timestamp_us) @@ -156,17 +163,20 @@ VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, RTC_DCHECK(buffer); } -VideoFrame::VideoFrame(uint16_t id, - const rtc::scoped_refptr& buffer, - int64_t timestamp_us, - uint32_t timestamp_rtp, - int64_t ntp_time_ms, - VideoRotation rotation, - const absl::optional& color_space, - const absl::optional& update_rect, - RtpPacketInfos packet_infos) +VideoFrame::VideoFrame( + uint16_t id, + const rtc::scoped_refptr& buffer, + int64_t timestamp_us, + uint32_t timestamp_rtp, + int64_t ntp_time_ms, + VideoRotation rotation, + const absl::optional& color_space, + const absl::optional& update_rect, + RtpPacketInfos packet_infos, + const rtc::scoped_refptr& encoded_frame_buffer) : id_(id), video_frame_buffer_(buffer), + encoded_frame_buffer_(encoded_frame_buffer), timestamp_rtp_(timestamp_rtp), ntp_time_ms_(ntp_time_ms), timestamp_us_(timestamp_us), @@ -215,4 +225,14 @@ int64_t VideoFrame::render_time_ms() const { return timestamp_us() / rtc::kNumMicrosecsPerMillisec; } +void VideoFrame::set_encoded_video_frame_buffer( + rtc::scoped_refptr encoded_frame_buffer) { + encoded_frame_buffer_ = std::move(encoded_frame_buffer); +} + +rtc::scoped_refptr +VideoFrame::encoded_video_frame_buffer() const { + return encoded_frame_buffer_; +} + } // namespace webrtc diff --git a/api/video/video_frame.h b/api/video/video_frame.h index 7c512ca5be..338e2fdbd1 100644 --- a/api/video/video_frame.h +++ b/api/video/video_frame.h @@ -13,16 +13,20 @@ #include +#include #include #include "absl/types/optional.h" +#include "api/array_view.h" #include "api/rtp_packet_infos.h" #include "api/scoped_refptr.h" #include "api/video/color_space.h" #include "api/video/hdr_metadata.h" +#include "api/video/video_codec_type.h" #include "api/video/video_frame_buffer.h" #include "api/video/video_rotation.h" #include "rtc_base/checks.h" +#include "rtc_base/ref_count.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -47,6 +51,23 @@ class RTC_EXPORT VideoFrame { bool IsEmpty() const; }; + // Interface for accessing elements of the encoded frame that was the base for + // the rest of the VideoFrame. + class EncodedVideoFrameBuffer : public rtc::RefCountInterface { + public: + // Returns a span of the bitstream data. + virtual rtc::ArrayView data() const = 0; + + // Returns the colorspace of the encoded frame, or nullptr if not present + virtual const webrtc::ColorSpace* color_space() const = 0; + + // Returns the codec of the encoded frame + virtual VideoCodecType codec() const = 0; + + // Returns wether the encoded frame is a keyframe + virtual bool is_key_frame() const = 0; + }; + // Preferred way of building VideoFrame objects. class RTC_EXPORT Builder { public: @@ -66,6 +87,8 @@ class RTC_EXPORT VideoFrame { Builder& set_id(uint16_t id); Builder& set_update_rect(const UpdateRect& update_rect); Builder& set_packet_infos(RtpPacketInfos packet_infos); + Builder& set_encoded_video_frame_buffer( + rtc::scoped_refptr encoded_frame_buffer); private: uint16_t id_ = 0; @@ -77,6 +100,7 @@ class RTC_EXPORT VideoFrame { absl::optional color_space_; absl::optional update_rect_; RtpPacketInfos packet_infos_; + rtc::scoped_refptr encoded_frame_buffer_; }; // To be deprecated. Migrate all use to Builder. @@ -166,6 +190,12 @@ class RTC_EXPORT VideoFrame { void set_video_frame_buffer( const rtc::scoped_refptr& buffer); + void set_encoded_video_frame_buffer( + rtc::scoped_refptr encoded_frame_buffer); + + rtc::scoped_refptr encoded_video_frame_buffer() + const; + // TODO(nisse): Deprecated. // Return true if the frame is stored in a texture. bool is_texture() const { @@ -207,11 +237,14 @@ class RTC_EXPORT VideoFrame { VideoRotation rotation, const absl::optional& color_space, const absl::optional& update_rect, - RtpPacketInfos packet_infos); + RtpPacketInfos packet_infos, + const rtc::scoped_refptr& encoded_frame); uint16_t id_; - // An opaque reference counted handle that stores the pixel data. + // A reference counted handle that stores the pixel data. rtc::scoped_refptr video_frame_buffer_; + // A reference counted handle that points to an encoded frame + rtc::scoped_refptr encoded_frame_buffer_; uint32_t timestamp_rtp_; int64_t ntp_time_ms_; int64_t timestamp_us_; diff --git a/common_video/BUILD.gn b/common_video/BUILD.gn index d22e4fddea..bd440efa94 100644 --- a/common_video/BUILD.gn +++ b/common_video/BUILD.gn @@ -99,6 +99,7 @@ if (rtc_include_tests) { "../:webrtc_common", "../api:scoped_refptr", "../api/units:time_delta", + "../api/video:encoded_frame", "../api/video:video_frame", "../api/video:video_frame_i010", "../api/video:video_frame_i420", diff --git a/common_video/video_frame_unittest.cc b/common_video/video_frame_unittest.cc index 57298d1175..f7a27be747 100644 --- a/common_video/video_frame_unittest.cc +++ b/common_video/video_frame_unittest.cc @@ -362,6 +362,39 @@ TEST(TestVideoFrame, TextureInitialValues) { EXPECT_EQ(20, frame.timestamp_us()); } +class TestEncodedFrame : public VideoFrame::EncodedVideoFrameBuffer { + public: + rtc::ArrayView data() const override { + return rtc::ArrayView(); + } + webrtc::ColorSpace* color_space() const override { return nullptr; } + VideoCodecType codec() const override { return kVideoCodecGeneric; } + bool is_key_frame() const { return false; } +}; + +TEST(TestVideoFrame, AcceptsEncodedFrameSource) { + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(10, 10, 10, 14, 90)) + .build(); + EXPECT_EQ(frame.encoded_video_frame_buffer(), nullptr); + auto encoded_frame = new rtc::RefCountedObject(); + frame.set_encoded_video_frame_buffer(encoded_frame); + EXPECT_EQ(frame.encoded_video_frame_buffer(), encoded_frame); +} + +TEST(TestVideoFrame, CopiesWithSameEncodedFrameSource) { + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(10, 10, 10, 14, 90)) + .set_encoded_video_frame_buffer( + new rtc::RefCountedObject()) + .build(); + VideoFrame frame2 = frame; + EXPECT_EQ(frame.encoded_video_frame_buffer().get(), + frame2.encoded_video_frame_buffer().get()); +} + class TestPlanarYuvBuffer : public ::testing::TestWithParam {};