From 12e5d392cc8fc0ba7a04587c190daa4232e412bb Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Thu, 31 Jan 2019 13:01:35 +0100 Subject: [PATCH] Reland "Partial frame capture API part 1" Reland with fixes to undefined behavior. Define new optional struct in VideoFrame to signal that the frame is a changed part of a whole picture and add a flag to signal that partial update may be issued by the VideoFrame source. Also, fix too strict assumptions in FrameBuffers PasteFrom methods. Also, add ability to set a new buffer in video frame. Original Reviewed-on: https://webrtc-review.googlesource.com/c/120405 Bug: webrtc:10152 Change-Id: I85790dfc7cec2f23abfe9d6cd18dc76a0c343bc0 Reviewed-on: https://webrtc-review.googlesource.com/c/120780 Reviewed-by: Niels Moller Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#26493} --- api/video/i010_buffer.cc | 6 ++-- api/video/i420_buffer.cc | 6 ++-- api/video/video_frame.cc | 43 ++++++++++++++++++----- api/video/video_frame.h | 56 +++++++++++++++++++++++++----- api/video/video_source_interface.h | 3 ++ 5 files changed, 92 insertions(+), 22 deletions(-) diff --git a/api/video/i010_buffer.cc b/api/video/i010_buffer.cc index 8f4541f2ec..7286676ded 100644 --- a/api/video/i010_buffer.cc +++ b/api/video/i010_buffer.cc @@ -243,8 +243,10 @@ void I010Buffer::PasteFrom(const I010BufferInterface& picture, // Pasted picture has to be aligned so subsumpled UV plane isn't corrupted. RTC_CHECK(offset_col % 2 == 0); RTC_CHECK(offset_row % 2 == 0); - RTC_CHECK(picture.width() % 2 == 0); - RTC_CHECK(picture.height() % 2 == 0); + RTC_CHECK(picture.width() % 2 == 0 || + picture.width() + offset_col == width()); + RTC_CHECK(picture.height() % 2 == 0 || + picture.height() + offset_row == height()); libyuv::CopyPlane_16(picture.DataY(), picture.StrideY(), MutableDataY() + StrideY() * offset_row + offset_col, diff --git a/api/video/i420_buffer.cc b/api/video/i420_buffer.cc index ed45b98ae2..46de581e84 100644 --- a/api/video/i420_buffer.cc +++ b/api/video/i420_buffer.cc @@ -237,8 +237,10 @@ void I420Buffer::PasteFrom(const I420BufferInterface& picture, // Pasted picture has to be aligned so subsumpled UV plane isn't corrupted. RTC_CHECK(offset_col % 2 == 0); RTC_CHECK(offset_row % 2 == 0); - RTC_CHECK(picture.width() % 2 == 0); - RTC_CHECK(picture.height() % 2 == 0); + RTC_CHECK(picture.width() % 2 == 0 || + picture.width() + offset_col == width()); + RTC_CHECK(picture.height() % 2 == 0 || + picture.height() + offset_row == height()); libyuv::CopyPlane(picture.DataY(), picture.StrideY(), MutableDataY() + StrideY() * offset_row + offset_col, diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc index 75a30b28a9..03bbd71c42 100644 --- a/api/video/video_frame.cc +++ b/api/video/video_frame.cc @@ -21,7 +21,9 @@ VideoFrame::Builder::~Builder() = default; VideoFrame VideoFrame::Builder::build() { return VideoFrame(id_, video_frame_buffer_, timestamp_us_, timestamp_rtp_, - ntp_time_ms_, rotation_, color_space_); + ntp_time_ms_, rotation_, color_space_, + partial_frame_description_, + cache_buffer_for_partial_updates_); } VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer( @@ -76,6 +78,18 @@ VideoFrame::Builder& VideoFrame::Builder::set_id(uint16_t id) { return *this; } +VideoFrame::Builder& VideoFrame::Builder::set_partial_frame_description( + const absl::optional& description) { + partial_frame_description_ = description; + return *this; +} + +VideoFrame::Builder& VideoFrame::Builder::set_cache_buffer_for_partial_updates( + bool cache_buffer_for_partial_updates) { + cache_buffer_for_partial_updates_ = cache_buffer_for_partial_updates; + return *this; +} + VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, webrtc::VideoRotation rotation, int64_t timestamp_us) @@ -97,20 +111,25 @@ VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, RTC_DCHECK(buffer); } -VideoFrame::VideoFrame(uint16_t id, - const rtc::scoped_refptr& buffer, - int64_t timestamp_us, - uint32_t timestamp_rtp, - int64_t ntp_time_ms, - VideoRotation rotation, - const absl::optional& color_space) +VideoFrame::VideoFrame( + uint16_t id, + const rtc::scoped_refptr& buffer, + int64_t timestamp_us, + uint32_t timestamp_rtp, + int64_t ntp_time_ms, + VideoRotation rotation, + const absl::optional& color_space, + const absl::optional partial_frame_description, + bool cache_buffer_for_partial_updates) : id_(id), video_frame_buffer_(buffer), timestamp_rtp_(timestamp_rtp), ntp_time_ms_(ntp_time_ms), timestamp_us_(timestamp_us), rotation_(rotation), - color_space_(color_space) {} + color_space_(color_space), + partial_frame_description_(partial_frame_description), + cache_buffer_for_partial_updates_(cache_buffer_for_partial_updates) {} VideoFrame::~VideoFrame() = default; @@ -135,6 +154,12 @@ rtc::scoped_refptr VideoFrame::video_frame_buffer() const { return video_frame_buffer_; } +void VideoFrame::set_video_frame_buffer( + rtc::scoped_refptr buffer) { + RTC_CHECK(buffer.get()); + video_frame_buffer_ = buffer; +} + int64_t VideoFrame::render_time_ms() const { return timestamp_us() / rtc::kNumMicrosecsPerMillisec; } diff --git a/api/video/video_frame.h b/api/video/video_frame.h index de3f0efdc5..f83345ac8f 100644 --- a/api/video/video_frame.h +++ b/api/video/video_frame.h @@ -25,6 +25,14 @@ namespace webrtc { class RTC_EXPORT VideoFrame { public: + // Describes a partial frame, which contains only a changed region compared + // to a previous frame. Shouldn't be set on the fully updated picture. + struct PartialFrameDescription { + // Coordinates of top-left corner of the changed region in the full picture. + int offset_x; + int offset_y; + }; + // Preferred way of building VideoFrame objects. class Builder { public: @@ -42,6 +50,10 @@ class RTC_EXPORT VideoFrame { Builder& set_color_space(const ColorSpace& color_space); Builder& set_color_space(const ColorSpace* color_space); Builder& set_id(uint16_t id); + Builder& set_partial_frame_description( + const absl::optional& description); + Builder& set_cache_buffer_for_partial_updates( + bool cache_buffer_for_partial_updates); private: uint16_t id_ = 0; @@ -51,6 +63,8 @@ class RTC_EXPORT VideoFrame { int64_t ntp_time_ms_ = 0; VideoRotation rotation_ = kVideoRotation_0; absl::optional color_space_; + absl::optional partial_frame_description_; + bool cache_buffer_for_partial_updates_ = false; }; // To be deprecated. Migrate all use to Builder. @@ -134,13 +148,31 @@ class RTC_EXPORT VideoFrame { color_space ? absl::make_optional(*color_space) : absl::nullopt; } + const PartialFrameDescription* partial_frame_description() const { + return partial_frame_description_ ? &partial_frame_description_.value() + : nullptr; + } + void set_partial_frame_description( + const absl::optional& description) { + partial_frame_description_ = description; + } + + void set_cache_buffer_for_partial_updates( + bool cache_buffer_for_partial_updates) { + cache_buffer_for_partial_updates_ = cache_buffer_for_partial_updates; + } + bool cache_buffer_for_partial_updates() const { + return cache_buffer_for_partial_updates_; + } + // Get render time in milliseconds. // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). int64_t render_time_ms() const; - // Return the underlying buffer. Never nullptr for a properly - // initialized VideoFrame. + // Return the underlying buffer. This can only be a nullptr for a partial + // update VideoFrame with no changed pixels. rtc::scoped_refptr video_frame_buffer() const; + void set_video_frame_buffer(rtc::scoped_refptr buffer); // TODO(nisse): Deprecated. // Return true if the frame is stored in a texture. @@ -149,13 +181,16 @@ class RTC_EXPORT VideoFrame { } private: - VideoFrame(uint16_t id, - const rtc::scoped_refptr& buffer, - int64_t timestamp_us, - uint32_t timestamp_rtp, - int64_t ntp_time_ms, - VideoRotation rotation, - const absl::optional& color_space); + VideoFrame( + uint16_t id, + const rtc::scoped_refptr& buffer, + int64_t timestamp_us, + uint32_t timestamp_rtp, + int64_t ntp_time_ms, + VideoRotation rotation, + const absl::optional& color_space, + const absl::optional partial_frame_description, + bool cache_buffer_for_partial_updates_); uint16_t id_; // An opaque reference counted handle that stores the pixel data. @@ -165,6 +200,9 @@ class RTC_EXPORT VideoFrame { int64_t timestamp_us_; VideoRotation rotation_; absl::optional color_space_; + absl::optional partial_frame_description_; + // Should be set on all frames, if the source may produce partial updates. + bool cache_buffer_for_partial_updates_; }; } // namespace webrtc diff --git a/api/video/video_source_interface.h b/api/video/video_source_interface.h index 9d1641cd0d..e589b7f36d 100644 --- a/api/video/video_source_interface.h +++ b/api/video/video_source_interface.h @@ -42,6 +42,9 @@ struct RTC_EXPORT VideoSinkWants { absl::optional target_pixel_count; // Tells the source the maximum framerate the sink wants. int max_framerate_fps = std::numeric_limits::max(); + + // Tells the source that the sink supports partial frame updates. + bool partial_frames = false; }; template