Reland "Partial frame capture API part 1"

Reland with fixes to undefined behavior.

Define new optional struct in VideoFrame to signal that the frame is a
changed part of a whole picture and add a flag to signal that partial
update may be issued by the VideoFrame source.

Also, fix too strict assumptions in FrameBuffers PasteFrom methods.
Also, add ability to set a new buffer in video frame.

Original Reviewed-on: https://webrtc-review.googlesource.com/c/120405

Bug: webrtc:10152
Change-Id: I85790dfc7cec2f23abfe9d6cd18dc76a0c343bc0
Reviewed-on: https://webrtc-review.googlesource.com/c/120780
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26493}

api/video/i010_buffer.cc

@@ -243,8 +243,10 @@ void I010Buffer::PasteFrom(const I010BufferInterface& picture,
   // Pasted picture has to be aligned so subsumpled UV plane isn't corrupted.
   RTC_CHECK(offset_col % 2 == 0);
   RTC_CHECK(offset_row % 2 == 0);
-  RTC_CHECK(picture.width() % 2 == 0);
+  RTC_CHECK(picture.width() % 2 == 0 ||
-  RTC_CHECK(picture.height() % 2 == 0);
+            picture.width() + offset_col == width());
+  RTC_CHECK(picture.height() % 2 == 0 ||
+            picture.height() + offset_row == height());
 
   libyuv::CopyPlane_16(picture.DataY(), picture.StrideY(),
                        MutableDataY() + StrideY() * offset_row + offset_col,

api/video/i420_buffer.cc

@@ -237,8 +237,10 @@ void I420Buffer::PasteFrom(const I420BufferInterface& picture,
   // Pasted picture has to be aligned so subsumpled UV plane isn't corrupted.
   RTC_CHECK(offset_col % 2 == 0);
   RTC_CHECK(offset_row % 2 == 0);
-  RTC_CHECK(picture.width() % 2 == 0);
+  RTC_CHECK(picture.width() % 2 == 0 ||
-  RTC_CHECK(picture.height() % 2 == 0);
+            picture.width() + offset_col == width());
+  RTC_CHECK(picture.height() % 2 == 0 ||
+            picture.height() + offset_row == height());
 
   libyuv::CopyPlane(picture.DataY(), picture.StrideY(),
                     MutableDataY() + StrideY() * offset_row + offset_col,

api/video/video_frame.cc

@@ -21,7 +21,9 @@ VideoFrame::Builder::~Builder() = default;
 
 VideoFrame VideoFrame::Builder::build() {
   return VideoFrame(id_, video_frame_buffer_, timestamp_us_, timestamp_rtp_,
-                    ntp_time_ms_, rotation_, color_space_);
+                    ntp_time_ms_, rotation_, color_space_,
+                    partial_frame_description_,
+                    cache_buffer_for_partial_updates_);
 }
 
 VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer(
@@ -76,6 +78,18 @@ VideoFrame::Builder& VideoFrame::Builder::set_id(uint16_t id) {
   return *this;
 }
 
+VideoFrame::Builder& VideoFrame::Builder::set_partial_frame_description(
+    const absl::optional<PartialFrameDescription>& description) {
+  partial_frame_description_ = description;
+  return *this;
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_cache_buffer_for_partial_updates(
+    bool cache_buffer_for_partial_updates) {
+  cache_buffer_for_partial_updates_ = cache_buffer_for_partial_updates;
+  return *this;
+}
+
 VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
                        webrtc::VideoRotation rotation,
                        int64_t timestamp_us)
@@ -97,20 +111,25 @@ VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
   RTC_DCHECK(buffer);
 }
 
-VideoFrame::VideoFrame(uint16_t id,
+VideoFrame::VideoFrame(
+    uint16_t id,
     const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
     int64_t timestamp_us,
     uint32_t timestamp_rtp,
     int64_t ntp_time_ms,
     VideoRotation rotation,
-                       const absl::optional<ColorSpace>& color_space)
+    const absl::optional<ColorSpace>& color_space,
+    const absl::optional<PartialFrameDescription> partial_frame_description,
+    bool cache_buffer_for_partial_updates)
     : id_(id),
       video_frame_buffer_(buffer),
       timestamp_rtp_(timestamp_rtp),
       ntp_time_ms_(ntp_time_ms),
       timestamp_us_(timestamp_us),
       rotation_(rotation),
-      color_space_(color_space) {}
+      color_space_(color_space),
+      partial_frame_description_(partial_frame_description),
+      cache_buffer_for_partial_updates_(cache_buffer_for_partial_updates) {}
 
 VideoFrame::~VideoFrame() = default;
 
@@ -135,6 +154,12 @@ rtc::scoped_refptr<VideoFrameBuffer> VideoFrame::video_frame_buffer() const {
   return video_frame_buffer_;
 }
 
+void VideoFrame::set_video_frame_buffer(
+    rtc::scoped_refptr<VideoFrameBuffer> buffer) {
+  RTC_CHECK(buffer.get());
+  video_frame_buffer_ = buffer;
+}
+
 int64_t VideoFrame::render_time_ms() const {
   return timestamp_us() / rtc::kNumMicrosecsPerMillisec;
 }

api/video/video_frame.h

@@ -25,6 +25,14 @@ namespace webrtc {
 
 class RTC_EXPORT VideoFrame {
  public:
+  // Describes a partial frame, which contains only a changed region compared
+  // to a previous frame. Shouldn't be set on the fully updated picture.
+  struct PartialFrameDescription {
+    // Coordinates of top-left corner of the changed region in the full picture.
+    int offset_x;
+    int offset_y;
+  };
+
   // Preferred way of building VideoFrame objects.
   class Builder {
    public:
@@ -42,6 +50,10 @@ class RTC_EXPORT VideoFrame {
     Builder& set_color_space(const ColorSpace& color_space);
     Builder& set_color_space(const ColorSpace* color_space);
     Builder& set_id(uint16_t id);
+    Builder& set_partial_frame_description(
+        const absl::optional<PartialFrameDescription>& description);
+    Builder& set_cache_buffer_for_partial_updates(
+        bool cache_buffer_for_partial_updates);
 
    private:
     uint16_t id_ = 0;
@@ -51,6 +63,8 @@ class RTC_EXPORT VideoFrame {
     int64_t ntp_time_ms_ = 0;
     VideoRotation rotation_ = kVideoRotation_0;
     absl::optional<ColorSpace> color_space_;
+    absl::optional<PartialFrameDescription> partial_frame_description_;
+    bool cache_buffer_for_partial_updates_ = false;
   };
 
   // To be deprecated. Migrate all use to Builder.
@@ -134,13 +148,31 @@ class RTC_EXPORT VideoFrame {
         color_space ? absl::make_optional(*color_space) : absl::nullopt;
   }
 
+  const PartialFrameDescription* partial_frame_description() const {
+    return partial_frame_description_ ? &partial_frame_description_.value()
+                                      : nullptr;
+  }
+  void set_partial_frame_description(
+      const absl::optional<PartialFrameDescription>& description) {
+    partial_frame_description_ = description;
+  }
+
+  void set_cache_buffer_for_partial_updates(
+      bool cache_buffer_for_partial_updates) {
+    cache_buffer_for_partial_updates_ = cache_buffer_for_partial_updates;
+  }
+  bool cache_buffer_for_partial_updates() const {
+    return cache_buffer_for_partial_updates_;
+  }
+
   // Get render time in milliseconds.
   // TODO(nisse): Deprecated. Migrate all users to timestamp_us().
   int64_t render_time_ms() const;
 
-  // Return the underlying buffer. Never nullptr for a properly
+  // Return the underlying buffer. This can only be a nullptr for a partial
-  // initialized VideoFrame.
+  // update VideoFrame with no changed pixels.
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const;
+  void set_video_frame_buffer(rtc::scoped_refptr<VideoFrameBuffer> buffer);
 
   // TODO(nisse): Deprecated.
   // Return true if the frame is stored in a texture.
@@ -149,13 +181,16 @@ class RTC_EXPORT VideoFrame {
   }
 
  private:
-  VideoFrame(uint16_t id,
+  VideoFrame(
+      uint16_t id,
       const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
       int64_t timestamp_us,
       uint32_t timestamp_rtp,
       int64_t ntp_time_ms,
       VideoRotation rotation,
-             const absl::optional<ColorSpace>& color_space);
+      const absl::optional<ColorSpace>& color_space,
+      const absl::optional<PartialFrameDescription> partial_frame_description,
+      bool cache_buffer_for_partial_updates_);
 
   uint16_t id_;
   // An opaque reference counted handle that stores the pixel data.
@@ -165,6 +200,9 @@ class RTC_EXPORT VideoFrame {
   int64_t timestamp_us_;
   VideoRotation rotation_;
   absl::optional<ColorSpace> color_space_;
+  absl::optional<PartialFrameDescription> partial_frame_description_;
+  // Should be set on all frames, if the source may produce partial updates.
+  bool cache_buffer_for_partial_updates_;
 };
 
 }  // namespace webrtc

api/video/video_source_interface.h

@@ -42,6 +42,9 @@ struct RTC_EXPORT VideoSinkWants {
   absl::optional<int> target_pixel_count;
   // Tells the source the maximum framerate the sink wants.
   int max_framerate_fps = std::numeric_limits<int>::max();
+
+  // Tells the source that the sink supports partial frame updates.
+  bool partial_frames = false;
 };
 
 template <typename VideoFrameT>