From 126648763184b7e224d6c4a2f85efb4a9307378f Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Mon, 4 Feb 2019 10:11:05 +0100 Subject: [PATCH] Partial frame capture API part 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement utility for applying partial updates to video frames. Bug: webrtc:10152 Change-Id: I295fa9f792b96bbf1140a13f1f04e4f9deaccd5c Reviewed-on: https://webrtc-review.googlesource.com/c/120408 Commit-Queue: Ilya Nikolaevskiy Reviewed-by: Niels Moller Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#26522} --- api/video/video_frame.cc | 1 - video/BUILD.gn | 3 + video/partial_frame_assembler.cc | 96 ++++++++ video/partial_frame_assembler.h | 45 ++++ video/partial_frame_assembler_unittest.cc | 279 ++++++++++++++++++++++ 5 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 video/partial_frame_assembler.cc create mode 100644 video/partial_frame_assembler.h create mode 100644 video/partial_frame_assembler_unittest.cc diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc index 03bbd71c42..c91aeda124 100644 --- a/api/video/video_frame.cc +++ b/api/video/video_frame.cc @@ -156,7 +156,6 @@ rtc::scoped_refptr VideoFrame::video_frame_buffer() const { void VideoFrame::set_video_frame_buffer( rtc::scoped_refptr buffer) { - RTC_CHECK(buffer.get()); video_frame_buffer_ = buffer; } diff --git a/video/BUILD.gn b/video/BUILD.gn index 4da1e2358a..9874742414 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -171,6 +171,8 @@ rtc_source_set("video_stream_encoder_impl") { sources = [ "overuse_frame_detector.cc", "overuse_frame_detector.h", + "partial_frame_assembler.cc", + "partial_frame_assembler.h", "video_stream_encoder.cc", "video_stream_encoder.h", ] @@ -475,6 +477,7 @@ if (rtc_include_tests) { "end_to_end_tests/stats_tests.cc", "end_to_end_tests/transport_feedback_tests.cc", "overuse_frame_detector_unittest.cc", + "partial_frame_assembler_unittest.cc", "picture_id_tests.cc", "quality_scaling_tests.cc", "quality_threshold_unittest.cc", diff --git a/video/partial_frame_assembler.cc b/video/partial_frame_assembler.cc new file mode 100644 index 0000000000..e7d1c12211 --- /dev/null +++ b/video/partial_frame_assembler.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/partial_frame_assembler.h" + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" + +namespace webrtc { + +PartialFrameAssembler::PartialFrameAssembler() = default; +PartialFrameAssembler::~PartialFrameAssembler() = default; + +bool PartialFrameAssembler::ApplyPartialUpdate( + const rtc::scoped_refptr& input_buffer, + VideoFrame* uncompressed_frame, + const VideoFrame::PartialFrameDescription* partial_desc) { + const int changed_rect_width = input_buffer ? input_buffer->width() : 0; + const int changed_rect_height = input_buffer ? input_buffer->height() : 0; + if (partial_desc == nullptr) { + // Full update. Copy whole picture to the cached buffer. May need to + // resize or create the cache buffer. + if (!cached_frame_buffer_ || + cached_frame_buffer_->height() < input_buffer->height() || + cached_frame_buffer_->width() < input_buffer->width()) { + cached_frame_buffer_ = + I420Buffer::Create(input_buffer->width(), input_buffer->height()); + } + cached_frame_buffer_->PasteFrom(*input_buffer->ToI420().get(), 0, 0); + } else { + // Have to apply partial input picture to the cached buffer. + // Check all possible error situations. + if (!cached_frame_buffer_) { + RTC_LOG(LS_ERROR) << "Partial picture received but no cached full picture" + "present."; + return false; + } + if (partial_desc->offset_x % 2 != 0 || partial_desc->offset_y % 2 != 0) { + RTC_LOG(LS_ERROR) << "Partial picture required to be at even offset." + " Actual: (" + << partial_desc->offset_x << ", " + << partial_desc->offset_y << ")."; + cached_frame_buffer_ = nullptr; + return false; + } + if ((changed_rect_width % 2 != 0 && + changed_rect_width + partial_desc->offset_x < + cached_frame_buffer_->width()) || + (changed_rect_height % 2 != 0 && + changed_rect_height + partial_desc->offset_y < + cached_frame_buffer_->height())) { + RTC_LOG(LS_ERROR) << "Partial picture required to have even dimensions." + " Actual: " + << input_buffer->width() << "x" + << input_buffer->height() << "."; + cached_frame_buffer_ = nullptr; + return false; + } + if (partial_desc->offset_x < 0 || + partial_desc->offset_x + changed_rect_width > + cached_frame_buffer_->width() || + partial_desc->offset_y < 0 || + partial_desc->offset_y + changed_rect_height > + cached_frame_buffer_->height()) { + RTC_LOG(LS_ERROR) << "Partial picture is outside of bounds."; + cached_frame_buffer_ = nullptr; + return false; + } + // No errors: apply new image to the cache and use the result. + if (input_buffer) { + cached_frame_buffer_->PasteFrom(*input_buffer->ToI420().get(), + partial_desc->offset_x, + partial_desc->offset_y); + } + } + // Remove partial frame description, as it doesn't make sense after update + // is applied. + uncompressed_frame->set_partial_frame_description(absl::nullopt); + uncompressed_frame->set_video_frame_buffer( + I420Buffer::Copy(*cached_frame_buffer_.get())); + return true; +} + +void PartialFrameAssembler::Reset() { + cached_frame_buffer_ = nullptr; +} + +} // namespace webrtc diff --git a/video/partial_frame_assembler.h b/video/partial_frame_assembler.h new file mode 100644 index 0000000000..9106bf867b --- /dev/null +++ b/video/partial_frame_assembler.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_PARTIAL_FRAME_ASSEMBLER_H_ +#define VIDEO_PARTIAL_FRAME_ASSEMBLER_H_ + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" + +namespace webrtc { + +// Maintains cache of a full resolution frame buffer and applies partial +// updates to it. +// This class is not thread-safe. +class PartialFrameAssembler { + public: + PartialFrameAssembler(); + ~PartialFrameAssembler(); + + // Applies |input_buffer| to the cached buffer and sets buffer for + // |uncompresed_frame| to a full updated image. + // Returns false on any error. In that case the buffer will be invalidated + // and subsequent updates will also return error until full resolution frame + // is processed. + bool ApplyPartialUpdate( + const rtc::scoped_refptr& buffer, + VideoFrame* uncompressed_frame, + const VideoFrame::PartialFrameDescription* partial_desc); + + // Clears internal buffer. + void Reset(); + + private: + rtc::scoped_refptr cached_frame_buffer_; +}; + +} // namespace webrtc +#endif // VIDEO_PARTIAL_FRAME_ASSEMBLER_H_ diff --git a/video/partial_frame_assembler_unittest.cc b/video/partial_frame_assembler_unittest.cc new file mode 100644 index 0000000000..25b0f3e355 --- /dev/null +++ b/video/partial_frame_assembler_unittest.cc @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/partial_frame_assembler.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr uint8_t kCol1 = 100; +constexpr uint8_t kCol2 = 200; + +constexpr int kWidth = 640; +constexpr int kHeight = 480; + +rtc::scoped_refptr CreatePicture(int width, + int height, + uint8_t data) { + rtc::scoped_refptr buf = I420Buffer::Create(width, height); + for (int row = 0; row < height; ++row) { + for (int col = 0; col < width; ++col) { + int pos_y = row * buf->StrideY() + col; + int pos_u = row / 2 * buf->StrideU() + col / 2; + int pos_v = row / 2 * buf->StrideV() + col / 2; + buf->MutableDataY()[pos_y] = data; + buf->MutableDataU()[pos_u] = data; + buf->MutableDataV()[pos_v] = data; + } + } + return buf; +} + +VideoFrame CreateFrame(rtc::scoped_refptr buf) { + VideoFrame frame = VideoFrame(buf, VideoRotation::kVideoRotation_0, 0); + frame.set_cache_buffer_for_partial_updates(true); + return frame; +} + +bool TestPictureWithOneRect(rtc::scoped_refptr buf, + int offset_x, + int offset_y, + int rect_width, + int rect_height, + uint8_t in_rect_data, + uint8_t out_rect_data) { + for (int row = 0; row < buf->height(); ++row) { + for (int col = 0; col < buf->width(); ++col) { + int pos_y = row * buf->StrideY() + col; + int pos_u = row / 2 * buf->StrideU() + col / 2; + int pos_v = row / 2 * buf->StrideV() + col / 2; + uint8_t y = buf->DataY()[pos_y]; + uint8_t u = buf->DataU()[pos_u]; + uint8_t v = buf->DataV()[pos_v]; + bool in_rect = col >= offset_x && col < offset_x + rect_width && + row >= offset_y && row < offset_y + rect_height; + uint8_t expected_data = in_rect ? in_rect_data : out_rect_data; + if (y != expected_data || u != expected_data || v != expected_data) + return false; + } + } + return true; +} + +TEST(PartialFrameAssembler, FullPictureUpdates) { + PartialFrameAssembler decompressor; + + // Full pic. + auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Full pic. + auto full_pic2 = CreatePicture(kWidth, kHeight, kCol2); + VideoFrame frame2 = CreateFrame(full_pic2); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + EXPECT_TRUE(TestPictureWithOneRect(frame1.video_frame_buffer()->ToI420(), 0, + 0, -1, -1, kCol1, kCol1)); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, nullptr)); + EXPECT_TRUE(TestPictureWithOneRect(frame2.video_frame_buffer()->ToI420(), 0, + 0, -1, -1, kCol2, kCol2)); +} + +TEST(PartialFrameAssembler, PartialUpdateFirstFails) { + PartialFrameAssembler decompressor; + + // Partial update. + auto full_pic1 = CreatePicture(20, 20, kCol1); + VideoFrame::PartialFrameDescription desc1{0, 0}; + VideoFrame frame1 = CreateFrame(full_pic1); + + EXPECT_FALSE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, &desc1)); +} + +TEST(PartialFrameAssembler, PartialUpdate) { + PartialFrameAssembler decompressor; + + // Full pic. + auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Partial pic update. + auto full_pic2 = CreatePicture(10, 20, kCol2); + VideoFrame::PartialFrameDescription desc2{30, 40}; + VideoFrame frame2 = CreateFrame(full_pic2); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + EXPECT_TRUE(TestPictureWithOneRect(frame1.video_frame_buffer()->ToI420(), 0, + 0, -1, -1, kCol1, kCol1)); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2)); + EXPECT_TRUE(TestPictureWithOneRect(frame2.video_frame_buffer()->ToI420(), 30, + 40, 10, 20, kCol2, kCol1)); +} + +TEST(PartialFrameAssembler, ProcessesUnchangedUpdate) { + PartialFrameAssembler decompressor; + + auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Partial pic update. + auto full_pic2 = CreatePicture(10, 20, kCol2); + VideoFrame::PartialFrameDescription desc2{30, 40}; + VideoFrame frame2 = CreateFrame(full_pic2); + + // Empty update. + VideoFrame::PartialFrameDescription desc3{0, 0}; + VideoFrame frame3 = CreateFrame(nullptr); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2)); + + EXPECT_TRUE(decompressor.ApplyPartialUpdate(nullptr, &frame3, &desc3)); +} + +TEST(PartialFrameAssembler, PartialUpdateFailsForOddXOffset) { + PartialFrameAssembler decompressor; + + // Full pic. + auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Partial pic update. + auto full_pic2 = CreatePicture(10, 20, kCol2); + // Offset is odd. + VideoFrame::PartialFrameDescription desc2{31, 40}; + VideoFrame frame2 = CreateFrame(full_pic2); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + + EXPECT_FALSE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2)); +} + +TEST(PartialFrameAssembler, PartialUpdateFailsForOddYOffset) { + PartialFrameAssembler decompressor; + + // Full pic. + auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Partial pic update. + auto full_pic2 = CreatePicture(10, 20, kCol2); + // Offset is odd. + VideoFrame::PartialFrameDescription desc2{30, 41}; + VideoFrame frame2 = CreateFrame(full_pic2); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + + EXPECT_FALSE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2)); +} + +TEST(PartialFrameAssembler, PartialUpdateFailsForOddWidth) { + PartialFrameAssembler decompressor; + + // Full pic. + auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Partial pic update. Odd width. + auto full_pic2 = CreatePicture(11, 20, kCol2); + VideoFrame::PartialFrameDescription desc2{30, 40}; + VideoFrame frame2 = CreateFrame(full_pic2); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + + EXPECT_FALSE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2)); +} + +TEST(PartialFrameAssembler, PartialUpdateWorksForOddWidthAtTheEnd) { + PartialFrameAssembler decompressor; + + // Full pic. Odd resolution. + auto full_pic1 = CreatePicture(kWidth + 1, kHeight + 1, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Partial pic update. Odd width. + auto full_pic2 = CreatePicture(11, 11, kCol2); + VideoFrame::PartialFrameDescription desc2{kWidth + 1 - 11, kHeight + 1 - 11}; + VideoFrame frame2 = CreateFrame(full_pic2); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2)); +} + +TEST(PartialFrameAssembler, PartialUpdateFailsForOddNotAtEnd) { + PartialFrameAssembler decompressor; + + // Full pic. Odd resolution. + auto full_pic1 = CreatePicture(kWidth + 1, kHeight + 1, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Partial pic update. Odd width. + auto full_pic2 = CreatePicture(11, 11, kCol2); + VideoFrame::PartialFrameDescription desc2{kWidth + 1 - 11, kHeight + 1 - 20}; + VideoFrame frame2 = CreateFrame(full_pic2); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + + EXPECT_FALSE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, &desc2)); +} + +TEST(PartialFrameAssembler, FullPictureUpdatesCanChangeResolution) { + PartialFrameAssembler decompressor; + + // Full pic. + auto full_pic1 = CreatePicture(kWidth, kHeight, kCol1); + VideoFrame frame1 = CreateFrame(full_pic1); + + // Full pic. + auto full_pic2 = CreatePicture(kWidth + 100, kHeight + 100, kCol2); + VideoFrame frame2 = CreateFrame(full_pic2); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic1.get(), &frame1, nullptr)); + EXPECT_TRUE(TestPictureWithOneRect(frame1.video_frame_buffer()->ToI420(), 0, + 0, -1, -1, kCol1, kCol1)); + + EXPECT_EQ(frame1.video_frame_buffer()->width(), kWidth); + EXPECT_EQ(frame1.video_frame_buffer()->height(), kHeight); + + EXPECT_TRUE( + decompressor.ApplyPartialUpdate(full_pic2.get(), &frame2, nullptr)); + EXPECT_EQ(frame2.video_frame_buffer()->width(), kWidth + 100); + EXPECT_EQ(frame2.video_frame_buffer()->height(), kHeight + 100); +} + +} // namespace +} // namespace webrtc