[PCLF] Add video writer which accounts for freezes

Bug: b/237997865
Change-Id: I6d6e3faa48e6bddbe298ead7b1350dd3c70481b2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/268545
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37543}

test/BUILD.gn

@@ -380,6 +380,23 @@ rtc_source_set("test_support") {
   ]
 }
 
+rtc_library("fixed_fps_video_frame_writer_adapter") {
+  visibility = [ "*" ]
+  testonly = true
+  sources = [
+    "testsupport/fixed_fps_video_frame_writer_adapter.cc",
+    "testsupport/fixed_fps_video_frame_writer_adapter.h",
+  ]
+  deps = [
+    ":video_test_support",
+    "../api/units:time_delta",
+    "../api/video:video_frame",
+    "../rtc_base:checks",
+    "../system_wrappers",
+  ]
+  absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
 rtc_library("video_test_support") {
   testonly = true
 
@@ -530,6 +547,22 @@ if (rtc_include_tests && !build_with_chromium) {
     }
   }
 
+  rtc_library("fixed_fps_video_frame_writer_adapter_test") {
+    testonly = true
+    sources = [ "testsupport/fixed_fps_video_frame_writer_adapter_test.cc" ]
+    deps = [
+      ":fixed_fps_video_frame_writer_adapter",
+      ":test_support",
+      ":video_test_support",
+      "../api/units:time_delta",
+      "../api/units:timestamp",
+      "../api/video:video_frame",
+      "../rtc_base/synchronization:mutex",
+      "time_controller",
+    ]
+    absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+  }
+
   rtc_test("test_support_unittests") {
     deps = [
       ":call_config_utils",
@@ -538,6 +571,7 @@ if (rtc_include_tests && !build_with_chromium) {
       ":fake_video_codecs",
       ":fileutils",
       ":fileutils_unittests",
+      ":fixed_fps_video_frame_writer_adapter_test",
       ":frame_generator_impl",
       ":perf_test",
       ":rtc_expect_death",

test/testsupport/fixed_fps_video_frame_writer_adapter.cc

@@ -0,0 +1,114 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h"
+
+#include <cmath>
+#include <utility>
+
+#include "absl/types/optional.h"
+#include "api/units/time_delta.h"
+#include "api/video/video_sink_interface.h"
+#include "rtc_base/checks.h"
+#include "test/testsupport/video_frame_writer.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+constexpr TimeDelta kOneSecond = TimeDelta::Seconds(1);
+
+}  // namespace
+
+FixedFpsVideoFrameWriterAdapter::FixedFpsVideoFrameWriterAdapter(
+    int fps,
+    Clock* clock,
+    std::unique_ptr<VideoFrameWriter> delegate)
+    : inter_frame_interval_(kOneSecond / fps),
+      clock_(clock),
+      delegate_(std::move(delegate)) {}
+
+FixedFpsVideoFrameWriterAdapter::~FixedFpsVideoFrameWriterAdapter() {
+  Close();
+}
+
+void FixedFpsVideoFrameWriterAdapter::Close() {
+  if (is_closed_) {
+    return;
+  }
+  is_closed_ = true;
+  if (!last_frame_.has_value()) {
+    return;
+  }
+  Timestamp now = Now();
+  RTC_CHECK(WriteMissedSlotsExceptLast(now));
+  RTC_CHECK(delegate_->WriteFrame(*last_frame_));
+  delegate_->Close();
+}
+
+bool FixedFpsVideoFrameWriterAdapter::WriteFrame(const VideoFrame& frame) {
+  RTC_CHECK(!is_closed_);
+  Timestamp now = Now();
+  if (!last_frame_.has_value()) {
+    RTC_CHECK(!last_frame_time_.IsFinite());
+    last_frame_ = frame;
+    last_frame_time_ = now;
+    return true;
+  }
+
+  RTC_CHECK(last_frame_time_.IsFinite());
+
+  if (last_frame_time_ > now) {
+    // New frame was recevied before expected time "slot" for current
+    // `last_frame_` came => just replace current `last_frame_` with
+    // received `frame`.
+    RTC_CHECK_LE(last_frame_time_ - now, inter_frame_interval_ / 2);
+    last_frame_ = frame;
+    return true;
+  }
+
+  if (!WriteMissedSlotsExceptLast(now)) {
+    return false;
+  }
+
+  if (now - last_frame_time_ < inter_frame_interval_ / 2) {
+    // New frame was received closer to the expected time "slot" for current
+    // `last_frame_` than to the next "slot" => just replace current
+    // `last_frame_` with received `frame`.
+    last_frame_ = frame;
+    return true;
+  }
+
+  if (!delegate_->WriteFrame(*last_frame_)) {
+    return false;
+  }
+  last_frame_ = frame;
+  last_frame_time_ = last_frame_time_ + inter_frame_interval_;
+  return true;
+}
+
+bool FixedFpsVideoFrameWriterAdapter::WriteMissedSlotsExceptLast(
+    Timestamp now) {
+  RTC_CHECK(last_frame_time_.IsFinite());
+  while (now - last_frame_time_ > inter_frame_interval_) {
+    if (!delegate_->WriteFrame(*last_frame_)) {
+      return false;
+    }
+    last_frame_time_ = last_frame_time_ + inter_frame_interval_;
+  }
+  return true;
+}
+
+Timestamp FixedFpsVideoFrameWriterAdapter::Now() const {
+  return clock_->CurrentTime();
+}
+
+}  // namespace test
+}  // namespace webrtc

test/testsupport/fixed_fps_video_frame_writer_adapter.h

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_TESTSUPPORT_FIXED_FPS_VIDEO_FRAME_WRITER_ADAPTER_H_
+#define TEST_TESTSUPPORT_FIXED_FPS_VIDEO_FRAME_WRITER_ADAPTER_H_
+
+#include <memory>
+
+#include "absl/types/optional.h"
+#include "api/video/video_sink_interface.h"
+#include "system_wrappers/include/clock.h"
+#include "test/testsupport/video_frame_writer.h"
+
+namespace webrtc {
+namespace test {
+
+// Writes video to the specified video writer with specified fixed frame rate.
+// If at the point in time X no new frames are passed to the writer, the
+// previous frame is used to fill the gap and preserve frame rate.
+//
+// This adaptor uses next algorithm:
+// There are output "slots" at a fixed frame rate (starting at the time of the
+// first received frame). Each incoming frame is assigned to the closest output
+// slot. Then empty slots are filled by repeating the closest filled slot before
+// empty one. If there are multiple frames closest to the same slot, the latest
+// received one is used.
+//
+// The frames are outputted for the whole duration of the class life after the
+// first frame was written or until it will be closed.
+//
+// For example if frames from A to F were received, then next output sequence
+// will be generated:
+// Received frames:  A            B       C      D  EF        Destructor called
+//                   |            |       |      |  ||          |
+//                   v            v       v      v  vv          v
+//                   X----X----X----X----X----X----X----X----X----+----+--
+//                   |    |    |    |    |    |    |    |    |
+// Produced frames:  A    A    A    B    C    C    F    F    F
+//
+// This class is not thread safe.
+class FixedFpsVideoFrameWriterAdapter : public VideoFrameWriter {
+ public:
+  FixedFpsVideoFrameWriterAdapter(int fps,
+                                  Clock* clock,
+                                  std::unique_ptr<VideoFrameWriter> delegate);
+  ~FixedFpsVideoFrameWriterAdapter() override;
+
+  bool WriteFrame(const webrtc::VideoFrame& frame) override;
+
+  // Closes adapter and underlying delegate. User mustn't call to the WriteFrame
+  // after calling this method.
+  void Close() override;
+
+ private:
+  // Writes `last_frame_` for each "slot" from `last_frame_time_` up to now
+  // excluding the last one.
+  // Updates `last_frame_time_` to the position of the last NOT WRITTEN frame.
+  // Returns true if all writes were successful, otherwise retuns false. In such
+  // case it is not guranteed how many frames were actually written.
+  bool WriteMissedSlotsExceptLast(Timestamp now);
+  Timestamp Now() const;
+
+  // Because `TimeDelta` stores time with microseconds precision
+  // `last_frame_time_` may have a small drift and for very long streams it
+  // must be updated to use double for time.
+  const TimeDelta inter_frame_interval_;
+  Clock* const clock_;
+  std::unique_ptr<VideoFrameWriter> delegate_;
+  bool is_closed_ = false;
+
+  // Expected time slot for the last frame.
+  Timestamp last_frame_time_ = Timestamp::MinusInfinity();
+  absl::optional<VideoFrame> last_frame_ = absl::nullopt;
+};
+
+}  // namespace test
+}  // namespace webrtc
+
+#endif  // TEST_TESTSUPPORT_FIXED_FPS_VIDEO_FRAME_WRITER_ADAPTER_H_

test/testsupport/fixed_fps_video_frame_writer_adapter_test.cc

@@ -0,0 +1,320 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h"
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/testsupport/video_frame_writer.h"
+#include "test/time_controller/simulated_time_controller.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+constexpr TimeDelta kOneSecond = TimeDelta::Seconds(1);
+
+using ::testing::ElementsAre;
+
+class InMemoryVideoWriter : public VideoFrameWriter {
+ public:
+  ~InMemoryVideoWriter() override = default;
+
+  bool WriteFrame(const webrtc::VideoFrame& frame) override {
+    MutexLock lock(&mutex_);
+    received_frames_.push_back(frame);
+    return true;
+  }
+
+  void Close() override {}
+
+  std::vector<VideoFrame> received_frames() const {
+    MutexLock lock(&mutex_);
+    return received_frames_;
+  }
+
+ private:
+  mutable Mutex mutex_;
+  std::vector<VideoFrame> received_frames_ RTC_GUARDED_BY(mutex_);
+};
+
+VideoFrame EmptyFrameWithId(uint16_t frame_id) {
+  return VideoFrame::Builder()
+      .set_video_frame_buffer(I420Buffer::Create(1, 1))
+      .set_id(frame_id)
+      .build();
+}
+
+std::vector<uint16_t> FrameIds(const std::vector<VideoFrame>& frames) {
+  std::vector<uint16_t> out;
+  for (const VideoFrame& frame : frames) {
+    out.push_back(frame.id());
+  }
+  return out;
+}
+
+std::unique_ptr<TimeController> CreateSimulatedTimeController() {
+  // Using an offset of 100000 to get nice fixed width and readable
+  // timestamps in typical test scenarios.
+  const Timestamp kSimulatedStartTime = Timestamp::Seconds(100000);
+  return std::make_unique<GlobalSimulatedTimeController>(kSimulatedStartTime);
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     WhenWrittenWithSameFpsVideoIsCorrect) {
+  auto time_controller = CreateSimulatedTimeController();
+  int fps = 25;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(fps, time_controller->GetClock(),
+                                               std::move(inmemory_writer));
+
+  for (int i = 1; i <= 30; ++i) {
+    video_writer.WriteFrame(EmptyFrameWithId(i));
+    time_controller->AdvanceTime(kOneSecond / fps);
+  }
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(
+      FrameIds(received_frames),
+      ElementsAre(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+                  19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest, FrameIsRepeatedWhenThereIsAFreeze) {
+  auto time_controller = CreateSimulatedTimeController();
+  int fps = 25;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(fps, time_controller->GetClock(),
+                                               std::move(inmemory_writer));
+
+  // Write 10 frames
+  for (int i = 1; i <= 10; ++i) {
+    video_writer.WriteFrame(EmptyFrameWithId(i));
+    time_controller->AdvanceTime(kOneSecond / fps);
+  }
+
+  // Freeze for 4 frames
+  time_controller->AdvanceTime(4 * kOneSecond / fps);
+
+  // Write 10 more frames
+  for (int i = 11; i <= 20; ++i) {
+    video_writer.WriteFrame(EmptyFrameWithId(i));
+    time_controller->AdvanceTime(kOneSecond / fps);
+  }
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames),
+              ElementsAre(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 10, 11, 12,
+                          13, 14, 15, 16, 17, 18, 19, 20));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest, NoFramesWritten) {
+  auto time_controller = CreateSimulatedTimeController();
+  int fps = 25;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(fps, time_controller->GetClock(),
+                                               std::move(inmemory_writer));
+  time_controller->AdvanceTime(TimeDelta::Millis(100));
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  ASSERT_TRUE(received_frames.empty());
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     FreezeInTheMiddleAndNewFrameReceivedBeforeMiddleOfExpectedInterval) {
+  auto time_controller = CreateSimulatedTimeController();
+  constexpr int kFps = 10;
+  constexpr TimeDelta kInterval = kOneSecond / kFps;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(
+      kFps, time_controller->GetClock(), std::move(inmemory_writer));
+  video_writer.WriteFrame(EmptyFrameWithId(1));
+  time_controller->AdvanceTime(2.3 * kInterval);
+  video_writer.WriteFrame(EmptyFrameWithId(2));
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames), ElementsAre(1, 1, 2));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     FreezeInTheMiddleAndNewFrameReceivedAfterMiddleOfExpectedInterval) {
+  auto time_controller = CreateSimulatedTimeController();
+  constexpr int kFps = 10;
+  constexpr TimeDelta kInterval = kOneSecond / kFps;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(
+      kFps, time_controller->GetClock(), std::move(inmemory_writer));
+  video_writer.WriteFrame(EmptyFrameWithId(1));
+  time_controller->AdvanceTime(2.5 * kInterval);
+  video_writer.WriteFrame(EmptyFrameWithId(2));
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames), ElementsAre(1, 1, 1, 2));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     NewFrameReceivedBeforeMiddleOfExpectedInterval) {
+  auto time_controller = CreateSimulatedTimeController();
+  constexpr int kFps = 10;
+  constexpr TimeDelta kInterval = kOneSecond / kFps;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(
+      kFps, time_controller->GetClock(), std::move(inmemory_writer));
+  video_writer.WriteFrame(EmptyFrameWithId(1));
+  time_controller->AdvanceTime(0.3 * kInterval);
+  video_writer.WriteFrame(EmptyFrameWithId(2));
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames), ElementsAre(2));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     NewFrameReceivedAfterMiddleOfExpectedInterval) {
+  auto time_controller = CreateSimulatedTimeController();
+  constexpr int kFps = 10;
+  constexpr TimeDelta kInterval = kOneSecond / kFps;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(
+      kFps, time_controller->GetClock(), std::move(inmemory_writer));
+  video_writer.WriteFrame(EmptyFrameWithId(1));
+  time_controller->AdvanceTime(0.5 * kInterval);
+  video_writer.WriteFrame(EmptyFrameWithId(2));
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames), ElementsAre(1, 2));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     FreeezeAtTheEndAndDestroyBeforeMiddleOfExpectedInterval) {
+  auto time_controller = CreateSimulatedTimeController();
+  constexpr int kFps = 10;
+  constexpr TimeDelta kInterval = kOneSecond / kFps;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(
+      kFps, time_controller->GetClock(), std::move(inmemory_writer));
+  video_writer.WriteFrame(EmptyFrameWithId(1));
+  time_controller->AdvanceTime(2.3 * kInterval);
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames), ElementsAre(1, 1, 1));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     FreeezeAtTheEndAndDestroyAfterMiddleOfExpectedInterval) {
+  auto time_controller = CreateSimulatedTimeController();
+  constexpr int kFps = 10;
+  constexpr TimeDelta kInterval = kOneSecond / kFps;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(
+      kFps, time_controller->GetClock(), std::move(inmemory_writer));
+  video_writer.WriteFrame(EmptyFrameWithId(1));
+  time_controller->AdvanceTime(2.5 * kInterval);
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames), ElementsAre(1, 1, 1));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     DestroyBeforeMiddleOfExpectedInterval) {
+  auto time_controller = CreateSimulatedTimeController();
+  constexpr int kFps = 10;
+  constexpr TimeDelta kInterval = kOneSecond / kFps;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(
+      kFps, time_controller->GetClock(), std::move(inmemory_writer));
+  video_writer.WriteFrame(EmptyFrameWithId(1));
+  time_controller->AdvanceTime(0.3 * kInterval);
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames), ElementsAre(1));
+}
+
+TEST(FixedFpsVideoFrameWriterAdapterTest,
+     DestroyAfterMiddleOfExpectedInterval) {
+  auto time_controller = CreateSimulatedTimeController();
+  constexpr int kFps = 10;
+  constexpr TimeDelta kInterval = kOneSecond / kFps;
+
+  auto inmemory_writer = std::make_unique<InMemoryVideoWriter>();
+  InMemoryVideoWriter* inmemory_writer_ref = inmemory_writer.get();
+
+  FixedFpsVideoFrameWriterAdapter video_writer(
+      kFps, time_controller->GetClock(), std::move(inmemory_writer));
+  video_writer.WriteFrame(EmptyFrameWithId(1));
+  time_controller->AdvanceTime(0.5 * kInterval);
+  video_writer.Close();
+
+  std::vector<VideoFrame> received_frames =
+      inmemory_writer_ref->received_frames();
+  EXPECT_THAT(FrameIds(received_frames), ElementsAre(1));
+}
+
+}  // namespace
+}  // namespace test
+}  // namespace webrtc