VideoCodecTextFixture and YuvFrameReader improvements.

Adds ability to specify desired frame size separate from actual clip
resolution, as well as clip and desired fps.
This allows e.g. reading an HD clip but running benchmarks in VGA, and
to specify e.g. 60fps for the clip but 30for encoding where frame
dropping kicks in so that motion is actually correct rather than just
plaing the clip slowly.

Bug: webrtc:12229
Change-Id: I4ad4fcc335611a449dc2723ffafbec6731e89f55
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195324
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32839}

api/test/videocodec_test_fixture.h

@@ -88,6 +88,17 @@ class VideoCodecTestFixture {
 
     // Plain name of YUV file to process without file extension.
     std::string filename;
+    // Dimensions of test clip. Falls back to (codec_settings.width/height) if
+    // not set.
+    absl::optional<int> clip_width;
+    absl::optional<int> clip_height;
+    // Framerate of input clip. Defaults to 30fps if not set.
+    absl::optional<int> clip_fps;
+
+    // The resolution at which psnr/ssim comparisons should be made. Frames
+    // will be scaled to this size if different.
+    absl::optional<int> reference_width;
+    absl::optional<int> reference_height;
 
     // File to process. This must be a video file in the YUV format.
     std::string filepath;

modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc

@@ -58,7 +58,7 @@ using VideoStatistics = VideoCodecTestStats::VideoStatistics;
 namespace {
 const int kBaseKeyFrameInterval = 3000;
 const double kBitratePriority = 1.0;
-const int kMaxFramerateFps = 30;
+const int kDefaultMaxFramerateFps = 30;
 const int kMaxQp = 56;
 
 void ConfigureSimulcast(VideoCodec* codec_settings) {
@@ -86,7 +86,7 @@ void ConfigureSvc(VideoCodec* codec_settings) {
   RTC_CHECK_EQ(kVideoCodecVP9, codec_settings->codecType);
 
   const std::vector<SpatialLayer> layers = GetSvcConfig(
-      codec_settings->width, codec_settings->height, kMaxFramerateFps,
+      codec_settings->width, codec_settings->height, kDefaultMaxFramerateFps,
       /*first_active_layer=*/0, codec_settings->VP9()->numberOfSpatialLayers,
       codec_settings->VP9()->numberOfTemporalLayers,
       /* is_screen_sharing = */ false);
@@ -646,10 +646,16 @@ void VideoCodecTestFixtureImpl::SetUpAndInitObjects(
   config_.codec_settings.startBitrate = static_cast<int>(initial_bitrate_kbps);
   config_.codec_settings.maxFramerate = std::ceil(initial_framerate_fps);
 
+  int clip_width = config_.clip_width.value_or(config_.codec_settings.width);
+  int clip_height = config_.clip_height.value_or(config_.codec_settings.height);
+
   // Create file objects for quality analysis.
-  source_frame_reader_.reset(
+  source_frame_reader_.reset(new YuvFrameReaderImpl(
-      new YuvFrameReaderImpl(config_.filepath, config_.codec_settings.width,
+      config_.filepath, clip_width, clip_height,
-                             config_.codec_settings.height));
+      config_.reference_width.value_or(clip_width),
+      config_.reference_height.value_or(clip_height),
+      YuvFrameReaderImpl::RepeatMode::kPingPong, config_.clip_fps,
+      config_.codec_settings.maxFramerate));
   EXPECT_TRUE(source_frame_reader_->Init());
 
   RTC_DCHECK(encoded_frame_writers_.empty());

modules/video_coding/codecs/test/videoprocessor.cc

@@ -251,8 +251,28 @@ void VideoProcessor::ProcessFrame() {
     if (input_frames_.size() == kMaxBufferedInputFrames) {
       input_frames_.erase(input_frames_.begin());
     }
+
+    if (config_.reference_width != -1 && config_.reference_height != -1 &&
+        (input_frame.width() != config_.reference_width ||
+         input_frame.height() != config_.reference_height)) {
+      rtc::scoped_refptr<I420Buffer> scaled_buffer = I420Buffer::Create(
+          config_.codec_settings.width, config_.codec_settings.height);
+      scaled_buffer->ScaleFrom(*input_frame.video_frame_buffer()->ToI420());
+
+      VideoFrame scaled_reference_frame = input_frame;
+      scaled_reference_frame.set_video_frame_buffer(scaled_buffer);
+      input_frames_.emplace(frame_number, scaled_reference_frame);
+
+      if (config_.reference_width == config_.codec_settings.width &&
+          config_.reference_height == config_.codec_settings.height) {
+        // Both encoding and comparison uses the same down-scale factor, reuse
+        // it for encoder below.
+        input_frame = scaled_reference_frame;
+      }
+    } else {
       input_frames_.emplace(frame_number, input_frame);
     }
+  }
   last_inputed_timestamp_ = timestamp;
 
   post_encode_time_ns_ = 0;
@@ -271,6 +291,14 @@ void VideoProcessor::ProcessFrame() {
     frame_stat->encode_start_ns = encode_start_ns;
   }
 
+  if (input_frame.width() != config_.codec_settings.width ||
+      input_frame.height() != config_.codec_settings.height) {
+    rtc::scoped_refptr<I420Buffer> scaled_buffer = I420Buffer::Create(
+        config_.codec_settings.width, config_.codec_settings.height);
+    scaled_buffer->ScaleFrom(*input_frame.video_frame_buffer()->ToI420());
+    input_frame.set_video_frame_buffer(scaled_buffer);
+  }
+
   // Encode.
   const std::vector<VideoFrameType> frame_types =
       (frame_number == 0)

test/testsupport/frame_reader.h

@@ -15,6 +15,7 @@
 
 #include <string>
 
+#include "absl/types/optional.h"
 #include "api/scoped_refptr.h"
 
 namespace webrtc {
@@ -47,11 +48,32 @@ class FrameReader {
 
 class YuvFrameReaderImpl : public FrameReader {
  public:
+  enum class RepeatMode { kSingle, kRepeat, kPingPong };
+  class DropperUtil {
+   public:
+    DropperUtil(int source_fps, int target_fps);
+
+    enum class DropDecision { kDropframe, kKeepFrame };
+    DropDecision UpdateLevel();
+
+   private:
+    const double frame_size_buckets_;
+    double bucket_level_;
+  };
+
   // Creates a file handler. The input file is assumed to exist and be readable.
   // Parameters:
   //   input_filename          The file to read from.
   //   width, height           Size of each frame to read.
   YuvFrameReaderImpl(std::string input_filename, int width, int height);
+  YuvFrameReaderImpl(std::string input_filename,
+                     int input_width,
+                     int input_height,
+                     int desired_width,
+                     int desired_height,
+                     RepeatMode repeat_mode,
+                     absl::optional<int> clip_fps,
+                     int target_fps);
   ~YuvFrameReaderImpl() override;
   bool Init() override;
   rtc::scoped_refptr<I420Buffer> ReadFrame() override;
@@ -63,9 +85,15 @@ class YuvFrameReaderImpl : public FrameReader {
   const std::string input_filename_;
   // It is not const, so subclasses will be able to add frame header size.
   size_t frame_length_in_bytes_;
-  const int width_;
+  const int input_width_;
-  const int height_;
+  const int input_height_;
+  const int desired_width_;
+  const int desired_height_;
+  const size_t frame_size_bytes_;
+  const RepeatMode repeat_mode_;
   int number_of_frames_;
+  int current_frame_index_;
+  std::unique_ptr<DropperUtil> dropper_;
   FILE* input_file_;
 };
 

test/testsupport/y4m_frame_reader.cc

@@ -40,9 +40,9 @@ Y4mFrameReaderImpl::~Y4mFrameReaderImpl() {
 }
 
 bool Y4mFrameReaderImpl::Init() {
-  if (width_ <= 0 || height_ <= 0) {
+  if (input_width_ <= 0 || input_height_ <= 0) {
-    fprintf(stderr, "Frame width and height must be >0, was %d x %d\n", width_,
+    fprintf(stderr, "Frame width and height must be >0, was %d x %d\n",
-            height_);
+            input_width_, input_height_);
     return false;
   }
   input_file_ = fopen(input_filename_.c_str(), "rb");

test/testsupport/yuv_frame_reader.cc

@@ -20,16 +20,64 @@
 
 namespace webrtc {
 namespace test {
+size_t FrameSizeBytes(int width, int height) {
+  int half_width = (width + 1) / 2;
+  size_t size_y = static_cast<size_t>(width) * height;
+  size_t size_uv = static_cast<size_t>(half_width) * ((height + 1) / 2);
+  return size_y + 2 * size_uv;
+}
+
+YuvFrameReaderImpl::DropperUtil::DropperUtil(int source_fps, int target_fps)
+    : frame_size_buckets_(
+          std::max(1.0, static_cast<double>(source_fps) / target_fps)),
+      bucket_level_(0.0) {}
+
+YuvFrameReaderImpl::DropperUtil::DropDecision
+YuvFrameReaderImpl::DropperUtil::UpdateLevel() {
+  DropDecision decision;
+  if (bucket_level_ <= 0.0) {
+    decision = DropDecision::kKeepFrame;
+    bucket_level_ += frame_size_buckets_;
+  } else {
+    decision = DropDecision::kDropframe;
+  }
+  bucket_level_ -= 1.0;
+  return decision;
+}
 
 YuvFrameReaderImpl::YuvFrameReaderImpl(std::string input_filename,
                                        int width,
                                        int height)
+    : YuvFrameReaderImpl(input_filename,
+                         width,
+                         height,
+                         width,
+                         height,
+                         RepeatMode::kSingle,
+                         30,
+                         30) {}
+YuvFrameReaderImpl::YuvFrameReaderImpl(std::string input_filename,
+                                       int input_width,
+                                       int input_height,
+                                       int desired_width,
+                                       int desired_height,
+                                       RepeatMode repeat_mode,
+                                       absl::optional<int> clip_fps,
+                                       int target_fps)
     : input_filename_(input_filename),
-      frame_length_in_bytes_(width * height +
+      frame_length_in_bytes_(input_width * input_height +
-                             2 * ((width + 1) / 2) * ((height + 1) / 2)),
+                             2 * ((input_width + 1) / 2) *
-      width_(width),
+                                 ((input_height + 1) / 2)),
-      height_(height),
+      input_width_(input_width),
+      input_height_(input_height),
+      desired_width_(desired_width),
+      desired_height_(desired_height),
+      frame_size_bytes_(FrameSizeBytes(input_width, input_height)),
+      repeat_mode_(repeat_mode),
       number_of_frames_(-1),
+      current_frame_index_(-1),
+      dropper_(clip_fps.has_value() ? new DropperUtil(*clip_fps, target_fps)
+                                    : nullptr),
       input_file_(nullptr) {}
 
 YuvFrameReaderImpl::~YuvFrameReaderImpl() {
@@ -37,9 +85,9 @@ YuvFrameReaderImpl::~YuvFrameReaderImpl() {
 }
 
 bool YuvFrameReaderImpl::Init() {
-  if (width_ <= 0 || height_ <= 0) {
+  if (input_width_ <= 0 || input_height_ <= 0) {
-    fprintf(stderr, "Frame width and height must be >0, was %d x %d\n", width_,
+    fprintf(stderr, "Frame width and height must be >0, was %d x %d\n",
-            height_);
+            input_width_, input_height_);
     return false;
   }
   input_file_ = fopen(input_filename_.c_str(), "rb");
@@ -56,6 +104,7 @@ bool YuvFrameReaderImpl::Init() {
   }
   number_of_frames_ =
       static_cast<int>(source_file_size / frame_length_in_bytes_);
+  current_frame_index_ = 0;
   return true;
 }
 
@@ -65,15 +114,51 @@ rtc::scoped_refptr<I420Buffer> YuvFrameReaderImpl::ReadFrame() {
             "YuvFrameReaderImpl is not initialized (input file is NULL)\n");
     return nullptr;
   }
-  rtc::scoped_refptr<I420Buffer> buffer(
+
-      ReadI420Buffer(width_, height_, input_file_));
+  rtc::scoped_refptr<I420Buffer> buffer;
+
+  do {
+    if (current_frame_index_ >= number_of_frames_) {
+      switch (repeat_mode_) {
+        case RepeatMode::kSingle:
+          return nullptr;
+        case RepeatMode::kRepeat:
+          fseek(input_file_, 0, SEEK_SET);
+          current_frame_index_ = 0;
+          break;
+        case RepeatMode::kPingPong:
+          if (current_frame_index_ == number_of_frames_ * 2) {
+            fseek(input_file_, 0, SEEK_SET);
+            current_frame_index_ = 0;
+          } else {
+            int reverse_frame_index = current_frame_index_ - number_of_frames_;
+            int seek_frame_pos = (number_of_frames_ - reverse_frame_index - 1);
+            fseek(input_file_, seek_frame_pos * frame_size_bytes_, SEEK_SET);
+          }
+          break;
+      }
+    }
+    ++current_frame_index_;
+
+    buffer = ReadI420Buffer(input_width_, input_height_, input_file_);
     if (!buffer && ferror(input_file_)) {
       fprintf(stderr, "Error reading from input file: %s\n",
               input_filename_.c_str());
     }
+  } while (dropper_ &&
+           dropper_->UpdateLevel() == DropperUtil::DropDecision::kDropframe);
+
+  if (input_width_ == desired_width_ && input_height_ == desired_height_) {
     return buffer;
   }
 
+  rtc::scoped_refptr<I420Buffer> rescaled_buffer(
+      I420Buffer::Create(desired_width_, desired_height_));
+  rescaled_buffer->ScaleFrom(*buffer.get());
+
+  return rescaled_buffer;
+}
+
 void YuvFrameReaderImpl::Close() {
   if (input_file_ != nullptr) {
     fclose(input_file_);