Analyze quality of dropped frames in VideoProcessor.

Calculate quality metrics for dropped frames by comparing original
frame against last decoded one.

This feature makes comparison of encoders which do/don't drop frames
more fair.

The feature is controlled by analyze_quality_of_dropped_frames flag
and is disabled by default.

Bug: none
Change-Id: Ifab8df92d0b76e743ff3193c05d7c8dbd14921c4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/190660
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32518}

api/test/videocodec_test_fixture.h

@@ -138,6 +138,9 @@ class VideoCodecTestFixture {
       bool save_encoded_ivf = false;
       bool save_decoded_y4m = false;
     } visualization_params;
+
+    // Enables quality analysis for dropped frames.
+    bool analyze_quality_of_dropped_frames = false;
   };
 
   virtual ~VideoCodecTestFixture() = default;

api/test/videocodec_test_stats.h

@@ -67,6 +67,7 @@ class VideoCodecTestStats {
     int qp = -1;
 
     // Quality.
+    bool quality_analysis_successful = false;
     float psnr_y = 0.0f;
     float psnr_u = 0.0f;
     float psnr_v = 0.0f;

modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc

@@ -450,6 +450,8 @@ void VideoCodecTestFixtureImpl::ProcessAllFrames(
     }
   }
 
+  task_queue->PostTask([this] { processor_->Finalize(); });
+
   // Wait until we know that the last frame has been sent for encode.
   task_queue->SendTask([] {}, RTC_FROM_HERE);
 

modules/video_coding/codecs/test/videocodec_test_stats_impl.cc

@@ -252,12 +252,6 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic(
       video_stat.height =
           std::max(video_stat.height, frame_stat.decoded_height);
 
-      psnr_y.AddSample(frame_stat.psnr_y);
-      psnr_u.AddSample(frame_stat.psnr_u);
-      psnr_v.AddSample(frame_stat.psnr_v);
-      psnr.AddSample(frame_stat.psnr);
-      ssim.AddSample(frame_stat.ssim);
-
       if (video_stat.num_decoded_frames > 1) {
         if (last_successfully_decoded_frame.decoded_width !=
                 frame_stat.decoded_width ||
@@ -271,6 +265,14 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic(
       last_successfully_decoded_frame = frame_stat;
     }
 
+    if (frame_stat.quality_analysis_successful) {
+      psnr_y.AddSample(frame_stat.psnr_y);
+      psnr_u.AddSample(frame_stat.psnr_u);
+      psnr_v.AddSample(frame_stat.psnr_v);
+      psnr.AddSample(frame_stat.psnr);
+      ssim.AddSample(frame_stat.ssim);
+    }
+
     if (video_stat.num_input_frames > 0) {
       if (video_stat.time_to_reach_target_bitrate_sec == 0.0f) {
         RTC_CHECK_GT(time_since_first_frame_sec, 0);

modules/video_coding/codecs/test/videoprocessor.cc

@@ -41,8 +41,6 @@
 namespace webrtc {
 namespace test {
 
-using FrameStatistics = VideoCodecTestStats::FrameStatistics;
-
 namespace {
 const int kMsToRtpTimestamp = kVideoPayloadTypeFrequency / 1000;
 const int kMaxBufferedInputFrames = 20;
@@ -86,34 +84,9 @@ int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) {
   return static_cast<int>(diff_us);
 }
 
-void ExtractI420BufferWithSize(const VideoFrame& image,
-                               int width,
-                               int height,
-                               rtc::Buffer* buffer) {
-  if (image.width() != width || image.height() != height) {
-    EXPECT_DOUBLE_EQ(static_cast<double>(width) / height,
-                     static_cast<double>(image.width()) / image.height());
-    // Same aspect ratio, no cropping needed.
-    rtc::scoped_refptr<I420Buffer> scaled(I420Buffer::Create(width, height));
-    scaled->ScaleFrom(*image.video_frame_buffer()->ToI420());
-
-    size_t length =
-        CalcBufferSize(VideoType::kI420, scaled->width(), scaled->height());
-    buffer->SetSize(length);
-    RTC_CHECK_NE(ExtractBuffer(scaled, length, buffer->data()), -1);
-    return;
-  }
-
-  // No resize.
-  size_t length =
-      CalcBufferSize(VideoType::kI420, image.width(), image.height());
-  buffer->SetSize(length);
-  RTC_CHECK_NE(ExtractBuffer(image, length, buffer->data()), -1);
-}
-
 void CalculateFrameQuality(const I420BufferInterface& ref_buffer,
                            const I420BufferInterface& dec_buffer,
-                           FrameStatistics* frame_stat,
+                           VideoCodecTestStats::FrameStatistics* frame_stat,
                            bool calc_ssim) {
   if (ref_buffer.width() != dec_buffer.width() ||
       ref_buffer.height() != dec_buffer.height()) {
@@ -174,6 +147,7 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
       num_simulcast_or_spatial_layers_(
           std::max(config_.NumberOfSimulcastStreams(),
                    config_.NumberOfSpatialLayers())),
+      analyze_frame_quality_(!config_.measure_cpu),
       stats_(stats),
       encoder_(encoder),
       decoders_(decoders),
@@ -192,8 +166,9 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
       last_encoded_frame_num_(num_simulcast_or_spatial_layers_),
       first_decoded_frame_(num_simulcast_or_spatial_layers_, true),
       last_decoded_frame_num_(num_simulcast_or_spatial_layers_),
-      decoded_frame_buffer_(num_simulcast_or_spatial_layers_),
-      post_encode_time_ns_(0) {
+      last_decoded_frame_buffer_(num_simulcast_or_spatial_layers_),
+      post_encode_time_ns_(0),
+      is_finalized_(false) {
   // Sanity checks.
   RTC_CHECK(TaskQueueBase::Current())
       << "VideoProcessor must be run on a task queue.";
@@ -234,6 +209,10 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
 VideoProcessor::~VideoProcessor() {
   RTC_DCHECK_RUN_ON(&sequence_checker_);
 
+  if (!is_finalized_) {
+    Finalize();
+  }
+
   // Explicitly reset codecs, in case they don't do that themselves when they
   // go out of scope.
   RTC_CHECK_EQ(encoder_->Release(), WEBRTC_VIDEO_CODEC_OK);
@@ -249,6 +228,8 @@ VideoProcessor::~VideoProcessor() {
 
 void VideoProcessor::ProcessFrame() {
   RTC_DCHECK_RUN_ON(&sequence_checker_);
+  RTC_DCHECK(!is_finalized_);
+
   const size_t frame_number = last_inputed_frame_num_++;
 
   // Get input frame and store for future quality calculation.
@@ -304,6 +285,8 @@ void VideoProcessor::ProcessFrame() {
 
 void VideoProcessor::SetRates(size_t bitrate_kbps, double framerate_fps) {
   RTC_DCHECK_RUN_ON(&sequence_checker_);
+  RTC_DCHECK(!is_finalized_);
+
   framerate_fps_ = framerate_fps;
   bitrate_allocation_ =
       bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
@@ -460,6 +443,56 @@ void VideoProcessor::FrameEncoded(
   }
 }
 
+void VideoProcessor::CalcFrameQuality(const I420BufferInterface& decoded_frame,
+                                      FrameStatistics* frame_stat) {
+  RTC_DCHECK_RUN_ON(&sequence_checker_);
+
+  const auto reference_frame = input_frames_.find(frame_stat->frame_number);
+  RTC_CHECK(reference_frame != input_frames_.cend())
+      << "The codecs are either buffering too much, dropping too much, or "
+         "being too slow relative to the input frame rate.";
+
+  // SSIM calculation is not optimized. Skip it in real-time mode.
+  const bool calc_ssim = !config_.encode_in_real_time;
+  CalculateFrameQuality(*reference_frame->second.video_frame_buffer()->ToI420(),
+                        decoded_frame, frame_stat, calc_ssim);
+
+  frame_stat->quality_analysis_successful = true;
+}
+
+void VideoProcessor::WriteDecodedFrame(const I420BufferInterface& decoded_frame,
+                                       FrameWriter& frame_writer) {
+  int input_video_width = config_.codec_settings.width;
+  int input_video_height = config_.codec_settings.height;
+
+  rtc::scoped_refptr<I420Buffer> scaled_buffer;
+  const I420BufferInterface* scaled_frame;
+
+  if (decoded_frame.width() == input_video_width &&
+      decoded_frame.height() == input_video_height) {
+    scaled_frame = &decoded_frame;
+  } else {
+    EXPECT_DOUBLE_EQ(
+        static_cast<double>(input_video_width) / input_video_height,
+        static_cast<double>(decoded_frame.width()) / decoded_frame.height());
+
+    scaled_buffer = I420Buffer::Create(input_video_width, input_video_height);
+    scaled_buffer->ScaleFrom(decoded_frame);
+
+    scaled_frame = scaled_buffer;
+  }
+
+  // Ensure there is no padding.
+  RTC_CHECK_EQ(scaled_frame->StrideY(), input_video_width);
+  RTC_CHECK_EQ(scaled_frame->StrideU(), input_video_width / 2);
+  RTC_CHECK_EQ(scaled_frame->StrideV(), input_video_width / 2);
+
+  RTC_CHECK_EQ(3 * input_video_width * input_video_height / 2,
+               frame_writer.FrameLength());
+
+  RTC_CHECK(frame_writer.WriteFrame(scaled_frame->DataY()));
+}
+
 void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
                                   size_t spatial_idx) {
   RTC_DCHECK_RUN_ON(&sequence_checker_);
@@ -472,13 +505,24 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
       stats_->GetFrameWithTimestamp(decoded_frame.timestamp(), spatial_idx);
   const size_t frame_number = frame_stat->frame_number;
 
-  if (decoded_frame_writers_ && !first_decoded_frame_[spatial_idx]) {
-    // Fill drops with last decoded frame to make them look like freeze at
-    // playback and to keep decoded layers in sync.
-    for (size_t i = last_decoded_frame_num_[spatial_idx] + 1; i < frame_number;
-         ++i) {
-      RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
-                    ->WriteFrame(decoded_frame_buffer_[spatial_idx].data()));
+  if (!first_decoded_frame_[spatial_idx]) {
+    for (size_t dropped_frame_number = last_decoded_frame_num_[spatial_idx] + 1;
+         dropped_frame_number < frame_number; ++dropped_frame_number) {
+      FrameStatistics* dropped_frame_stat =
+          stats_->GetFrame(dropped_frame_number, spatial_idx);
+
+      if (analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) {
+        // Calculate frame quality comparing input frame with last decoded one.
+        CalcFrameQuality(*last_decoded_frame_buffer_[spatial_idx],
+                         dropped_frame_stat);
+      }
+
+      if (decoded_frame_writers_ != nullptr) {
+        // Fill drops with last decoded frame to make them look like freeze at
+        // playback and to keep decoded layers in sync.
+        WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx],
+                          *decoded_frame_writers_->at(spatial_idx));
+      }
     }
   }
 
@@ -497,41 +541,40 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
   frame_stat->decoded_height = decoded_frame.height();
 
   // Skip quality metrics calculation to not affect CPU usage.
-  if (!config_.measure_cpu) {
-    const auto reference_frame = input_frames_.find(frame_number);
-    RTC_CHECK(reference_frame != input_frames_.cend())
-        << "The codecs are either buffering too much, dropping too much, or "
-           "being too slow relative the input frame rate.";
+  if (analyze_frame_quality_ || decoded_frame_writers_) {
+    // Save last decoded frame to handle possible future drops.
+    rtc::scoped_refptr<I420BufferInterface> i420buffer =
+        decoded_frame.video_frame_buffer()->ToI420();
 
-    // SSIM calculation is not optimized. Skip it in real-time mode.
-    const bool calc_ssim = !config_.encode_in_real_time;
-    CalculateFrameQuality(
-        *reference_frame->second.video_frame_buffer()->ToI420(),
-        *decoded_frame.video_frame_buffer()->ToI420(), frame_stat, calc_ssim);
-
-    // Erase all buffered input frames that we have moved past for all
-    // simulcast/spatial layers. Never buffer more than
-    // |kMaxBufferedInputFrames| frames, to protect against long runs of
-    // consecutive frame drops for a particular layer.
-    const auto min_last_decoded_frame_num = std::min_element(
-        last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend());
-    const size_t min_buffered_frame_num = std::max(
-        0, static_cast<int>(frame_number) - kMaxBufferedInputFrames + 1);
-    RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend());
-    const auto input_frames_erase_before = input_frames_.lower_bound(
-        std::max(*min_last_decoded_frame_num, min_buffered_frame_num));
-    input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before);
+    // Copy decoded frame to a buffer without padding/stride such that we can
+    // dump Y, U and V planes into a file in one shot.
+    last_decoded_frame_buffer_[spatial_idx] = I420Buffer::Copy(
+        i420buffer->width(), i420buffer->height(), i420buffer->DataY(),
+        i420buffer->StrideY(), i420buffer->DataU(), i420buffer->StrideU(),
+        i420buffer->DataV(), i420buffer->StrideV());
   }
 
-  if (decoded_frame_writers_) {
-    ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width,
-                              config_.codec_settings.height,
-                              &decoded_frame_buffer_[spatial_idx]);
-    RTC_CHECK_EQ(decoded_frame_buffer_[spatial_idx].size(),
-                 decoded_frame_writers_->at(spatial_idx)->FrameLength());
-    RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
-                  ->WriteFrame(decoded_frame_buffer_[spatial_idx].data()));
+  if (analyze_frame_quality_) {
+    CalcFrameQuality(*decoded_frame.video_frame_buffer()->ToI420(), frame_stat);
   }
+
+  if (decoded_frame_writers_ != nullptr) {
+    WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx],
+                      *decoded_frame_writers_->at(spatial_idx));
+  }
+
+  // Erase all buffered input frames that we have moved past for all
+  // simulcast/spatial layers. Never buffer more than
+  // |kMaxBufferedInputFrames| frames, to protect against long runs of
+  // consecutive frame drops for a particular layer.
+  const auto min_last_decoded_frame_num = std::min_element(
+      last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend());
+  const size_t min_buffered_frame_num =
+      std::max(0, static_cast<int>(frame_number) - kMaxBufferedInputFrames + 1);
+  RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend());
+  const auto input_frames_erase_before = input_frames_.lower_bound(
+      std::max(*min_last_decoded_frame_num, min_buffered_frame_num));
+  input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before);
 }
 
 void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image,
@@ -588,5 +631,41 @@ const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
   return &merged_encoded_frames_.at(spatial_idx);
 }
 
+void VideoProcessor::Finalize() {
+  RTC_DCHECK_RUN_ON(&sequence_checker_);
+  RTC_DCHECK(!is_finalized_);
+  is_finalized_ = true;
+
+  if (!(analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) &&
+      decoded_frame_writers_ == nullptr) {
+    return;
+  }
+
+  for (size_t spatial_idx = 0; spatial_idx < num_simulcast_or_spatial_layers_;
+       ++spatial_idx) {
+    if (first_decoded_frame_[spatial_idx]) {
+      continue;  // No decoded frames on this spatial layer.
+    }
+
+    for (size_t dropped_frame_number = last_decoded_frame_num_[spatial_idx] + 1;
+         dropped_frame_number < last_inputed_frame_num_;
+         ++dropped_frame_number) {
+      FrameStatistics* frame_stat =
+          stats_->GetFrame(dropped_frame_number, spatial_idx);
+
+      RTC_DCHECK(!frame_stat->decoding_successful);
+
+      if (analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) {
+        CalcFrameQuality(*last_decoded_frame_buffer_[spatial_idx], frame_stat);
+      }
+
+      if (decoded_frame_writers_ != nullptr) {
+        WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx],
+                          *decoded_frame_writers_->at(spatial_idx));
+      }
+    }
+  }
+}
+
 }  // namespace test
 }  // namespace webrtc

modules/video_coding/codecs/test/videoprocessor.h

@@ -24,6 +24,7 @@
 #include "api/task_queue/task_queue_base.h"
 #include "api/test/videocodec_test_fixture.h"
 #include "api/video/encoded_image.h"
+#include "api/video/i420_buffer.h"
 #include "api/video/video_bitrate_allocation.h"
 #include "api/video/video_bitrate_allocator.h"
 #include "api/video/video_frame.h"
@@ -58,6 +59,7 @@ class VideoProcessor {
   // TODO(brandtr): Consider changing FrameWriterList to be a FrameWriterMap,
   // to be able to save different TLs separately.
   using FrameWriterList = std::vector<std::unique_ptr<FrameWriter>>;
+  using FrameStatistics = VideoCodecTestStats::FrameStatistics;
 
   VideoProcessor(webrtc::VideoEncoder* encoder,
                  VideoDecoderList* decoders,
@@ -77,6 +79,11 @@ class VideoProcessor {
   // Updates the encoder with target rates. Must be called at least once.
   void SetRates(size_t bitrate_kbps, double framerate_fps);
 
+  // Signals processor to finalize frame processing and handle possible tail
+  // drops. If not called expelicitly, this will be called in dtor. It is
+  // unexpected to get ProcessFrame() or SetRates() calls after Finalize().
+  void Finalize();
+
  private:
   class VideoProcessorEncodeCompleteCallback
       : public webrtc::EncodedImageCallback {
@@ -182,9 +189,20 @@ class VideoProcessor {
       size_t simulcast_svc_idx,
       bool inter_layer_predicted) RTC_RUN_ON(sequence_checker_);
 
-  // Test input/output.
-  VideoCodecTestFixture::Config config_ RTC_GUARDED_BY(sequence_checker_);
+  void CalcFrameQuality(const I420BufferInterface& decoded_frame,
+                        FrameStatistics* frame_stat);
+
+  void WriteDecodedFrame(const I420BufferInterface& decoded_frame,
+                         FrameWriter& frame_writer);
+
+  void HandleTailDrops();
+
+  // Test config.
+  const VideoCodecTestFixture::Config config_;
   const size_t num_simulcast_or_spatial_layers_;
+  const bool analyze_frame_quality_;
+
+  // Frame statistics.
   VideoCodecTestStatsImpl* const stats_;
 
   // Codecs.
@@ -240,7 +258,7 @@ class VideoProcessor {
   // simulcast_svc_idx -> frame_number.
   std::vector<size_t> last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
   // simulcast_svc_idx -> buffer.
-  std::vector<rtc::Buffer> decoded_frame_buffer_
+  std::vector<rtc::scoped_refptr<I420Buffer>> last_decoded_frame_buffer_
       RTC_GUARDED_BY(sequence_checker_);
 
   // Time spent in frame encode callback. It is accumulated for layers and
@@ -248,6 +266,9 @@ class VideoProcessor {
   // is substracted from measured encode time. Thus we get pure encode time.
   int64_t post_encode_time_ns_ RTC_GUARDED_BY(sequence_checker_);
 
+  // Indicates whether Finalize() was called or not.
+  bool is_finalized_ RTC_GUARDED_BY(sequence_checker_);
+
   // This class must be operated on a TaskQueue.
   SequenceChecker sequence_checker_;
 

test/testsupport/frame_writer.h

@@ -32,7 +32,7 @@ class FrameWriter {
 
   // Writes a frame of the configured frame length to the output file.
   // Returns true if the write was successful, false otherwise.
-  virtual bool WriteFrame(uint8_t* frame_buffer) = 0;
+  virtual bool WriteFrame(const uint8_t* frame_buffer) = 0;
 
   // Closes the output file if open. Essentially makes this class impossible
   // to use anymore. Will also be invoked by the destructor.
@@ -54,7 +54,7 @@ class YuvFrameWriterImpl : public FrameWriter {
   YuvFrameWriterImpl(std::string output_filename, int width, int height);
   ~YuvFrameWriterImpl() override;
   bool Init() override;
-  bool WriteFrame(uint8_t* frame_buffer) override;
+  bool WriteFrame(const uint8_t* frame_buffer) override;
   void Close() override;
   size_t FrameLength() override;
 
@@ -76,7 +76,7 @@ class Y4mFrameWriterImpl : public YuvFrameWriterImpl {
                      int frame_rate);
   ~Y4mFrameWriterImpl() override;
   bool Init() override;
-  bool WriteFrame(uint8_t* frame_buffer) override;
+  bool WriteFrame(const uint8_t* frame_buffer) override;
 
  private:
   const int frame_rate_;

test/testsupport/y4m_frame_writer.cc

@@ -41,7 +41,7 @@ bool Y4mFrameWriterImpl::Init() {
   return true;
 }
 
-bool Y4mFrameWriterImpl::WriteFrame(uint8_t* frame_buffer) {
+bool Y4mFrameWriterImpl::WriteFrame(const uint8_t* frame_buffer) {
   if (output_file_ == nullptr) {
     fprintf(stderr,
             "Y4mFrameWriterImpl is not initialized (output file is NULL)\n");

test/testsupport/yuv_frame_writer.cc

@@ -50,7 +50,7 @@ bool YuvFrameWriterImpl::Init() {
   return true;
 }
 
-bool YuvFrameWriterImpl::WriteFrame(uint8_t* frame_buffer) {
+bool YuvFrameWriterImpl::WriteFrame(const uint8_t* frame_buffer) {
   RTC_DCHECK(frame_buffer);
   if (output_file_ == nullptr) {
     fprintf(stderr,