Add helper for comparing FrameInstrumentationData with a VideoFrame

Bug: webrtc:358039777
Change-Id: Ibe597160658dbc66aba427f4e30dade4d6fe56e2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/363701
Auto-Submit: Fanny Linderborg <linderborg@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43134}

video/corruption_detection/BUILD.gn

@@ -21,6 +21,23 @@ rtc_library("corruption_classifier") {
   ]
 }
 
+rtc_library("frame_instrumentation_evaluation") {
+  sources = [
+    "frame_instrumentation_evaluation.cc",
+    "frame_instrumentation_evaluation.h",
+  ]
+  deps = [
+    ":corruption_classifier",
+    ":halton_frame_sampler",
+    "../../api:array_view",
+    "../../api:scoped_refptr",
+    "../../api/video:video_frame",
+    "../../common_video:frame_instrumentation_data",
+    "../../rtc_base:checks",
+    "../../rtc_base:logging",
+  ]
+}
+
 rtc_library("frame_instrumentation_generator") {
   sources = [
     "frame_instrumentation_generator.cc",
@@ -89,6 +106,18 @@ if (rtc_include_tests) {
     ]
   }
 
+  rtc_library("frame_instrumentation_evaluation_unittest") {
+    testonly = true
+    sources = [ "frame_instrumentation_evaluation_unittest.cc" ]
+    deps = [
+      ":frame_instrumentation_evaluation",
+      "../../api:scoped_refptr",
+      "../../api/video:video_frame",
+      "../../common_video:frame_instrumentation_data",
+      "../../test:test_support",
+    ]
+  }
+
   rtc_library("frame_instrumentation_generator_unittest") {
     testonly = true
     sources = [ "frame_instrumentation_generator_unittest.cc" ]
@@ -139,6 +168,7 @@ if (rtc_include_tests) {
     sources = []
     deps = [
       ":corruption_classifier_unittest",
+      ":frame_instrumentation_evaluation_unittest",
       ":frame_instrumentation_generator_unittest",
       ":generic_mapping_functions_unittest",
       ":halton_frame_sampler_unittest",

video/corruption_detection/frame_instrumentation_evaluation.cc

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2024 The WebRTC project authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS.  All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/corruption_detection/frame_instrumentation_evaluation.h"
+
+#include <cstddef>
+#include <optional>
+#include <vector>
+
+#include "api/array_view.h"
+#include "api/scoped_refptr.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_buffer.h"
+#include "common_video/frame_instrumentation_data.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "video/corruption_detection/corruption_classifier.h"
+#include "video/corruption_detection/halton_frame_sampler.h"
+
+namespace webrtc {
+
+namespace {
+
+std::vector<FilteredSample> ConvertSampleValuesToFilteredSamples(
+    rtc::ArrayView<const double> values,
+    rtc::ArrayView<const FilteredSample> samples) {
+  RTC_CHECK_EQ(values.size(), samples.size())
+      << "values and samples must have the same size";
+  std::vector<FilteredSample> filtered_samples;
+  filtered_samples.reserve(values.size());
+  for (size_t i = 0; i < values.size(); ++i) {
+    filtered_samples.push_back({.value = values[i], .plane = samples[i].plane});
+  }
+  return filtered_samples;
+}
+
+}  // namespace
+
+std::optional<double> GetCorruptionScore(const FrameInstrumentationData& data,
+                                         const VideoFrame& frame) {
+  if (data.sample_values.empty()) {
+    RTC_LOG(LS_WARNING)
+        << "Samples are needed to calculate a corruption score.";
+    return std::nullopt;
+  }
+
+  scoped_refptr<I420BufferInterface> frame_buffer_as_i420 =
+      frame.video_frame_buffer()->ToI420();
+  if (!frame_buffer_as_i420) {
+    RTC_LOG(LS_ERROR) << "Failed to convert "
+                      << VideoFrameBufferTypeToString(
+                             frame.video_frame_buffer()->type())
+                      << " image to I420";
+    return std::nullopt;
+  }
+
+  HaltonFrameSampler frame_sampler;
+  frame_sampler.SetCurrentIndex(data.sequence_index);
+  std::vector<HaltonFrameSampler::Coordinates> sample_coordinates =
+      frame_sampler.GetSampleCoordinatesForFrame(data.sample_values.size());
+  if (sample_coordinates.empty()) {
+    RTC_LOG(LS_ERROR) << "Failed to get sample coordinates for frame.";
+    return std::nullopt;
+  }
+
+  std::vector<FilteredSample> samples =
+      GetSampleValuesForFrame(frame_buffer_as_i420, sample_coordinates,
+                              frame.width(), frame.height(), data.std_dev);
+  if (samples.empty()) {
+    RTC_LOG(LS_ERROR) << "Failed to get sample values for frame";
+    return std::nullopt;
+  }
+
+  std::vector<FilteredSample> data_samples =
+      ConvertSampleValuesToFilteredSamples(data.sample_values, samples);
+  if (data_samples.empty()) {
+    RTC_LOG(LS_ERROR) << "Failed to convert sample values to filtered samples";
+    return std::nullopt;
+  }
+
+  // TODO: bugs.webrtc.org/358039777 - Update before rollout. Which variant of
+  // classifier should we use? What input parameters should it have?
+  CorruptionClassifier classifier(2.5);
+
+  return classifier.CalculateCorruptionProbablility(
+      data_samples, samples, data.luma_error_threshold,
+      data.chroma_error_threshold);
+}
+
+}  // namespace webrtc

video/corruption_detection/frame_instrumentation_evaluation.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2024 The WebRTC project authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS.  All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_
+#define VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_
+
+#include <optional>
+
+#include "api/video/video_frame.h"
+#include "common_video/frame_instrumentation_data.h"
+
+namespace webrtc {
+
+std::optional<double> GetCorruptionScore(const FrameInstrumentationData& data,
+                                         const VideoFrame& frame);
+
+}  // namespace webrtc
+
+#endif  // VIDEO_CORRUPTION_DETECTION_FRAME_INSTRUMENTATION_EVALUATION_H_

video/corruption_detection/frame_instrumentation_evaluation_unittest.cc

@@ -0,0 +1,144 @@
+/*
+ * Copyright 2024 The WebRTC project authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS.  All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/corruption_detection/frame_instrumentation_evaluation.h"
+
+#include <cstdint>
+#include <optional>
+#include <vector>
+
+#include "api/scoped_refptr.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "common_video/frame_instrumentation_data.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+scoped_refptr<I420Buffer> MakeI420FrameBufferWithDifferentPixelValues() {
+  // Create an I420 frame of size 4x4.
+  const int kDefaultLumaWidth = 4;
+  const int kDefaultLumaHeight = 4;
+  const int kDefaultChromaWidth = 2;
+  std::vector<uint8_t> kDefaultYContent = {1, 2,  3,  4,  5,  6,  7,  8,
+                                           9, 10, 11, 12, 13, 14, 15, 16};
+  std::vector<uint8_t> kDefaultUContent = {17, 18, 19, 20};
+  std::vector<uint8_t> kDefaultVContent = {21, 22, 23, 24};
+
+  return I420Buffer::Copy(kDefaultLumaWidth, kDefaultLumaHeight,
+                          kDefaultYContent.data(), kDefaultLumaWidth,
+                          kDefaultUContent.data(), kDefaultChromaWidth,
+                          kDefaultVContent.data(), kDefaultChromaWidth);
+}
+
+TEST(FrameInstrumentationEvaluationTest,
+     HaveNoCorruptionScoreWhenNoSampleValuesAreProvided) {
+  FrameInstrumentationData data = {.sequence_index = 0,
+                                   .communicate_upper_bits = false,
+                                   .std_dev = 0.0,
+                                   .luma_error_threshold = 0,
+                                   .chroma_error_threshold = 0,
+                                   .sample_values = {}};
+  VideoFrame frame =
+      VideoFrame::Builder()
+          .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+          .build();
+
+  std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+  EXPECT_FALSE(corruption_score.has_value());
+}
+
+TEST(FrameInstrumentationEvaluationTest,
+     HaveACorruptionScoreWhenSampleValuesAreProvided) {
+  FrameInstrumentationData data = {
+      .sequence_index = 0,
+      .communicate_upper_bits = false,
+      .std_dev = 0.0,
+      .luma_error_threshold = 0,
+      .chroma_error_threshold = 0,
+      .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}};
+  VideoFrame frame =
+      VideoFrame::Builder()
+          .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+          .build();
+
+  std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+  ASSERT_TRUE(corruption_score.has_value());
+  EXPECT_DOUBLE_EQ(*corruption_score, 1.0);
+}
+
+TEST(FrameInstrumentationEvaluationTest,
+     ApplyThresholdsWhenNonNegativeThresholdsAreProvided) {
+  FrameInstrumentationData data = {
+      .sequence_index = 0,
+      .communicate_upper_bits = false,
+      .std_dev = 0.0,
+      .luma_error_threshold = 8,
+      .chroma_error_threshold = 8,
+      .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}};
+  VideoFrame frame =
+      VideoFrame::Builder()
+          .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+          .build();
+
+  std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+  ASSERT_TRUE(corruption_score.has_value());
+  EXPECT_DOUBLE_EQ(*corruption_score, 0.55);
+}
+
+TEST(FrameInstrumentationEvaluationTest,
+     ApplyStdDevWhenNonNegativeStdDevIsProvided) {
+  FrameInstrumentationData data = {
+      .sequence_index = 0,
+      .communicate_upper_bits = false,
+      .std_dev = 0.6,
+      .luma_error_threshold = 8,
+      .chroma_error_threshold = 8,
+      .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}};
+
+  std::vector<double> sample_values = {12, 12, 12, 12, 12, 12, 12, 12};
+  VideoFrame frame =
+      VideoFrame::Builder()
+          .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+          .build();
+
+  std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+  ASSERT_TRUE(corruption_score.has_value());
+  EXPECT_DOUBLE_EQ(*corruption_score, 0.3302493109581533);
+}
+
+TEST(FrameInstrumentationEvaluationTest, ApplySequenceIndexWhenProvided) {
+  FrameInstrumentationData data = {
+      .sequence_index = 1,
+      .communicate_upper_bits = false,
+      .std_dev = 0.6,
+      .luma_error_threshold = 8,
+      .chroma_error_threshold = 8,
+      .sample_values = {12, 12, 12, 12, 12, 12, 12, 12}};
+
+  std::vector<double> sample_values = {12, 12, 12, 12, 12, 12, 12, 12};
+  VideoFrame frame =
+      VideoFrame::Builder()
+          .set_video_frame_buffer(MakeI420FrameBufferWithDifferentPixelValues())
+          .build();
+
+  std::optional<double> corruption_score = GetCorruptionScore(data, frame);
+
+  ASSERT_TRUE(corruption_score.has_value());
+  EXPECT_DOUBLE_EQ(*corruption_score, 0.12983429453668965);
+}
+
+}  // namespace
+}  // namespace webrtc

video/corruption_detection/frame_instrumentation_generator.cc

@@ -142,7 +142,7 @@ FrameInstrumentationGenerator::OnEncodedImage(
       contexts_[layer_id]
           .frame_sampler.GetSampleCoordinatesForFrameIfFrameShouldBeSampled(
               is_key_frame, captured_frame.rtp_timestamp(),
-              /*sample_size=*/13);
+              /*num_samples=*/13);
   if (sample_coordinates.empty()) {
     if (!is_key_frame) {
       return std::nullopt;