Calculate corruption score once the frame is decoded

Bug: webrtc:358039777
Change-Id: I291e8e505f2ea7f9f95da4c83cd7679b49f2bc56
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/364600
Reviewed-by: Erik Språng <sprang@webrtc.org>
Auto-Submit: Fanny Linderborg <linderborg@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43192}

common_video/BUILD.gn

@@ -32,6 +32,14 @@ rtc_library("frame_instrumentation_data") {
   sources = [ "frame_instrumentation_data.h" ]
 }
 
+rtc_library("corruption_score_calculator") {
+  sources = [ "include/corruption_score_calculator.h" ]
+  deps = [
+    ":frame_instrumentation_data",
+    "../api/video:video_frame",
+  ]
+}
+
 rtc_library("common_video") {
   visibility = [ "*" ]
 

common_video/include/corruption_score_calculator.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2024 The WebRTC project authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS.  All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef COMMON_VIDEO_INCLUDE_CORRUPTION_SCORE_CALCULATOR_H_
+#define COMMON_VIDEO_INCLUDE_CORRUPTION_SCORE_CALCULATOR_H_
+
+#include <optional>
+
+#include "api/video/video_frame.h"
+#include "common_video/frame_instrumentation_data.h"
+
+namespace webrtc {
+
+// Allow classes to have their own implementations of how to calculate a score
+// for automatic corruption detection.
+class CorruptionScoreCalculator {
+ public:
+  virtual ~CorruptionScoreCalculator() = default;
+
+  virtual std::optional<double> CalculateCorruptionScore(
+      const VideoFrame& frame,
+      const FrameInstrumentationData& frame_instrumentation_data) = 0;
+};
+
+}  // namespace webrtc
+
+#endif  // COMMON_VIDEO_INCLUDE_CORRUPTION_SCORE_CALCULATOR_H_

modules/video_coding/BUILD.gn

@@ -236,6 +236,8 @@ rtc_library("video_coding") {
     "../../api/video:video_rtp_headers",
     "../../api/video_codecs:video_codecs_api",
     "../../common_video",
+    "../../common_video:corruption_score_calculator",
+    "../../common_video:frame_instrumentation_data",
     "../../rtc_base:buffer",
     "../../rtc_base:byte_buffer",
     "../../rtc_base:checks",
@@ -260,6 +262,7 @@ rtc_library("video_coding") {
     "../../system_wrappers",
     "../../system_wrappers:metrics",
     "../../video/config:encoder_config",
+    "../../video/corruption_detection:frame_instrumentation_evaluation",
     "../rtp_rtcp",
     "../rtp_rtcp:rtp_rtcp_format",
     "../rtp_rtcp:rtp_video_header",
@@ -329,6 +332,7 @@ rtc_library("video_coding_legacy") {
     "../../api/video:video_rtp_headers",
     "../../api/video_codecs:video_codecs_api",
     "../../common_video",
+    "../../common_video:corruption_score_calculator",
     "../../modules/rtp_rtcp:rtp_video_header",
     "../../rtc_base:checks",
     "../../rtc_base:event_tracer",
@@ -1196,6 +1200,7 @@ if (rtc_include_tests) {
       "../../api/video:builtin_video_bitrate_allocator_factory",
       "../../api/video:encoded_frame",
       "../../api/video:encoded_image",
+      "../../api/video:frame_buffer",
       "../../api/video:render_resolution",
       "../../api/video:video_adaptation",
       "../../api/video:video_bitrate_allocation",
@@ -1207,6 +1212,8 @@ if (rtc_include_tests) {
       "../../api/video_codecs:video_codecs_api",
       "../../api/video_codecs:vp8_temporal_layers_factory",
       "../../common_video",
+      "../../common_video:corruption_score_calculator",
+      "../../common_video:frame_instrumentation_data",
       "../../common_video/generic_frame_descriptor",
       "../../common_video/test:utilities",
       "../../media:media_constants",

modules/video_coding/generic_decoder.cc

@@ -19,8 +19,11 @@
 #include <utility>
 
 #include "absl/algorithm/container.h"
+#include "absl/types/variant.h"
 #include "api/video/video_timing.h"
 #include "api/video_codecs/video_decoder.h"
+#include "common_video/frame_instrumentation_data.h"
+#include "common_video/include/corruption_score_calculator.h"
 #include "modules/include/module_common_types_public.h"
 #include "modules/video_coding/include/video_error_codes.h"
 #include "rtc_base/checks.h"
@@ -41,8 +44,11 @@ constexpr size_t kDecoderFrameMemoryLength = 10;
 VCMDecodedFrameCallback::VCMDecodedFrameCallback(
     VCMTiming* timing,
     Clock* clock,
-    const FieldTrialsView& field_trials)
-    : _clock(clock), _timing(timing) {
+    const FieldTrialsView& field_trials,
+    CorruptionScoreCalculator* corruption_score_calculator)
+    : _clock(clock),
+      _timing(timing),
+      corruption_score_calculator_(corruption_score_calculator) {
   ntp_offset_ =
       _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds();
 }
@@ -128,6 +134,18 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
     return;
   }
 
+  // TODO: bugs.webrtc.org/358039777 - Use the score.
+  std::optional<double> score;
+  if (corruption_score_calculator_ &&
+      frame_info->frame_instrumentation_data.has_value()) {
+    if (const FrameInstrumentationData* data =
+            absl::get_if<FrameInstrumentationData>(
+                &*frame_info->frame_instrumentation_data)) {
+      score = corruption_score_calculator_->CalculateCorruptionScore(
+          decodedImage, *data);
+    }
+  }
+
   decodedImage.set_ntp_time_ms(frame_info->ntp_time_ms);
   decodedImage.set_packet_infos(frame_info->packet_infos);
   decodedImage.set_rotation(frame_info->rotation);
@@ -285,16 +303,22 @@ bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) {
 }
 
 int32_t VCMGenericDecoder::Decode(const EncodedFrame& frame, Timestamp now) {
-  return Decode(frame, now, frame.RenderTimeMs());
+  return Decode(frame, now, frame.RenderTimeMs(),
+                frame.CodecSpecific()->frame_instrumentation_data);
 }
 
 int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
-  return Decode(frame, now, frame.RenderTimeMs());
+  return Decode(frame, now, frame.RenderTimeMs(),
+                frame.CodecSpecific()->frame_instrumentation_data);
 }
 
-int32_t VCMGenericDecoder::Decode(const EncodedImage& frame,
+int32_t VCMGenericDecoder::Decode(
+    const EncodedImage& frame,
     Timestamp now,
-                                  int64_t render_time_ms) {
+    int64_t render_time_ms,
+    const std::optional<
+        absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>&
+        frame_instrumentation_data) {
   TRACE_EVENT("webrtc", "VCMGenericDecoder::Decode",
               perfetto::Flow::ProcessScoped(frame.RtpTimestamp()));
   FrameInfo frame_info;
@@ -308,6 +332,7 @@ int32_t VCMGenericDecoder::Decode(const EncodedImage& frame,
   frame_info.timing = frame.video_timing();
   frame_info.ntp_time_ms = frame.ntp_time_ms_;
   frame_info.packet_infos = frame.PacketInfos();
+  frame_info.frame_instrumentation_data = frame_instrumentation_data;
 
   // Set correctly only for key frames. Thus, use latest key frame
   // content type. If the corresponding key frame was lost, decode will fail

modules/video_coding/generic_decoder.h

@@ -13,13 +13,17 @@
 
 #include <cstdint>
 #include <deque>
+#include <optional>
 #include <string>
 #include <utility>
 
+#include "absl/types/variant.h"
 #include "api/field_trials_view.h"
 #include "api/sequence_checker.h"
 #include "api/video/encoded_frame.h"
 #include "api/video_codecs/video_decoder.h"
+#include "common_video/frame_instrumentation_data.h"
+#include "common_video/include/corruption_score_calculator.h"
 #include "modules/video_coding/encoded_frame.h"
 #include "modules/video_coding/timing/timing.h"
 #include "rtc_base/synchronization/mutex.h"
@@ -48,13 +52,18 @@ struct FrameInfo {
   RtpPacketInfos packet_infos;
   // ColorSpace is not stored here, as it might be modified by decoders.
   VideoFrameType frame_type;
+  std::optional<
+      absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+      frame_instrumentation_data;
 };
 
 class VCMDecodedFrameCallback : public DecodedImageCallback {
  public:
-  VCMDecodedFrameCallback(VCMTiming* timing,
+  VCMDecodedFrameCallback(
+      VCMTiming* timing,
       Clock* clock,
-                          const FieldTrialsView& field_trials);
+      const FieldTrialsView& field_trials,
+      CorruptionScoreCalculator* corruption_score_calculator);
   ~VCMDecodedFrameCallback() override;
   void SetUserReceiveCallback(VCMReceiveCallback* receiveCallback);
   VCMReceiveCallback* UserReceiveCallback();
@@ -86,6 +95,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback {
   Mutex lock_;
   std::deque<FrameInfo> frame_infos_ RTC_GUARDED_BY(lock_);
   int64_t ntp_offset_;
+  CorruptionScoreCalculator* const corruption_score_calculator_;
 };
 
 class VCMGenericDecoder {
@@ -120,7 +130,10 @@ class VCMGenericDecoder {
  private:
   int32_t Decode(const EncodedImage& frame,
                  Timestamp now,
-                 int64_t render_time_ms);
+                 int64_t render_time_ms,
+                 const std::optional<absl::variant<FrameInstrumentationSyncData,
+                                                   FrameInstrumentationData>>&
+                     frame_instrumentation_data);
   VCMDecodedFrameCallback* _callback = nullptr;
   VideoDecoder* const decoder_;
   VideoContentType _last_keyframe_content_type;

modules/video_coding/generic_decoder_unittest.cc

@@ -13,15 +13,21 @@
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <utility>
 #include <vector>
 
 #include "api/array_view.h"
 #include "api/rtp_packet_infos.h"
+#include "api/scoped_refptr.h"
 #include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "api/video/i420_buffer.h"
 #include "api/video/video_content_type.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_frame_type.h"
 #include "api/video_codecs/video_decoder.h"
+#include "common_video/frame_instrumentation_data.h"
+#include "common_video/include/corruption_score_calculator.h"
 #include "common_video/test/utilities.h"
 #include "modules/video_coding/timing/timing.h"
 #include "system_wrappers/include/clock.h"
@@ -34,6 +40,15 @@
 namespace webrtc {
 namespace video_coding {
 
+class MockCorruptionScoreCalculator : public CorruptionScoreCalculator {
+ public:
+  MOCK_METHOD(std::optional<double>,
+              CalculateCorruptionScore,
+              (const VideoFrame& frame,
+               const FrameInstrumentationData& frame_instrumentation_data),
+              (override));
+};
+
 class ReceiveCallback : public VCMReceiveCallback {
  public:
   int32_t FrameToRender(VideoFrame& frame,
@@ -81,7 +96,10 @@ class GenericDecoderTest : public ::testing::Test {
         clock_(time_controller_.GetClock()),
         timing_(time_controller_.GetClock(), field_trials_),
         decoder_(time_controller_.GetTaskQueueFactory()),
-        vcm_callback_(&timing_, time_controller_.GetClock(), field_trials_),
+        vcm_callback_(&timing_,
+                      time_controller_.GetClock(),
+                      field_trials_,
+                      &corruption_score_calculator_),
         generic_decoder_(&decoder_) {}
 
   void SetUp() override {
@@ -102,6 +120,7 @@ class GenericDecoderTest : public ::testing::Test {
   VCMDecodedFrameCallback vcm_callback_;
   VCMGenericDecoder generic_decoder_;
   ReceiveCallback user_callback_;
+  MockCorruptionScoreCalculator corruption_score_calculator_;
 };
 
 TEST_F(GenericDecoderTest, PassesPacketInfos) {
@@ -200,5 +219,24 @@ TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) {
   EXPECT_TRUE(decoded_frame->render_parameters().use_low_latency_rendering);
 }
 
+TEST_F(GenericDecoderTest, CallCalculateCorruptionScoreInDecoded) {
+  EXPECT_CALL(corruption_score_calculator_, CalculateCorruptionScore);
+
+  uint32_t rtp_timestamp = 1;
+  FrameInfo frame_info;
+  frame_info.frame_instrumentation_data = FrameInstrumentationData{};
+  frame_info.rtp_timestamp = rtp_timestamp;
+  frame_info.decode_start = Timestamp::Zero();
+  frame_info.content_type = VideoContentType::UNSPECIFIED;
+  frame_info.frame_type = VideoFrameType::kVideoFrameDelta;
+  VideoFrame video_frame = VideoFrame::Builder()
+                               .set_video_frame_buffer(I420Buffer::Create(5, 5))
+                               .set_rtp_timestamp(rtp_timestamp)
+                               .build();
+  vcm_callback_.Map(std::move(frame_info));
+
+  vcm_callback_.Decoded(video_frame);
+}
+
 }  // namespace video_coding
 }  // namespace webrtc

modules/video_coding/video_receiver.cc

@@ -44,7 +44,10 @@ VideoReceiver::VideoReceiver(Clock* clock,
     : clock_(clock),
       _timing(timing),
       _receiver(_timing, clock_, field_trials),
-      _decodedFrameCallback(_timing, clock_, field_trials),
+      _decodedFrameCallback(_timing,
+                            clock_,
+                            field_trials,
+                            /*corruption_score_calculator=*/nullptr),
       _frameTypeCallback(nullptr),
       _packetRequestCallback(nullptr),
       _scheduleKeyRequest(false),

modules/video_coding/video_receiver2.cc

@@ -19,6 +19,7 @@
 #include "absl/algorithm/container.h"
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_decoder.h"
+#include "common_video/include/corruption_score_calculator.h"
 #include "modules/video_coding/decoder_database.h"
 #include "modules/video_coding/generic_decoder.h"
 #include "modules/video_coding/include/video_coding_defines.h"
@@ -29,11 +30,16 @@
 
 namespace webrtc {
 
-VideoReceiver2::VideoReceiver2(Clock* clock,
+VideoReceiver2::VideoReceiver2(
+    Clock* clock,
     VCMTiming* timing,
-                               const FieldTrialsView& field_trials)
+    const FieldTrialsView& field_trials,
+    CorruptionScoreCalculator* corruption_score_calculator)
     : clock_(clock),
-      decoded_frame_callback_(timing, clock_, field_trials),
+      decoded_frame_callback_(timing,
+                              clock_,
+                              field_trials,
+                              corruption_score_calculator),
       codec_database_() {
   decoder_sequence_checker_.Detach();
 }

modules/video_coding/video_receiver2.h

@@ -18,6 +18,7 @@
 #include "api/sequence_checker.h"
 #include "api/video/encoded_frame.h"
 #include "api/video_codecs/video_decoder.h"
+#include "common_video/include/corruption_score_calculator.h"
 #include "modules/video_coding/decoder_database.h"
 #include "modules/video_coding/generic_decoder.h"
 #include "modules/video_coding/timing/timing.h"
@@ -35,7 +36,8 @@ class VideoReceiver2 {
  public:
   VideoReceiver2(Clock* clock,
                  VCMTiming* timing,
-                 const FieldTrialsView& field_trials);
+                 const FieldTrialsView& field_trials,
+                 CorruptionScoreCalculator* corruption_score_calculator);
   ~VideoReceiver2();
 
   void RegisterReceiveCodec(uint8_t payload_type,

modules/video_coding/video_receiver2_unittest.cc

@@ -99,7 +99,8 @@ class VideoReceiver2Test : public ::testing::Test {
   SimulatedClock clock_{Timestamp::Millis(1337)};
   VCMTiming timing_{&clock_, field_trials_};
   NiceMock<MockVCMReceiveCallback> receive_callback_;
-  VideoReceiver2 receiver_{&clock_, &timing_, field_trials_};
+  VideoReceiver2 receiver_{&clock_, &timing_, field_trials_,
+                           /*corruption_score_calculator=*/nullptr};
 };
 
 TEST_F(VideoReceiver2Test, RegisterExternalDecoder) {

video/BUILD.gn

@@ -119,6 +119,7 @@ rtc_library("video") {
     "../common_video",
     "../common_video:corruption_detection_converters",
     "../common_video:corruption_detection_message",
+    "../common_video:corruption_score_calculator",
     "../common_video:frame_instrumentation_data",
     "../media:media_constants",
     "../media:rtc_sdp_video_format_utils",
@@ -168,6 +169,7 @@ rtc_library("video") {
     "../system_wrappers:field_trial",
     "../system_wrappers:metrics",
     "../video/config:encoder_config",
+    "../video/corruption_detection:frame_instrumentation_evaluation",
     "adaptation:video_adaptation",
     "render:incoming_video_stream",
     "//third_party/abseil-cpp/absl/algorithm:container",

video/video_receive_stream2.cc

@@ -31,11 +31,13 @@
 #include "api/units/time_delta.h"
 #include "api/units/timestamp.h"
 #include "api/video/encoded_image.h"
+#include "api/video/video_frame.h"
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_decoder_factory.h"
 #include "call/rtp_stream_receiver_controller_interface.h"
 #include "call/rtx_receive_stream.h"
+#include "common_video/frame_instrumentation_data.h"
 #include "modules/video_coding/include/video_codec_interface.h"
 #include "modules/video_coding/include/video_coding_defines.h"
 #include "modules/video_coding/include/video_error_codes.h"
@@ -51,6 +53,7 @@
 #include "rtc_base/trace_event.h"
 #include "system_wrappers/include/clock.h"
 #include "video/call_stats2.h"
+#include "video/corruption_detection/frame_instrumentation_evaluation.h"
 #include "video/frame_dumping_decoder.h"
 #include "video/receive_statistics_proxy.h"
 #include "video/render/incoming_video_stream.h"
@@ -198,7 +201,7 @@ VideoReceiveStream2::VideoReceiveStream2(
       stats_proxy_(remote_ssrc(), &env_.clock(), call->worker_thread()),
       rtp_receive_statistics_(ReceiveStatistics::Create(&env_.clock())),
       timing_(std::move(timing)),
-      video_receiver_(&env_.clock(), timing_.get(), env_.field_trials()),
+      video_receiver_(&env_.clock(), timing_.get(), env_.field_trials(), this),
       rtp_video_stream_receiver_(env_,
                                  call->worker_thread(),
                                  &transport_adapter_,
@@ -604,6 +607,12 @@ void VideoReceiveStream2::UpdateHistograms() {
   stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, nullptr);
 }
 
+std::optional<double> VideoReceiveStream2::CalculateCorruptionScore(
+    const VideoFrame& frame,
+    const FrameInstrumentationData& frame_instrumentation_data) {
+  return GetCorruptionScore(frame_instrumentation_data, frame);
+}
+
 bool VideoReceiveStream2::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
   RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
   TimeDelta delay = TimeDelta::Millis(delay_ms);

video/video_receive_stream2.h

@@ -24,10 +24,13 @@
 #include "api/units/time_delta.h"
 #include "api/units/timestamp.h"
 #include "api/video/recordable_encoded_frame.h"
+#include "api/video/video_frame.h"
 #include "call/call.h"
 #include "call/rtp_packet_sink_interface.h"
 #include "call/syncable.h"
 #include "call/video_receive_stream.h"
+#include "common_video/frame_instrumentation_data.h"
+#include "common_video/include/corruption_score_calculator.h"
 #include "modules/rtp_rtcp/source/source_tracker.h"
 #include "modules/video_coding/nack_requester.h"
 #include "modules/video_coding/video_receiver2.h"
@@ -87,7 +90,8 @@ class VideoReceiveStream2
       public RtpVideoStreamReceiver2::OnCompleteFrameCallback,
       public Syncable,
       public CallStatsObserver,
-      public FrameSchedulingReceiver {
+      public FrameSchedulingReceiver,
+      public CorruptionScoreCalculator {
  public:
   // The maximum number of buffered encoded frames when encoded output is
   // configured.
@@ -239,6 +243,9 @@ class VideoReceiveStream2
       RTC_RUN_ON(decode_sequence_checker_);
 
   void UpdateHistograms();
+  std::optional<double> CalculateCorruptionScore(
+      const VideoFrame& frame,
+      const FrameInstrumentationData& frame_instrumentation_data) override;
 
   const Environment env_;