Add FrameInstrumentationGenerator to VideoStreamEncoder

The FrameInstrumentationGenerator is responsible for generating the
instrumentation data that will be used to detect corruption. The data is
then passed to the encoder in the CodecSpecificInfo.

Bug: webrtc:358039777
Change-Id: I79d0534920b4c7fa001e1138371dfd36c13424fb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/362584
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Fanny Linderborg <linderborg@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43060}

video/BUILD.gn

@@ -425,6 +425,7 @@ rtc_library("video_stream_encoder_impl") {
     "../api/video_codecs:video_codecs_api",
     "../call/adaptation:resource_adaptation",
     "../common_video",
+    "../common_video:frame_instrumentation_data",
     "../media:media_channel",
     "../modules:module_api_public",
     "../modules/video_coding",
@@ -461,9 +462,11 @@ rtc_library("video_stream_encoder_impl") {
     "adaptation:video_adaptation",
     "config:encoder_config",
     "config:streams_config",
+    "corruption_detection:frame_instrumentation_generator",
     "//third_party/abseil-cpp/absl/algorithm:container",
     "//third_party/abseil-cpp/absl/cleanup",
     "//third_party/abseil-cpp/absl/container:inlined_vector",
+    "//third_party/abseil-cpp/absl/types:variant",
   ]
 }
 
@@ -873,6 +876,7 @@ if (rtc_include_tests) {
       "../call/adaptation:resource_adaptation",
       "../call/adaptation:resource_adaptation_test_utilities",
       "../common_video",
+      "../common_video:frame_instrumentation_data",
       "../common_video/test:utilities",
       "../media:codec",
       "../media:media_constants",

video/video_stream_encoder.cc

@@ -20,6 +20,7 @@
 
 #include "absl/algorithm/container.h"
 #include "absl/cleanup/cleanup.h"
+#include "absl/types/variant.h"
 #include "api/field_trials_view.h"
 #include "api/sequence_checker.h"
 #include "api/task_queue/task_queue_base.h"
@@ -30,13 +31,16 @@
 #include "api/video/video_bitrate_allocator_factory.h"
 #include "api/video/video_codec_constants.h"
 #include "api/video/video_layers_allocation.h"
+#include "api/video/video_stream_encoder_settings.h"
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_encoder.h"
 #include "call/adaptation/resource_adaptation_processor.h"
 #include "call/adaptation/video_source_restrictions.h"
 #include "call/adaptation/video_stream_adapter.h"
+#include "common_video/frame_instrumentation_data.h"
 #include "media/base/media_channel.h"
 #include "modules/video_coding/include/video_codec_initializer.h"
+#include "modules/video_coding/include/video_codec_interface.h"
 #include "modules/video_coding/svc/scalability_mode_util.h"
 #include "modules/video_coding/svc/svc_rate_allocator.h"
 #include "modules/video_coding/utility/vp8_constants.h"
@@ -54,6 +58,8 @@
 #include "video/adaptation/video_stream_encoder_resource_manager.h"
 #include "video/alignment_adjuster.h"
 #include "video/config/encoder_stream_factory.h"
+#include "video/config/video_encoder_config.h"
+#include "video/corruption_detection/frame_instrumentation_generator.h"
 #include "video/frame_cadence_adapter.h"
 #include "video/frame_dumping_encoder.h"
 
@@ -757,6 +763,7 @@ void VideoStreamEncoder::Stop() {
     ReleaseEncoder();
     encoder_ = nullptr;
     frame_cadence_adapter_ = nullptr;
+    frame_instrumentation_generator_ = nullptr;
   });
   shutdown_event.Wait(rtc::Event::kForever);
 }
@@ -934,6 +941,12 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
     max_data_payload_length_ = max_data_payload_length;
     pending_encoder_reconfiguration_ = true;
 
+    if (settings_.enable_frame_instrumentation_generator) {
+      frame_instrumentation_generator_ =
+          std::make_unique<FrameInstrumentationGenerator>(
+              encoder_config_.codec_type);
+    }
+
     // Reconfigure the encoder now if the frame resolution is known.
     // Otherwise, the reconfiguration is deferred until the next frame to
     // minimize the number of reconfigurations. The codec configuration
@@ -1532,6 +1545,9 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time,
 
   encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(),
                                            incoming_frame.height());
+  if (frame_instrumentation_generator_) {
+    frame_instrumentation_generator_->OnCapturedFrame(incoming_frame);
+  }
   ++captured_frame_count_;
   bool cwnd_frame_drop =
       cwnd_frame_drop_interval_ &&
@@ -2164,6 +2180,22 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage(
   // running in parallel on different threads.
   encoder_stats_observer_->OnSendEncodedImage(image_copy, codec_specific_info);
 
+  std::unique_ptr<CodecSpecificInfo> codec_specific_info_copy;
+  if (codec_specific_info && frame_instrumentation_generator_) {
+    std::optional<
+        absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+        frame_instrumentation_data =
+            frame_instrumentation_generator_->OnEncodedImage(image_copy);
+    RTC_CHECK(!codec_specific_info->frame_instrumentation_data.has_value())
+        << "CodecSpecificInfo must not have frame_instrumentation_data set.";
+    if (frame_instrumentation_data.has_value()) {
+      codec_specific_info_copy =
+          std::make_unique<CodecSpecificInfo>(*codec_specific_info);
+      codec_specific_info_copy->frame_instrumentation_data =
+          frame_instrumentation_data;
+      codec_specific_info = codec_specific_info_copy.get();
+    }
+  }
   EncodedImageCallback::Result result =
       sink_->OnEncodedImage(image_copy, codec_specific_info);
 

video/video_stream_encoder.h

@@ -44,6 +44,7 @@
 #include "rtc_base/rate_statistics.h"
 #include "rtc_base/thread_annotations.h"
 #include "video/adaptation/video_stream_encoder_resource_manager.h"
+#include "video/corruption_detection/frame_instrumentation_generator.h"
 #include "video/encoder_bitrate_adjuster.h"
 #include "video/frame_cadence_adapter.h"
 #include "video/frame_encode_metadata_writer.h"
@@ -447,6 +448,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
   ScopedTaskSafety task_safety_;
 
   std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue_;
+
+  //  Required for automatic corruption detection.
+  std::unique_ptr<FrameInstrumentationGenerator>
+      frame_instrumentation_generator_;
 };
 
 }  // namespace webrtc

video/video_stream_encoder_unittest.cc

@@ -1,4 +1,3 @@
-
 /*
  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
  *
@@ -14,10 +13,12 @@
 #include <cstdint>
 #include <limits>
 #include <memory>
+#include <optional>
 #include <tuple>
 #include <utility>
 
 #include "absl/memory/memory.h"
+#include "absl/types/variant.h"
 #include "api/environment/environment.h"
 #include "api/environment/environment_factory.h"
 #include "api/field_trials_view.h"
@@ -31,6 +32,7 @@
 #include "api/units/data_rate.h"
 #include "api/units/time_delta.h"
 #include "api/video/builtin_video_bitrate_allocator_factory.h"
+#include "api/video/encoded_image.h"
 #include "api/video/i420_buffer.h"
 #include "api/video/nv12_buffer.h"
 #include "api/video/resolution.h"
@@ -44,6 +46,7 @@
 #include "api/video_codecs/vp8_temporal_layers_factory.h"
 #include "call/adaptation/test/fake_adaptation_constraint.h"
 #include "call/adaptation/test/fake_resource.h"
+#include "common_video/frame_instrumentation_data.h"
 #include "common_video/h264/h264_common.h"
 #include "common_video/include/video_frame_buffer.h"
 #include "media/base/video_adapter.h"
@@ -54,6 +57,7 @@
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
 #include "modules/video_coding/codecs/vp9/svc_config.h"
+#include "modules/video_coding/include/video_codec_interface.h"
 #include "modules/video_coding/utility/quality_scaler.h"
 #include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "modules/video_coding/utility/vp8_constants.h"
@@ -63,6 +67,7 @@
 #include "rtc_base/logging.h"
 #include "rtc_base/ref_counted_object.h"
 #include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
 #include "system_wrappers/include/metrics.h"
 #include "test/encoder_settings.h"
 #include "test/fake_encoder.h"
@@ -1513,6 +1518,13 @@ class VideoStreamEncoderTest : public ::testing::Test {
       return number_of_layers_allocations_;
     }
 
+    std::optional<
+        absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+    GetLastFrameInstrumentationData() const {
+      MutexLock lock(&mutex_);
+      return last_frame_instrumentation_data_;
+    }
+
    private:
     Result OnEncodedImage(
         const EncodedImage& encoded_image,
@@ -1538,6 +1550,11 @@ class VideoStreamEncoderTest : public ::testing::Test {
       if (num_received_layers_ == num_expected_layers_) {
         encoded_frame_event_.Set();
       }
+      if (codec_specific_info &&
+          codec_specific_info->frame_instrumentation_data.has_value()) {
+        last_frame_instrumentation_data_ =
+            codec_specific_info->frame_instrumentation_data;
+      }
 
       return Result(Result::OK, last_timestamp_);
     }
@@ -1597,6 +1614,9 @@ class VideoStreamEncoderTest : public ::testing::Test {
     int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
     VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
     int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
+    std::optional<
+        absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+        last_frame_instrumentation_data_ RTC_GUARDED_BY(&mutex_);
   };
 
   class VideoBitrateAllocatorProxyFactory
@@ -1664,6 +1684,44 @@ TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
   video_stream_encoder_->Stop();
 }
 
+TEST_F(VideoStreamEncoderTest, PopulatesFrameInstrumentationDataWhenSetTo) {
+  video_send_config_.encoder_settings.enable_frame_instrumentation_generator =
+      true;
+  ConfigureEncoder(video_encoder_config_.Copy());
+  video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+      kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+  // We need a QP for the encoded frame.
+  fake_encoder_.SetEncodedImageData(EncodedImageBuffer::Create(
+      kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+  video_source_.IncomingCapturedFrame(
+      CreateFrame(1, codec_width_, codec_height_));
+  WaitForEncodedFrame(1);
+
+  EXPECT_TRUE(sink_.GetLastFrameInstrumentationData().has_value());
+  video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+       DoesNotPopulateFrameInstrumentationDataWhenSetNotTo) {
+  video_send_config_.encoder_settings.enable_frame_instrumentation_generator =
+      false;
+  ConfigureEncoder(video_encoder_config_.Copy());
+  video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+      kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+  // We need a QP for the encoded frame.
+  fake_encoder_.SetEncodedImageData(EncodedImageBuffer::Create(
+      kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+  rtc::Event frame_destroyed_event;
+  video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
+  WaitForEncodedFrame(1);
+
+  EXPECT_FALSE(sink_.GetLastFrameInstrumentationData().has_value());
+  video_stream_encoder_->Stop();
+  EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
+}
+
 TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
   // Dropped since no target bitrate has been set.
   rtc::Event frame_destroyed_event;