Set CodecSpecific.FrameInstrumentationData in RtpFrameObject ctor

Bug: webrtc:358039777
Change-Id: Ib0a663f06b293c62a4eb0689b82b3bf919cff25f
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/364282
Commit-Queue: Fanny Linderborg <linderborg@webrtc.org>
Auto-Submit: Fanny Linderborg <linderborg@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43136}

api/video/rtp_video_frame_assembler.cc

@@ -190,6 +190,7 @@ RtpVideoFrameAssembler::Impl::AssembleFrames(
           last_packet.video_header.content_type,                //
           first_packet->video_header,                           //
           last_packet.video_header.color_space,                 //
+          last_packet.video_header.frame_instrumentation_data,  //
           /*packet_infos=*/RtpPacketInfos(),                    //
           std::move(bitstream)));
     }

modules/rtp_rtcp/source/frame_object.cc

@@ -12,10 +12,13 @@
 
 #include <string.h>
 
+#include <optional>
 #include <utility>
 
+#include "absl/types/variant.h"
 #include "api/video/encoded_image.h"
 #include "api/video/video_timing.h"
+#include "common_video/frame_instrumentation_data.h"
 #include "rtc_base/checks.h"
 
 namespace webrtc {
@@ -35,6 +38,9 @@ RtpFrameObject::RtpFrameObject(
     VideoContentType content_type,
     const RTPVideoHeader& video_header,
     const std::optional<webrtc::ColorSpace>& color_space,
+    const std::optional<
+        absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>&
+        frame_instrumentation_data,
     RtpPacketInfos packet_infos,
     rtc::scoped_refptr<EncodedImageBuffer> image_buffer)
     : image_buffer_(image_buffer),
@@ -49,6 +55,7 @@ RtpFrameObject::RtpFrameObject(
 
   // TODO(philipel): Remove when encoded image is replaced by EncodedFrame.
   // VCMEncodedFrame members
+  _codecSpecificInfo.frame_instrumentation_data = frame_instrumentation_data;
   CopyCodecSpecific(&rtp_video_header_);
   _payloadType = payload_type;
   SetRtpTimestamp(rtp_timestamp);

modules/rtp_rtcp/source/frame_object.h

@@ -14,8 +14,10 @@
 #include <optional>
 #include <vector>
 
+#include "absl/types/variant.h"
 #include "api/video/encoded_frame.h"
 #include "api/video/video_frame_metadata.h"
+#include "common_video/frame_instrumentation_data.h"
 
 namespace webrtc {
 
@@ -36,6 +38,9 @@ class RtpFrameObject : public EncodedFrame {
                  VideoContentType content_type,
                  const RTPVideoHeader& video_header,
                  const std::optional<webrtc::ColorSpace>& color_space,
+                 const std::optional<absl::variant<FrameInstrumentationSyncData,
+                                                   FrameInstrumentationData>>&
+                     frame_instrumentation_data,
                  RtpPacketInfos packet_infos,
                  rtc::scoped_refptr<EncodedImageBuffer> image_buffer);
 

modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc

@@ -207,7 +207,8 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
         /*rtp_timestamp=*/transformed_frame->GetTimestamp(),
         /*ntp_time_ms=*/0, timing, transformed_frame->GetPayloadType(),
         metadata.GetCodec(), metadata.GetRotation(), metadata.GetContentType(),
-        video_header, video_header.color_space, RtpPacketInfos(),
+        video_header, video_header.color_space,
+        video_header.frame_instrumentation_data, RtpPacketInfos(),
         EncodedImageBuffer::Create(data.data(), data.size())));
   }
 }

modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc

@@ -12,6 +12,7 @@
 
 #include <cstdio>
 #include <memory>
+#include <optional>
 #include <utility>
 #include <vector>
 
@@ -50,8 +51,8 @@ std::unique_ptr<RtpFrameObject> CreateRtpFrameObject(
       /*last_packet_received_time=*/5, /*rtp_timestamp=*/6, /*ntp_time_ms=*/7,
       VideoSendTiming(), /*payload_type=*/8, video_header.codec,
       kVideoRotation_0, VideoContentType::UNSPECIFIED, video_header,
-      std::nullopt, RtpPacketInfos({packet_info}),
+      /*color_space=*/std::nullopt, /*frame_instrumentation_data=*/std::nullopt,
-      EncodedImageBuffer::Create(0));
+      RtpPacketInfos({packet_info}), EncodedImageBuffer::Create(0));
 }
 
 std::unique_ptr<RtpFrameObject> CreateRtpFrameObject() {

modules/video_coding/rtp_frame_reference_finder_unittest.cc

@@ -13,6 +13,7 @@
 #include <cstring>
 #include <limits>
 #include <map>
+#include <optional>
 #include <set>
 #include <utility>
 #include <vector>
@@ -55,6 +56,7 @@ std::unique_ptr<RtpFrameObject> CreateFrame(
       VideoContentType::UNSPECIFIED,
       video_header,
       /*color_space=*/std::nullopt,
+      /*frame_instrumentation_data=*/std::nullopt,
       RtpPacketInfos(),
       EncodedImageBuffer::Create(/*size=*/0));
   // clang-format on

modules/video_coding/rtp_vp8_ref_finder_unittest.cc

@@ -10,6 +10,7 @@
 
 #include "modules/video_coding/rtp_vp8_ref_finder.h"
 
+#include <optional>
 #include <utility>
 #include <vector>
 
@@ -93,6 +94,7 @@ class Frame {
         VideoContentType::UNSPECIFIED,
         video_header,
         /*color_space=*/std::nullopt,
+        /*frame_instrumentation_data=*/std::nullopt,
         RtpPacketInfos(),
         EncodedImageBuffer::Create(/*size=*/0));
     // clang-format on

modules/video_coding/rtp_vp9_ref_finder_unittest.cc

@@ -10,6 +10,7 @@
 
 #include "modules/video_coding/rtp_vp9_ref_finder.h"
 
+#include <optional>
 #include <utility>
 #include <vector>
 
@@ -129,6 +130,7 @@ class Frame {
         VideoContentType::UNSPECIFIED,
         video_header,
         /*color_space=*/std::nullopt,
+        /*frame_instrumentation_data=*/std::nullopt,
         RtpPacketInfos(),
         EncodedImageBuffer::Create(/*size=*/0));
     // clang-format on

test/fuzzers/rtp_frame_reference_finder_fuzzer.cc

@@ -157,6 +157,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
         VideoContentType::UNSPECIFIED,
         video_header,
         /*color_space=*/std::nullopt,
+        /*frame_instrumentation_data=*/std::nullopt,
         RtpPacketInfos(),
         EncodedImageBuffer::Create(/*size=*/0));
     // clang-format on

video/buffered_frame_decryptor_unittest.cc

@@ -12,6 +12,7 @@
 
 #include <map>
 #include <memory>
+#include <optional>
 #include <vector>
 
 #include "api/test/mock_frame_decryptor.h"
@@ -75,6 +76,7 @@ class BufferedFrameDecryptorTest : public ::testing::Test,
         VideoContentType::UNSPECIFIED,
         rtp_video_header,
         /*color_space=*/std::nullopt,
+        /*frame_instrumentation_data=*/std::nullopt,
         RtpPacketInfos(),
         EncodedImageBuffer::Create(/*size=*/0));
     // clang-format on

video/rtp_video_stream_receiver2.cc

@@ -805,6 +805,7 @@ void RtpVideoStreamReceiver2::OnInsertedPacket(
           last_packet.video_header.content_type,                //
           first_packet->video_header,                           //
           last_packet.video_header.color_space,                 //
+          last_packet.video_header.frame_instrumentation_data,  //
           RtpPacketInfos(std::move(packet_infos)),              //
           std::move(bitstream)));
       payloads.clear();