From a49ab28fca40e8a33099e0f26ad34abfcd93e11e Mon Sep 17 00:00:00 2001 From: Fanny Linderborg Date: Tue, 1 Oct 2024 10:50:45 +0200 Subject: [PATCH] Set CodecSpecific.FrameInstrumentationData in RtpFrameObject ctor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:358039777 Change-Id: Ib0a663f06b293c62a4eb0689b82b3bf919cff25f Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/364282 Commit-Queue: Fanny Linderborg Auto-Submit: Fanny Linderborg Reviewed-by: Rasmus Brandt Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/main@{#43136} --- api/video/rtp_video_frame_assembler.cc | 33 ++++++++++--------- modules/rtp_rtcp/source/frame_object.cc | 7 ++++ modules/rtp_rtcp/source/frame_object.h | 5 +++ ...eam_receiver_frame_transformer_delegate.cc | 3 +- ...ver_frame_transformer_delegate_unittest.cc | 5 +-- .../rtp_frame_reference_finder_unittest.cc | 2 ++ .../rtp_vp8_ref_finder_unittest.cc | 2 ++ .../rtp_vp9_ref_finder_unittest.cc | 2 ++ .../rtp_frame_reference_finder_fuzzer.cc | 1 + video/buffered_frame_decryptor_unittest.cc | 2 ++ video/rtp_video_stream_receiver2.cc | 33 ++++++++++--------- 11 files changed, 60 insertions(+), 35 deletions(-) diff --git a/api/video/rtp_video_frame_assembler.cc b/api/video/rtp_video_frame_assembler.cc index 5dcb8f34be..2485eb34a8 100644 --- a/api/video/rtp_video_frame_assembler.cc +++ b/api/video/rtp_video_frame_assembler.cc @@ -175,22 +175,23 @@ RtpVideoFrameAssembler::Impl::AssembleFrames( const video_coding::PacketBuffer::Packet& last_packet = *packet; result.push_back(std::make_unique( - first_packet->seq_num(), // - last_packet.seq_num(), // - last_packet.marker_bit, // - /*times_nacked=*/0, // - /*first_packet_received_time=*/0, // - /*last_packet_received_time=*/0, // - first_packet->timestamp, // - /*ntp_time_ms=*/0, // - /*timing=*/VideoSendTiming(), // - first_packet->payload_type, // - first_packet->codec(), // - last_packet.video_header.rotation, // - last_packet.video_header.content_type, // - first_packet->video_header, // - last_packet.video_header.color_space, // - /*packet_infos=*/RtpPacketInfos(), // + first_packet->seq_num(), // + last_packet.seq_num(), // + last_packet.marker_bit, // + /*times_nacked=*/0, // + /*first_packet_received_time=*/0, // + /*last_packet_received_time=*/0, // + first_packet->timestamp, // + /*ntp_time_ms=*/0, // + /*timing=*/VideoSendTiming(), // + first_packet->payload_type, // + first_packet->codec(), // + last_packet.video_header.rotation, // + last_packet.video_header.content_type, // + first_packet->video_header, // + last_packet.video_header.color_space, // + last_packet.video_header.frame_instrumentation_data, // + /*packet_infos=*/RtpPacketInfos(), // std::move(bitstream))); } } diff --git a/modules/rtp_rtcp/source/frame_object.cc b/modules/rtp_rtcp/source/frame_object.cc index 5fa82252b1..fdd5ce63c9 100644 --- a/modules/rtp_rtcp/source/frame_object.cc +++ b/modules/rtp_rtcp/source/frame_object.cc @@ -12,10 +12,13 @@ #include +#include #include +#include "absl/types/variant.h" #include "api/video/encoded_image.h" #include "api/video/video_timing.h" +#include "common_video/frame_instrumentation_data.h" #include "rtc_base/checks.h" namespace webrtc { @@ -35,6 +38,9 @@ RtpFrameObject::RtpFrameObject( VideoContentType content_type, const RTPVideoHeader& video_header, const std::optional& color_space, + const std::optional< + absl::variant>& + frame_instrumentation_data, RtpPacketInfos packet_infos, rtc::scoped_refptr image_buffer) : image_buffer_(image_buffer), @@ -49,6 +55,7 @@ RtpFrameObject::RtpFrameObject( // TODO(philipel): Remove when encoded image is replaced by EncodedFrame. // VCMEncodedFrame members + _codecSpecificInfo.frame_instrumentation_data = frame_instrumentation_data; CopyCodecSpecific(&rtp_video_header_); _payloadType = payload_type; SetRtpTimestamp(rtp_timestamp); diff --git a/modules/rtp_rtcp/source/frame_object.h b/modules/rtp_rtcp/source/frame_object.h index 4c57b69ce2..5bc28720c6 100644 --- a/modules/rtp_rtcp/source/frame_object.h +++ b/modules/rtp_rtcp/source/frame_object.h @@ -14,8 +14,10 @@ #include #include +#include "absl/types/variant.h" #include "api/video/encoded_frame.h" #include "api/video/video_frame_metadata.h" +#include "common_video/frame_instrumentation_data.h" namespace webrtc { @@ -36,6 +38,9 @@ class RtpFrameObject : public EncodedFrame { VideoContentType content_type, const RTPVideoHeader& video_header, const std::optional& color_space, + const std::optional>& + frame_instrumentation_data, RtpPacketInfos packet_infos, rtc::scoped_refptr image_buffer); diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc index 5ef0a80492..42402e79b8 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -207,7 +207,8 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( /*rtp_timestamp=*/transformed_frame->GetTimestamp(), /*ntp_time_ms=*/0, timing, transformed_frame->GetPayloadType(), metadata.GetCodec(), metadata.GetRotation(), metadata.GetContentType(), - video_header, video_header.color_space, RtpPacketInfos(), + video_header, video_header.color_space, + video_header.frame_instrumentation_data, RtpPacketInfos(), EncodedImageBuffer::Create(data.data(), data.size()))); } } diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc index 6fd467e952..f7c6e71de0 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -50,8 +51,8 @@ std::unique_ptr CreateRtpFrameObject( /*last_packet_received_time=*/5, /*rtp_timestamp=*/6, /*ntp_time_ms=*/7, VideoSendTiming(), /*payload_type=*/8, video_header.codec, kVideoRotation_0, VideoContentType::UNSPECIFIED, video_header, - std::nullopt, RtpPacketInfos({packet_info}), - EncodedImageBuffer::Create(0)); + /*color_space=*/std::nullopt, /*frame_instrumentation_data=*/std::nullopt, + RtpPacketInfos({packet_info}), EncodedImageBuffer::Create(0)); } std::unique_ptr CreateRtpFrameObject() { diff --git a/modules/video_coding/rtp_frame_reference_finder_unittest.cc b/modules/video_coding/rtp_frame_reference_finder_unittest.cc index 8ade2143f6..ac3caacedf 100644 --- a/modules/video_coding/rtp_frame_reference_finder_unittest.cc +++ b/modules/video_coding/rtp_frame_reference_finder_unittest.cc @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +56,7 @@ std::unique_ptr CreateFrame( VideoContentType::UNSPECIFIED, video_header, /*color_space=*/std::nullopt, + /*frame_instrumentation_data=*/std::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(/*size=*/0)); // clang-format on diff --git a/modules/video_coding/rtp_vp8_ref_finder_unittest.cc b/modules/video_coding/rtp_vp8_ref_finder_unittest.cc index 01ff2336d2..38dc858ebb 100644 --- a/modules/video_coding/rtp_vp8_ref_finder_unittest.cc +++ b/modules/video_coding/rtp_vp8_ref_finder_unittest.cc @@ -10,6 +10,7 @@ #include "modules/video_coding/rtp_vp8_ref_finder.h" +#include #include #include @@ -93,6 +94,7 @@ class Frame { VideoContentType::UNSPECIFIED, video_header, /*color_space=*/std::nullopt, + /*frame_instrumentation_data=*/std::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(/*size=*/0)); // clang-format on diff --git a/modules/video_coding/rtp_vp9_ref_finder_unittest.cc b/modules/video_coding/rtp_vp9_ref_finder_unittest.cc index 35d30ade82..2e5efc67c9 100644 --- a/modules/video_coding/rtp_vp9_ref_finder_unittest.cc +++ b/modules/video_coding/rtp_vp9_ref_finder_unittest.cc @@ -10,6 +10,7 @@ #include "modules/video_coding/rtp_vp9_ref_finder.h" +#include #include #include @@ -129,6 +130,7 @@ class Frame { VideoContentType::UNSPECIFIED, video_header, /*color_space=*/std::nullopt, + /*frame_instrumentation_data=*/std::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(/*size=*/0)); // clang-format on diff --git a/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc b/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc index 5be9837c0d..0839c2edb7 100644 --- a/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc +++ b/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc @@ -157,6 +157,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) { VideoContentType::UNSPECIFIED, video_header, /*color_space=*/std::nullopt, + /*frame_instrumentation_data=*/std::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(/*size=*/0)); // clang-format on diff --git a/video/buffered_frame_decryptor_unittest.cc b/video/buffered_frame_decryptor_unittest.cc index 5089a0e93c..fdfc764c66 100644 --- a/video/buffered_frame_decryptor_unittest.cc +++ b/video/buffered_frame_decryptor_unittest.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include "api/test/mock_frame_decryptor.h" @@ -75,6 +76,7 @@ class BufferedFrameDecryptorTest : public ::testing::Test, VideoContentType::UNSPECIFIED, rtp_video_header, /*color_space=*/std::nullopt, + /*frame_instrumentation_data=*/std::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(/*size=*/0)); // clang-format on diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index d6c859f535..818353cdea 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -790,22 +790,23 @@ void RtpVideoStreamReceiver2::OnInsertedPacket( const video_coding::PacketBuffer::Packet& last_packet = *packet; OnAssembledFrame(std::make_unique( - first_packet->seq_num(), // - last_packet.seq_num(), // - last_packet.marker_bit, // - max_nack_count, // - min_recv_time, // - max_recv_time, // - first_packet->timestamp, // - ntp_estimator_.Estimate(first_packet->timestamp), // - last_packet.video_header.video_timing, // - first_packet->payload_type, // - first_packet->codec(), // - last_packet.video_header.rotation, // - last_packet.video_header.content_type, // - first_packet->video_header, // - last_packet.video_header.color_space, // - RtpPacketInfos(std::move(packet_infos)), // + first_packet->seq_num(), // + last_packet.seq_num(), // + last_packet.marker_bit, // + max_nack_count, // + min_recv_time, // + max_recv_time, // + first_packet->timestamp, // + ntp_estimator_.Estimate(first_packet->timestamp), // + last_packet.video_header.video_timing, // + first_packet->payload_type, // + first_packet->codec(), // + last_packet.video_header.rotation, // + last_packet.video_header.content_type, // + first_packet->video_header, // + last_packet.video_header.color_space, // + last_packet.video_header.frame_instrumentation_data, // + RtpPacketInfos(std::move(packet_infos)), // std::move(bitstream))); payloads.clear(); packet_infos.clear();