From 6aa61a31187bbeb3ebd45945c8a03895086b82cc Mon Sep 17 00:00:00 2001 From: philipel Date: Fri, 15 Oct 2021 10:57:10 +0200 Subject: [PATCH] Return first and last RTP packet sequence number for completed frames. Change-Id: Icab5c36489317ee2dd62bdda7340437abd07eb7e Bug: webrtc:12579 Change-Id: Icab5c36489317ee2dd62bdda7340437abd07eb7e Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/235041 Reviewed-by: Niels Moller Reviewed-by: Danil Chapovalov Commit-Queue: Philip Eliasson Cr-Commit-Position: refs/heads/main@{#35216} --- api/video/rtp_video_frame_assembler.cc | 13 +- api/video/rtp_video_frame_assembler.h | 25 ++- .../rtp_video_frame_assembler_unittests.cc | 187 ++++++++++++++---- 3 files changed, 179 insertions(+), 46 deletions(-) diff --git a/api/video/rtp_video_frame_assembler.cc b/api/video/rtp_video_frame_assembler.cc index 2629040b5b..271ac2225d 100644 --- a/api/video/rtp_video_frame_assembler.cc +++ b/api/video/rtp_video_frame_assembler.cc @@ -187,7 +187,10 @@ RtpVideoFrameAssembler::Impl::FindReferences(RtpFrameVector frames) { for (auto& frame : frames) { auto complete_frames = reference_finder_.ManageFrame(std::move(frame)); for (std::unique_ptr& complete_frame : complete_frames) { - res.push_back(std::move(complete_frame)); + uint16_t rtp_seq_num_start = complete_frame->first_seq_num(); + uint16_t rtp_seq_num_end = complete_frame->last_seq_num(); + res.emplace_back(rtp_seq_num_start, rtp_seq_num_end, + std::move(complete_frame)); } } return res; @@ -199,8 +202,12 @@ RtpVideoFrameAssembler::Impl::UpdateWithPadding(uint16_t seq_num) { FindReferences(AssembleFrames(packet_buffer_.InsertPadding(seq_num))); auto ref_finder_update = reference_finder_.PaddingReceived(seq_num); - res.insert(res.end(), std::make_move_iterator(ref_finder_update.begin()), - std::make_move_iterator(ref_finder_update.end())); + for (std::unique_ptr& complete_frame : ref_finder_update) { + uint16_t rtp_seq_num_start = complete_frame->first_seq_num(); + uint16_t rtp_seq_num_end = complete_frame->last_seq_num(); + res.emplace_back(rtp_seq_num_start, rtp_seq_num_end, + std::move(complete_frame)); + } return res; } diff --git a/api/video/rtp_video_frame_assembler.h b/api/video/rtp_video_frame_assembler.h index 353942bdc8..83162cb818 100644 --- a/api/video/rtp_video_frame_assembler.h +++ b/api/video/rtp_video_frame_assembler.h @@ -13,6 +13,7 @@ #include #include +#include #include "absl/container/inlined_vector.h" #include "api/video/encoded_frame.h" @@ -26,9 +27,31 @@ namespace webrtc { // monotonic in decode order, dependencies are expressed as frame IDs. class RtpVideoFrameAssembler { public: + // The RtpVideoFrameAssembler should return "RTP frames", but for now there + // is no good class for this purpose. For now return an EncodedFrame bundled + // with some minimal RTP information. + class AssembledFrame { + public: + AssembledFrame(uint16_t rtp_seq_num_start, + uint16_t rtp_seq_num_end, + std::unique_ptr frame) + : rtp_seq_num_start_(rtp_seq_num_start), + rtp_seq_num_end_(rtp_seq_num_end), + frame_(std::move(frame)) {} + + uint16_t RtpSeqNumStart() const { return rtp_seq_num_start_; } + uint16_t RtpSeqNumEnd() const { return rtp_seq_num_end_; } + std::unique_ptr ExtractFrame() { return std::move(frame_); } + + private: + uint16_t rtp_seq_num_start_; + uint16_t rtp_seq_num_end_; + std::unique_ptr frame_; + }; + // FrameVector is just a vector-like type of std::unique_ptr. // The vector type may change without notice. - using FrameVector = absl::InlinedVector, 3>; + using FrameVector = absl::InlinedVector; enum PayloadFormat { kRaw, kH264, kVp8, kVp9, kAv1, kGeneric }; explicit RtpVideoFrameAssembler(PayloadFormat payload_format); diff --git a/api/video/rtp_video_frame_assembler_unittests.cc b/api/video/rtp_video_frame_assembler_unittests.cc index fd70f721c9..74e49064ec 100644 --- a/api/video/rtp_video_frame_assembler_unittests.cc +++ b/api/video/rtp_video_frame_assembler_unittests.cc @@ -162,13 +162,15 @@ TEST(RtpVideoFrameAssembler, Vp8Packetization) { ASSERT_THAT(frames, SizeIs(2)); - EXPECT_THAT(frames[0]->Id(), Eq(10)); - EXPECT_THAT(References(frames[0]), IsEmpty()); - EXPECT_THAT(Payload(frames[0]), ElementsAreArray(kKeyframePayload)); + auto first_frame = frames[0].ExtractFrame(); + EXPECT_THAT(first_frame->Id(), Eq(10)); + EXPECT_THAT(References(first_frame), IsEmpty()); + EXPECT_THAT(Payload(first_frame), ElementsAreArray(kKeyframePayload)); - EXPECT_THAT(frames[1]->Id(), Eq(11)); - EXPECT_THAT(References(frames[1]), UnorderedElementsAre(10)); - EXPECT_THAT(Payload(frames[1]), ElementsAreArray(kDeltaframePayload)); + auto second_frame = frames[1].ExtractFrame(); + EXPECT_THAT(second_frame->Id(), Eq(11)); + EXPECT_THAT(References(second_frame), UnorderedElementsAre(10)); + EXPECT_THAT(Payload(second_frame), ElementsAreArray(kDeltaframePayload)); } TEST(RtpVideoFrameAssembler, Vp9Packetization) { @@ -201,13 +203,15 @@ TEST(RtpVideoFrameAssembler, Vp9Packetization) { ASSERT_THAT(frames, SizeIs(2)); - EXPECT_THAT(frames[0]->Id(), Eq(10)); - EXPECT_THAT(Payload(frames[0]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[0]), IsEmpty()); + auto first_frame = frames[0].ExtractFrame(); + EXPECT_THAT(first_frame->Id(), Eq(10)); + EXPECT_THAT(Payload(first_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(first_frame), IsEmpty()); - EXPECT_THAT(frames[1]->Id(), Eq(11)); - EXPECT_THAT(Payload(frames[1]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[1]), UnorderedElementsAre(10)); + auto second_frame = frames[1].ExtractFrame(); + EXPECT_THAT(second_frame->Id(), Eq(11)); + EXPECT_THAT(Payload(second_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(second_frame), UnorderedElementsAre(10)); } TEST(RtpVideoFrameAssembler, Av1Packetization) { @@ -239,13 +243,15 @@ TEST(RtpVideoFrameAssembler, Av1Packetization) { ASSERT_THAT(frames, SizeIs(2)); - EXPECT_THAT(frames[0]->Id(), Eq(20)); - EXPECT_THAT(Payload(frames[0]), ElementsAreArray(kKeyframePayload)); - EXPECT_THAT(References(frames[0]), IsEmpty()); + auto first_frame = frames[0].ExtractFrame(); + EXPECT_THAT(first_frame->Id(), Eq(20)); + EXPECT_THAT(Payload(first_frame), ElementsAreArray(kKeyframePayload)); + EXPECT_THAT(References(first_frame), IsEmpty()); - EXPECT_THAT(frames[1]->Id(), Eq(21)); - EXPECT_THAT(Payload(frames[1]), ElementsAreArray(kDeltaframePayload)); - EXPECT_THAT(References(frames[1]), UnorderedElementsAre(20)); + auto second_frame = frames[1].ExtractFrame(); + EXPECT_THAT(second_frame->Id(), Eq(21)); + EXPECT_THAT(Payload(second_frame), ElementsAreArray(kDeltaframePayload)); + EXPECT_THAT(References(second_frame), UnorderedElementsAre(20)); } TEST(RtpVideoFrameAssembler, RawPacketizationDependencyDescriptorExtension) { @@ -290,13 +296,15 @@ TEST(RtpVideoFrameAssembler, RawPacketizationDependencyDescriptorExtension) { ASSERT_THAT(frames, SizeIs(2)); - EXPECT_THAT(frames[0]->Id(), Eq(10)); - EXPECT_THAT(Payload(frames[0]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[0]), IsEmpty()); + auto first_frame = frames[0].ExtractFrame(); + EXPECT_THAT(first_frame->Id(), Eq(10)); + EXPECT_THAT(Payload(first_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(first_frame), IsEmpty()); - EXPECT_THAT(frames[1]->Id(), Eq(20)); - EXPECT_THAT(Payload(frames[1]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[1]), UnorderedElementsAre(10)); + auto second_frame = frames[1].ExtractFrame(); + EXPECT_THAT(second_frame->Id(), Eq(20)); + EXPECT_THAT(Payload(second_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(second_frame), UnorderedElementsAre(10)); } TEST(RtpVideoFrameAssembler, RawPacketizationGenericDescriptor00Extension) { @@ -329,13 +337,15 @@ TEST(RtpVideoFrameAssembler, RawPacketizationGenericDescriptor00Extension) { ASSERT_THAT(frames, SizeIs(2)); - EXPECT_THAT(frames[0]->Id(), Eq(100)); - EXPECT_THAT(Payload(frames[0]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[0]), IsEmpty()); + auto first_frame = frames[0].ExtractFrame(); + EXPECT_THAT(first_frame->Id(), Eq(100)); + EXPECT_THAT(Payload(first_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(first_frame), IsEmpty()); - EXPECT_THAT(frames[1]->Id(), Eq(102)); - EXPECT_THAT(Payload(frames[1]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[1]), UnorderedElementsAre(100)); + auto second_frame = frames[1].ExtractFrame(); + EXPECT_THAT(second_frame->Id(), Eq(102)); + EXPECT_THAT(Payload(second_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(second_frame), UnorderedElementsAre(100)); } TEST(RtpVideoFrameAssembler, RawPacketizationGenericPayloadDescriptor) { @@ -363,13 +373,15 @@ TEST(RtpVideoFrameAssembler, RawPacketizationGenericPayloadDescriptor) { ASSERT_THAT(frames, SizeIs(2)); - EXPECT_THAT(frames[0]->Id(), Eq(123)); - EXPECT_THAT(Payload(frames[0]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[0]), IsEmpty()); + auto first_frame = frames[0].ExtractFrame(); + EXPECT_THAT(first_frame->Id(), Eq(123)); + EXPECT_THAT(Payload(first_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(first_frame), IsEmpty()); - EXPECT_THAT(frames[1]->Id(), Eq(124)); - EXPECT_THAT(Payload(frames[1]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[1]), UnorderedElementsAre(123)); + auto second_frame = frames[1].ExtractFrame(); + EXPECT_THAT(second_frame->Id(), Eq(124)); + EXPECT_THAT(Payload(second_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(second_frame), UnorderedElementsAre(123)); } TEST(RtpVideoFrameAssembler, Padding) { @@ -396,16 +408,18 @@ TEST(RtpVideoFrameAssembler, Padding) { frames); ASSERT_THAT(frames, SizeIs(1)); - EXPECT_THAT(frames[0]->Id(), Eq(123)); - EXPECT_THAT(Payload(frames[0]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[0]), IsEmpty()); + auto first_frame = frames[0].ExtractFrame(); + EXPECT_THAT(first_frame->Id(), Eq(123)); + EXPECT_THAT(Payload(first_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(first_frame), IsEmpty()); AppendFrames(assembler.InsertPacket(PaddingPacket(/*seq_num=*/124)), frames); ASSERT_THAT(frames, SizeIs(2)); - EXPECT_THAT(frames[1]->Id(), Eq(125)); - EXPECT_THAT(Payload(frames[1]), ElementsAreArray(kPayload)); - EXPECT_THAT(References(frames[1]), UnorderedElementsAre(123)); + auto second_frame = frames[1].ExtractFrame(); + EXPECT_THAT(second_frame->Id(), Eq(125)); + EXPECT_THAT(Payload(second_frame), ElementsAreArray(kPayload)); + EXPECT_THAT(References(second_frame), UnorderedElementsAre(123)); } TEST(RtpVideoFrameAssembler, ClearOldPackets) { @@ -476,5 +490,94 @@ TEST(RtpVideoFrameAssembler, ClearOldPacketsWithPadding) { SizeIs(1)); } +TEST(RtpVideoFrameAssembler, SeqNumStartAndSeqNumEndSet) { + RtpVideoFrameAssembler assembler(RtpVideoFrameAssembler::kGeneric); + RtpVideoFrameAssembler::FrameVector frames; + uint8_t kPayload[] = + "Some payload that will get split into two when packetized."; + + RTPVideoHeader video_header; + video_header.frame_type = VideoFrameType::kVideoFrameKey; + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = sizeof(kPayload) - 1; + + auto packetizer = + RtpPacketizer::Create(kVideoCodecGeneric, kPayload, limits, video_header); + ASSERT_THAT(packetizer->NumPackets(), Eq(2U)); + + RtpPacketReceived::ExtensionManager extension_manager; + { + RtpPacketToSend send_packet(&extension_manager); + packetizer->NextPacket(&send_packet); + send_packet.SetSequenceNumber(123); + RtpPacketReceived received_packet(&extension_manager); + received_packet.Parse(send_packet.Buffer()); + assembler.InsertPacket(received_packet); + } + + { + RtpPacketToSend send_packet(&extension_manager); + packetizer->NextPacket(&send_packet); + send_packet.SetSequenceNumber(124); + RtpPacketReceived received_packet(&extension_manager); + received_packet.Parse(send_packet.Buffer()); + AppendFrames(assembler.InsertPacket(received_packet), frames); + } + + ASSERT_THAT(frames, SizeIs(1)); + EXPECT_THAT(frames[0].RtpSeqNumStart(), Eq(123)); + EXPECT_THAT(frames[0].RtpSeqNumEnd(), Eq(124)); +} + +TEST(RtpVideoFrameAssembler, SeqNumStartAndSeqNumEndSetWhenPaddingReceived) { + RtpVideoFrameAssembler assembler(RtpVideoFrameAssembler::kGeneric); + RtpVideoFrameAssembler::FrameVector frames; + uint8_t kPayload[] = + "Some payload that will get split into two when packetized."; + + RTPVideoHeader video_header; + video_header.frame_type = VideoFrameType::kVideoFrameKey; + + EXPECT_THAT(assembler.InsertPacket(PacketBuilder(PayloadFormat::kGeneric) + .WithPayload(kPayload) + .WithVideoHeader(video_header) + .WithSeqNum(121) + .Build()), + SizeIs(1)); + + video_header.frame_type = VideoFrameType::kVideoFrameDelta; + RtpPacketReceived::ExtensionManager extension_manager; + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = sizeof(kPayload) - 1; + + auto packetizer = + RtpPacketizer::Create(kVideoCodecGeneric, kPayload, limits, video_header); + ASSERT_THAT(packetizer->NumPackets(), Eq(2U)); + + { + RtpPacketToSend send_packet(&extension_manager); + packetizer->NextPacket(&send_packet); + send_packet.SetSequenceNumber(123); + RtpPacketReceived received_packet(&extension_manager); + received_packet.Parse(send_packet.Buffer()); + assembler.InsertPacket(received_packet); + } + + { + RtpPacketToSend send_packet(&extension_manager); + packetizer->NextPacket(&send_packet); + send_packet.SetSequenceNumber(124); + RtpPacketReceived received_packet(&extension_manager); + received_packet.Parse(send_packet.Buffer()); + assembler.InsertPacket(received_packet); + } + + AppendFrames(assembler.InsertPacket(PaddingPacket(/*seq_num=*/122)), frames); + + ASSERT_THAT(frames, SizeIs(1)); + EXPECT_THAT(frames[0].RtpSeqNumStart(), Eq(123)); + EXPECT_THAT(frames[0].RtpSeqNumEnd(), Eq(124)); +} + } // namespace } // namespace webrtc