diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index 73abbbe25a..98307842fd 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -21,15 +21,12 @@ #include "absl/types/variant.h" #include "api/array_view.h" #include "api/rtp_packet_info.h" -#include "api/video/encoded_frame.h" #include "api/video/video_frame_type.h" #include "common_video/h264/h264_common.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" -#include "modules/rtp_rtcp/source/video_rtp_depacketizer_av1.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" -#include "modules/video_coding/frame_object.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/mod_ops.h" @@ -135,7 +132,7 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket( UpdateMissingPackets(seq_num); - result.frames = FindFrames(seq_num); + result.packets = FindFrames(seq_num); return result; } @@ -176,20 +173,6 @@ void PacketBuffer::ClearTo(uint16_t seq_num) { } } -void PacketBuffer::ClearInterval(uint16_t start_seq_num, - uint16_t stop_seq_num) { - size_t iterations = ForwardDiff(start_seq_num, stop_seq_num + 1); - RTC_DCHECK_LE(iterations, buffer_.size()); - uint16_t seq_num = start_seq_num; - for (size_t i = 0; i < iterations; ++i) { - size_t index = seq_num % buffer_.size(); - RTC_DCHECK_EQ(buffer_[index].seq_num(), seq_num); - buffer_[index].packet = nullptr; - - ++seq_num; - } -} - void PacketBuffer::Clear() { rtc::CritScope lock(&crit_); for (StoredPacket& entry : buffer_) { @@ -208,7 +191,7 @@ PacketBuffer::InsertResult PacketBuffer::InsertPadding(uint16_t seq_num) { PacketBuffer::InsertResult result; rtc::CritScope lock(&crit_); UpdateMissingPackets(seq_num); - result.frames = FindFrames(static_cast(seq_num + 1)); + result.packets = FindFrames(static_cast(seq_num + 1)); return result; } @@ -265,15 +248,15 @@ bool PacketBuffer::PotentialNewFrame(uint16_t seq_num) const { return false; } -std::vector> PacketBuffer::FindFrames( +std::vector> PacketBuffer::FindFrames( uint16_t seq_num) { - std::vector> found_frames; + std::vector> found_frames; for (size_t i = 0; i < buffer_.size() && PotentialNewFrame(seq_num); ++i) { size_t index = seq_num % buffer_.size(); buffer_[index].continuous = true; // If all packets of the frame is continuous, find the first packet of the - // frame and create an RtpFrameObject. + // frame and add all packets of the frame to the returned packets. if (buffer_[index].frame_end()) { uint16_t start_seq_num = seq_num; @@ -393,97 +376,29 @@ std::vector> PacketBuffer::FindFrames( } } - if (auto frame = AssembleFrame(start_seq_num, seq_num)) { - found_frames.push_back(std::move(frame)); - } else { - RTC_LOG(LS_ERROR) << "Failed to assemble frame from packets " - << start_seq_num << "-" << seq_num; + const uint16_t end_seq_num = seq_num + 1; + // Use uint16_t type to handle sequence number wrap around case. + uint16_t num_packets = end_seq_num - start_seq_num; + found_frames.reserve(found_frames.size() + num_packets); + for (uint16_t i = start_seq_num; i != end_seq_num; ++i) { + StoredPacket& entry = buffer_[i % buffer_.size()]; + RTC_DCHECK(entry.used()); + RTC_DCHECK_EQ(i, entry.seq_num()); + // Ensure frame boundary flags are properly set. + entry.packet->video_header.is_first_packet_in_frame = + (i == start_seq_num); + entry.packet->video_header.is_last_packet_in_frame = (i == seq_num); + found_frames.push_back(std::move(entry.packet)); } missing_packets_.erase(missing_packets_.begin(), missing_packets_.upper_bound(seq_num)); - ClearInterval(start_seq_num, seq_num); } ++seq_num; } return found_frames; } -std::unique_ptr PacketBuffer::AssembleFrame( - uint16_t first_seq_num, - uint16_t last_seq_num) { - const uint16_t end_seq_num = last_seq_num + 1; - const uint16_t num_packets = end_seq_num - first_seq_num; - int max_nack_count = -1; - int64_t min_recv_time = std::numeric_limits::max(); - int64_t max_recv_time = std::numeric_limits::min(); - size_t frame_size = 0; - - std::vector> payloads; - RtpPacketInfos::vector_type packet_infos; - payloads.reserve(num_packets); - packet_infos.reserve(num_packets); - - for (uint16_t seq_num = first_seq_num; seq_num != end_seq_num; ++seq_num) { - const Packet& packet = GetPacket(seq_num); - - max_nack_count = std::max(max_nack_count, packet.times_nacked); - min_recv_time = - std::min(min_recv_time, packet.packet_info.receive_time_ms()); - max_recv_time = - std::max(max_recv_time, packet.packet_info.receive_time_ms()); - frame_size += packet.video_payload.size(); - payloads.emplace_back(packet.video_payload); - packet_infos.push_back(packet.packet_info); - } - - const Packet& first_packet = GetPacket(first_seq_num); - rtc::scoped_refptr bitstream; - // TODO(danilchap): Hide codec-specific code paths behind an interface. - if (first_packet.codec() == VideoCodecType::kVideoCodecAV1) { - bitstream = VideoRtpDepacketizerAv1::AssembleFrame(payloads); - if (!bitstream) { - // Failed to assemble a frame. Discard and continue. - return nullptr; - } - } else { - bitstream = EncodedImageBuffer::Create(frame_size); - - uint8_t* write_at = bitstream->data(); - for (rtc::ArrayView payload : payloads) { - memcpy(write_at, payload.data(), payload.size()); - write_at += payload.size(); - } - RTC_DCHECK_EQ(write_at - bitstream->data(), bitstream->size()); - } - const Packet& last_packet = GetPacket(last_seq_num); - return std::make_unique( - first_seq_num, // - last_seq_num, // - last_packet.marker_bit, // - max_nack_count, // - min_recv_time, // - max_recv_time, // - first_packet.timestamp, // - first_packet.ntp_time_ms, // - last_packet.video_header.video_timing, // - first_packet.payload_type, // - first_packet.codec(), // - last_packet.video_header.rotation, // - last_packet.video_header.content_type, // - first_packet.video_header, // - last_packet.video_header.color_space, // - RtpPacketInfos(std::move(packet_infos)), // - std::move(bitstream)); -} - -const PacketBuffer::Packet& PacketBuffer::GetPacket(uint16_t seq_num) const { - const StoredPacket& entry = buffer_[seq_num % buffer_.size()]; - RTC_DCHECK(entry.used()); - RTC_DCHECK_EQ(seq_num, entry.seq_num()); - return *entry.packet; -} - void PacketBuffer::UpdateMissingPackets(uint16_t seq_num) { if (!newest_inserted_seq_num_) newest_inserted_seq_num_ = seq_num; diff --git a/modules/video_coding/packet_buffer.h b/modules/video_coding/packet_buffer.h index 5ce67bafef..1877cce2b7 100644 --- a/modules/video_coding/packet_buffer.h +++ b/modules/video_coding/packet_buffer.h @@ -21,7 +21,6 @@ #include "api/video/encoded_image.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" -#include "modules/video_coding/frame_object.h" #include "rtc_base/copy_on_write_buffer.h" #include "rtc_base/critical_section.h" #include "rtc_base/numerics/sequence_number_util.h" @@ -70,7 +69,7 @@ class PacketBuffer { RtpPacketInfo packet_info; }; struct InsertResult { - std::vector> frames; + std::vector> packets; // Indicates if the packet buffer was cleared, which means that a key // frame request should be sent. bool buffer_cleared = false; @@ -118,22 +117,9 @@ class PacketBuffer { bool PotentialNewFrame(uint16_t seq_num) const RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); - // Test if all packets of a frame has arrived, and if so, creates a frame. - // Returns a vector of received frames. - std::vector> FindFrames(uint16_t seq_num) - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); - - std::unique_ptr AssembleFrame(uint16_t first_seq_num, - uint16_t last_seq_num) - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); - - // Get the packet with sequence number |seq_num|. - const Packet& GetPacket(uint16_t seq_num) const - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); - - // Clears the packet buffer from |start_seq_num| to |stop_seq_num| where the - // endpoints are inclusive. - void ClearInterval(uint16_t start_seq_num, uint16_t stop_seq_num) + // Test if all packets of a frame has arrived, and if so, returns packets to + // create frames. + std::vector> FindFrames(uint16_t seq_num) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); void UpdateMissingPackets(uint16_t seq_num) diff --git a/modules/video_coding/packet_buffer_unittest.cc b/modules/video_coding/packet_buffer_unittest.cc index 7779999fc1..242fff2526 100644 --- a/modules/video_coding/packet_buffer_unittest.cc +++ b/modules/video_coding/packet_buffer_unittest.cc @@ -40,29 +40,39 @@ constexpr int kMaxSize = 64; void IgnoreResult(PacketBuffer::InsertResult /*result*/) {} +// Validates frame boundaries are valid and returns first sequence_number for +// each frame. std::vector StartSeqNums( - rtc::ArrayView> frames) { + rtc::ArrayView> packets) { std::vector result; - for (const auto& frame : frames) { - result.push_back(frame->first_seq_num()); + bool frame_boundary = true; + for (const auto& packet : packets) { + EXPECT_EQ(frame_boundary, packet->is_first_packet_in_frame()); + if (packet->is_first_packet_in_frame()) { + result.push_back(packet->seq_num); + } + frame_boundary = packet->is_last_packet_in_frame(); } + EXPECT_TRUE(frame_boundary); return result; } MATCHER_P(StartSeqNumsAre, seq_num, "") { - return Matches(ElementsAre(seq_num))(StartSeqNums(arg.frames)); + return Matches(ElementsAre(seq_num))(StartSeqNums(arg.packets)); } MATCHER_P2(StartSeqNumsAre, seq_num1, seq_num2, "") { - return Matches(ElementsAre(seq_num1, seq_num2))(StartSeqNums(arg.frames)); + return Matches(ElementsAre(seq_num1, seq_num2))(StartSeqNums(arg.packets)); } MATCHER(KeyFrame, "") { - return arg->frame_type() == VideoFrameType::kVideoFrameKey; + return arg->is_first_packet_in_frame() && + arg->video_header.frame_type == VideoFrameType::kVideoFrameKey; } MATCHER(DeltaFrame, "") { - return arg->frame_type() == VideoFrameType::kVideoFrameDelta; + return arg->is_first_packet_in_frame() && + arg->video_header.frame_type == VideoFrameType::kVideoFrameDelta; } struct PacketBufferInsertResult : public PacketBuffer::InsertResult { @@ -72,18 +82,15 @@ struct PacketBufferInsertResult : public PacketBuffer::InsertResult { void PrintTo(const PacketBufferInsertResult& result, std::ostream* os) { *os << "frames: { "; - for (size_t i = 0; i < result.frames.size(); ++i) { - const RtpFrameObject& frame = *result.frames[i]; - if (i > 0) { - *os << ", "; + for (const auto& packet : result.packets) { + if (packet->is_first_packet_in_frame() && + packet->is_last_packet_in_frame()) { + *os << "{sn: " << packet->seq_num << " }"; + } else if (packet->is_first_packet_in_frame()) { + *os << "{sn: [" << packet->seq_num << "-"; + } else if (packet->is_last_packet_in_frame()) { + *os << packet->seq_num << "] }, "; } - *os << "{sn: "; - if (frame.first_seq_num() == frame.last_seq_num()) { - *os << frame.first_seq_num(); - } else { - *os << "[" << frame.first_seq_num() << "-" << frame.last_seq_num() << "]"; - } - *os << "}"; } *os << " }"; if (result.buffer_cleared) { @@ -134,23 +141,23 @@ class PacketBufferTest : public ::testing::Test { TEST_F(PacketBufferTest, InsertOnePacket) { const uint16_t seq_num = Rand(); - EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kLast).frames, SizeIs(1)); + EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kLast).packets, SizeIs(1)); } TEST_F(PacketBufferTest, InsertMultiplePackets) { const uint16_t seq_num = Rand(); - EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kLast).frames, SizeIs(1)); - EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kFirst, kLast).frames, SizeIs(1)); - EXPECT_THAT(Insert(seq_num + 2, kKeyFrame, kFirst, kLast).frames, SizeIs(1)); - EXPECT_THAT(Insert(seq_num + 3, kKeyFrame, kFirst, kLast).frames, SizeIs(1)); + EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kLast).packets, SizeIs(1)); + EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kFirst, kLast).packets, SizeIs(1)); + EXPECT_THAT(Insert(seq_num + 2, kKeyFrame, kFirst, kLast).packets, SizeIs(1)); + EXPECT_THAT(Insert(seq_num + 3, kKeyFrame, kFirst, kLast).packets, SizeIs(1)); } TEST_F(PacketBufferTest, InsertDuplicatePacket) { const uint16_t seq_num = Rand(); - EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).frames, IsEmpty()); - EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).frames, IsEmpty()); - EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kNotFirst, kLast).frames, - SizeIs(1)); + EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).packets, IsEmpty()); + EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).packets, IsEmpty()); + EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kNotFirst, kLast).packets, + SizeIs(2)); } TEST_F(PacketBufferTest, SeqNumWrapOneFrame) { @@ -166,57 +173,17 @@ TEST_F(PacketBufferTest, SeqNumWrapTwoFrames) { } TEST_F(PacketBufferTest, InsertOldPackets) { - EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).frames, IsEmpty()); - EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).frames, SizeIs(1)); - EXPECT_THAT(Insert(101, kKeyFrame, kNotFirst, kLast).frames, SizeIs(1)); + EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).packets, IsEmpty()); + EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).packets, SizeIs(1)); + EXPECT_THAT(Insert(101, kKeyFrame, kNotFirst, kLast).packets, SizeIs(2)); - EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).frames, IsEmpty()); - EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).frames, IsEmpty()); - EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).frames, SizeIs(1)); + EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).packets, IsEmpty()); + EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).packets, IsEmpty()); + EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).packets, SizeIs(1)); packet_buffer_.ClearTo(102); - EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).frames, IsEmpty()); - EXPECT_THAT(Insert(103, kDeltaFrame, kFirst, kLast).frames, SizeIs(1)); -} - -TEST_F(PacketBufferTest, NackCount) { - const uint16_t seq_num = Rand(); - - auto packet = std::make_unique(); - packet->video_header.codec = kVideoCodecGeneric; - packet->seq_num = seq_num; - packet->video_header.frame_type = VideoFrameType::kVideoFrameKey; - packet->video_header.is_first_packet_in_frame = true; - packet->video_header.is_last_packet_in_frame = false; - packet->times_nacked = 0; - IgnoreResult(packet_buffer_.InsertPacket(std::move(packet))); - - packet = std::make_unique(); - packet->seq_num = seq_num + 1; - packet->video_header.frame_type = VideoFrameType::kVideoFrameKey; - packet->video_header.is_first_packet_in_frame = false; - packet->video_header.is_last_packet_in_frame = false; - packet->times_nacked = 1; - IgnoreResult(packet_buffer_.InsertPacket(std::move(packet))); - - packet = std::make_unique(); - packet->seq_num = seq_num + 2; - packet->video_header.frame_type = VideoFrameType::kVideoFrameKey; - packet->video_header.is_first_packet_in_frame = false; - packet->video_header.is_last_packet_in_frame = false; - packet->times_nacked = 3; - IgnoreResult(packet_buffer_.InsertPacket(std::move(packet))); - - packet = std::make_unique(); - packet->seq_num = seq_num + 3; - packet->video_header.frame_type = VideoFrameType::kVideoFrameKey; - packet->video_header.is_first_packet_in_frame = false; - packet->video_header.is_last_packet_in_frame = true; - packet->times_nacked = 1; - auto frames = packet_buffer_.InsertPacket(std::move(packet)).frames; - - ASSERT_THAT(frames, SizeIs(1)); - EXPECT_EQ(frames.front()->times_nacked(), 3); + EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).packets, IsEmpty()); + EXPECT_THAT(Insert(103, kDeltaFrame, kFirst, kLast).packets, SizeIs(1)); } TEST_F(PacketBufferTest, FrameSize) { @@ -229,8 +196,11 @@ TEST_F(PacketBufferTest, FrameSize) { Insert(seq_num, kKeyFrame, kFirst, kNotLast, data1); Insert(seq_num + 1, kKeyFrame, kNotFirst, kNotLast, data2); Insert(seq_num + 2, kKeyFrame, kNotFirst, kNotLast, data3); - EXPECT_THAT(Insert(seq_num + 3, kKeyFrame, kNotFirst, kLast, data4).frames, - ElementsAre(Pointee(SizeIs(20)))); + auto packets = + Insert(seq_num + 3, kKeyFrame, kNotFirst, kLast, data4).packets; + // Expect one frame of 4 packets. + EXPECT_THAT(StartSeqNums(packets), ElementsAre(seq_num)); + EXPECT_THAT(packets, SizeIs(4)); } TEST_F(PacketBufferTest, ExpandBuffer) { @@ -289,7 +259,7 @@ TEST_F(PacketBufferTest, TwoPacketsTwoFrames) { TEST_F(PacketBufferTest, TwoPacketsOneFrames) { const uint16_t seq_num = Rand(); - EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).frames, IsEmpty()); + EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).packets, IsEmpty()); EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kNotFirst, kLast), StartSeqNumsAre(seq_num)); } @@ -297,8 +267,8 @@ TEST_F(PacketBufferTest, TwoPacketsOneFrames) { TEST_F(PacketBufferTest, ThreePacketReorderingOneFrame) { const uint16_t seq_num = Rand(); - EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).frames, IsEmpty()); - EXPECT_THAT(Insert(seq_num + 2, kKeyFrame, kNotFirst, kLast).frames, + EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).packets, IsEmpty()); + EXPECT_THAT(Insert(seq_num + 2, kKeyFrame, kNotFirst, kLast).packets, IsEmpty()); EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kNotFirst, kNotLast), StartSeqNumsAre(seq_num)); @@ -343,7 +313,7 @@ TEST_F(PacketBufferTest, DontClearNewerPacket) { packet_buffer_.ClearTo(0); EXPECT_THAT(Insert(2 * kStartSize, kKeyFrame, kFirst, kLast), StartSeqNumsAre(2 * kStartSize)); - EXPECT_THAT(Insert(3 * kStartSize + 1, kKeyFrame, kFirst, kNotLast).frames, + EXPECT_THAT(Insert(3 * kStartSize + 1, kKeyFrame, kFirst, kNotLast).packets, IsEmpty()); packet_buffer_.ClearTo(2 * kStartSize); EXPECT_THAT(Insert(3 * kStartSize + 2, kKeyFrame, kNotFirst, kLast), @@ -353,10 +323,11 @@ TEST_F(PacketBufferTest, DontClearNewerPacket) { TEST_F(PacketBufferTest, OneIncompleteFrame) { const uint16_t seq_num = Rand(); - EXPECT_THAT(Insert(seq_num, kDeltaFrame, kFirst, kNotLast).frames, IsEmpty()); + EXPECT_THAT(Insert(seq_num, kDeltaFrame, kFirst, kNotLast).packets, + IsEmpty()); EXPECT_THAT(Insert(seq_num + 1, kDeltaFrame, kNotFirst, kLast), StartSeqNumsAre(seq_num)); - EXPECT_THAT(Insert(seq_num - 1, kDeltaFrame, kNotFirst, kLast).frames, + EXPECT_THAT(Insert(seq_num - 1, kDeltaFrame, kNotFirst, kLast).packets, IsEmpty()); } @@ -365,8 +336,9 @@ TEST_F(PacketBufferTest, TwoIncompleteFramesFullBuffer) { for (int i = 1; i < kMaxSize - 1; ++i) Insert(seq_num + i, kDeltaFrame, kNotFirst, kNotLast); - EXPECT_THAT(Insert(seq_num, kDeltaFrame, kFirst, kNotLast).frames, IsEmpty()); - EXPECT_THAT(Insert(seq_num - 1, kDeltaFrame, kNotFirst, kLast).frames, + EXPECT_THAT(Insert(seq_num, kDeltaFrame, kFirst, kNotLast).packets, + IsEmpty()); + EXPECT_THAT(Insert(seq_num - 1, kDeltaFrame, kNotFirst, kLast).packets, IsEmpty()); } @@ -383,111 +355,6 @@ TEST_F(PacketBufferTest, FramesReordered) { StartSeqNumsAre(seq_num + 2)); } -TEST_F(PacketBufferTest, GetBitstream) { - // "many bitstream, such data" with null termination. - uint8_t many[] = {0x6d, 0x61, 0x6e, 0x79, 0x20}; - uint8_t bitstream[] = {0x62, 0x69, 0x74, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x2c, 0x20}; - uint8_t such[] = {0x73, 0x75, 0x63, 0x68, 0x20}; - uint8_t data[] = {0x64, 0x61, 0x74, 0x61, 0x0}; - - const uint16_t seq_num = Rand(); - - Insert(seq_num, kKeyFrame, kFirst, kNotLast, many); - Insert(seq_num + 1, kDeltaFrame, kNotFirst, kNotLast, bitstream); - Insert(seq_num + 2, kDeltaFrame, kNotFirst, kNotLast, such); - auto frames = Insert(seq_num + 3, kDeltaFrame, kNotFirst, kLast, data).frames; - - ASSERT_THAT(frames, SizeIs(1)); - EXPECT_EQ(frames[0]->first_seq_num(), seq_num); - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()), - ElementsAreArray("many bitstream, such data")); -} - -TEST_F(PacketBufferTest, GetBitstreamOneFrameOnePacket) { - uint8_t bitstream[] = "All the bitstream data for this frame!"; - - auto frames = Insert(0, kKeyFrame, kFirst, kLast, bitstream).frames; - ASSERT_THAT(StartSeqNums(frames), ElementsAre(0)); - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()), - ElementsAreArray(bitstream)); -} - -TEST_F(PacketBufferTest, GetBitstreamOneFrameFullBuffer) { - uint8_t data_arr[kStartSize][1]; - uint8_t expected[kStartSize]; - - for (uint8_t i = 0; i < kStartSize; ++i) { - data_arr[i][0] = i; - expected[i] = i; - } - - Insert(0, kKeyFrame, kFirst, kNotLast, data_arr[0]); - for (uint8_t i = 1; i < kStartSize - 1; ++i) - Insert(i, kKeyFrame, kNotFirst, kNotLast, data_arr[i]); - auto frames = Insert(kStartSize - 1, kKeyFrame, kNotFirst, kLast, - data_arr[kStartSize - 1]) - .frames; - - ASSERT_THAT(StartSeqNums(frames), ElementsAre(0)); - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()), - ElementsAreArray(expected)); -} - -TEST_F(PacketBufferTest, GetBitstreamAv1) { - const uint8_t data1[] = {0b01'01'0000, 0b0'0100'000, 'm', 'a', 'n', 'y', ' '}; - const uint8_t data2[] = {0b10'01'0000, 'b', 'i', 't', 's', 0}; - - auto packet1 = std::make_unique(); - packet1->video_header.codec = kVideoCodecAV1; - packet1->seq_num = 13; - packet1->video_header.is_first_packet_in_frame = true; - packet1->video_header.is_last_packet_in_frame = false; - packet1->video_payload = data1; - auto frames = packet_buffer_.InsertPacket(std::move(packet1)).frames; - EXPECT_THAT(frames, IsEmpty()); - - auto packet2 = std::make_unique(); - packet2->video_header.codec = kVideoCodecAV1; - packet2->seq_num = 14; - packet2->video_header.is_first_packet_in_frame = false; - packet2->video_header.is_last_packet_in_frame = true; - packet2->video_payload = data2; - frames = packet_buffer_.InsertPacket(std::move(packet2)).frames; - - ASSERT_THAT(frames, SizeIs(1)); - EXPECT_EQ(frames[0]->first_seq_num(), 13); - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), 2), - ElementsAre(0b0'0100'010, 10)); // obu_header and obu_size. - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data() + 2, frames[0]->size() - 2), - ElementsAreArray("many bits")); -} - -TEST_F(PacketBufferTest, GetBitstreamInvalidAv1) { - // Two av1 payloads that can't be combined into proper frame. - const uint8_t data1[] = {0b01'01'0000, 0b0'0100'000, 'm', 'a', 'n', 'y', ' '}; - const uint8_t data2[] = {0b00'01'0000, 'b', 'i', 't', 's', 0}; - - auto packet1 = std::make_unique(); - packet1->video_header.codec = kVideoCodecAV1; - packet1->seq_num = 13; - packet1->video_header.is_first_packet_in_frame = true; - packet1->video_header.is_last_packet_in_frame = false; - packet1->video_payload = data1; - auto frames = packet_buffer_.InsertPacket(std::move(packet1)).frames; - EXPECT_THAT(frames, IsEmpty()); - - auto packet2 = std::make_unique(); - packet2->video_header.codec = kVideoCodecAV1; - packet2->seq_num = 14; - packet2->video_header.is_first_packet_in_frame = false; - packet2->video_header.is_last_packet_in_frame = true; - packet2->video_payload = data2; - frames = packet_buffer_.InsertPacket(std::move(packet2)).frames; - - EXPECT_THAT(frames, IsEmpty()); -} - TEST_F(PacketBufferTest, InsertPacketAfterSequenceNumberWrapAround) { uint16_t kFirstSeqNum = 0; uint32_t kTimestampDelta = 100; @@ -510,9 +377,11 @@ TEST_F(PacketBufferTest, InsertPacketAfterSequenceNumberWrapAround) { for (int i = 0; i < 5; ++i) { Insert(seq_num++, kKeyFrame, kNotFirst, kNotLast, {}, timestamp); } - EXPECT_THAT( - Insert(seq_num++, kKeyFrame, kNotFirst, kLast, {}, timestamp).frames, - SizeIs(1)); + auto packets = + Insert(seq_num++, kKeyFrame, kNotFirst, kLast, {}, timestamp).packets; + // One frame of 7 packets. + EXPECT_THAT(StartSeqNums(packets), SizeIs(1)); + EXPECT_THAT(packets, SizeIs(7)); } // If |sps_pps_idr_is_keyframe| is true, we require keyframes to contain @@ -614,7 +483,7 @@ TEST_P(PacketBufferH264ParameterizedTest, DontRemoveMissingPacketOnClearTo) { InsertH264(2, kDeltaFrame, kFirst, kNotLast, 2); packet_buffer_.ClearTo(0); // Expect no frame because of missing of packet #1 - EXPECT_THAT(InsertH264(3, kDeltaFrame, kNotFirst, kLast, 2).frames, + EXPECT_THAT(InsertH264(3, kDeltaFrame, kNotFirst, kLast, 2).packets, IsEmpty()); } @@ -632,17 +501,19 @@ TEST_P(PacketBufferH264ParameterizedTest, GetBitstreamOneFrameFullBuffer) { InsertH264(i, kKeyFrame, kNotFirst, kNotLast, 1, data_arr[i]); } - auto frames = InsertH264(kStartSize - 1, kKeyFrame, kNotFirst, kLast, 1, - data_arr[kStartSize - 1]) - .frames; - ASSERT_THAT(StartSeqNums(frames), ElementsAre(0)); - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()), - ElementsAreArray(expected)); + auto packets = InsertH264(kStartSize - 1, kKeyFrame, kNotFirst, kLast, 1, + data_arr[kStartSize - 1]) + .packets; + ASSERT_THAT(StartSeqNums(packets), ElementsAre(0)); + EXPECT_THAT(packets, SizeIs(kStartSize)); + for (size_t i = 0; i < packets.size(); ++i) { + EXPECT_THAT(packets[i]->video_payload, SizeIs(1)) << "Packet #" << i; + } } TEST_P(PacketBufferH264ParameterizedTest, GetBitstreamBufferPadding) { uint16_t seq_num = Rand(); - uint8_t data[] = "some plain old data"; + rtc::CopyOnWriteBuffer data = "some plain old data"; auto packet = std::make_unique(); auto& h264_header = @@ -655,14 +526,11 @@ TEST_P(PacketBufferH264ParameterizedTest, GetBitstreamBufferPadding) { packet->video_payload = data; packet->video_header.is_first_packet_in_frame = true; packet->video_header.is_last_packet_in_frame = true; - auto frames = packet_buffer_.InsertPacket(std::move(packet)).frames; + auto frames = packet_buffer_.InsertPacket(std::move(packet)).packets; ASSERT_THAT(frames, SizeIs(1)); - EXPECT_EQ(frames[0]->first_seq_num(), seq_num); - EXPECT_EQ(frames[0]->EncodedImage().size(), sizeof(data)); - EXPECT_EQ(frames[0]->EncodedImage().capacity(), sizeof(data)); - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()), - ElementsAreArray(data)); + EXPECT_EQ(frames[0]->seq_num, seq_num); + EXPECT_EQ(frames[0]->video_payload, data); } TEST_P(PacketBufferH264ParameterizedTest, FrameResolution) { @@ -672,15 +540,13 @@ TEST_P(PacketBufferH264ParameterizedTest, FrameResolution) { uint32_t height = 360; uint32_t timestamp = 1000; - auto frames = InsertH264(seq_num, kKeyFrame, kFirst, kLast, timestamp, data, - width, height) - .frames; + auto packets = InsertH264(seq_num, kKeyFrame, kFirst, kLast, timestamp, data, + width, height) + .packets; - ASSERT_THAT(frames, SizeIs(1)); - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()), - ElementsAreArray(data)); - EXPECT_EQ(frames[0]->EncodedImage()._encodedWidth, width); - EXPECT_EQ(frames[0]->EncodedImage()._encodedHeight, height); + ASSERT_THAT(packets, SizeIs(1)); + EXPECT_EQ(packets[0]->video_header.width, width); + EXPECT_EQ(packets[0]->video_header.height, height); } TEST_P(PacketBufferH264ParameterizedTest, FrameResolutionNaluBeforeSPS) { @@ -690,18 +556,13 @@ TEST_P(PacketBufferH264ParameterizedTest, FrameResolutionNaluBeforeSPS) { uint32_t height = 360; uint32_t timestamp = 1000; - auto frames = InsertH264KeyFrameWithAud(seq_num, kKeyFrame, kFirst, kLast, - timestamp, data, width, height) - .frames; + auto packets = InsertH264KeyFrameWithAud(seq_num, kKeyFrame, kFirst, kLast, + timestamp, data, width, height) + .packets; - ASSERT_THAT(StartSeqNums(frames), ElementsAre(seq_num)); - - EXPECT_EQ(frames[0]->EncodedImage().size(), sizeof(data)); - EXPECT_EQ(frames[0]->EncodedImage().capacity(), sizeof(data)); - EXPECT_EQ(frames[0]->EncodedImage()._encodedWidth, width); - EXPECT_EQ(frames[0]->EncodedImage()._encodedHeight, height); - EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()), - ElementsAreArray(data)); + ASSERT_THAT(StartSeqNums(packets), ElementsAre(seq_num)); + EXPECT_EQ(packets[0]->video_header.width, width); + EXPECT_EQ(packets[0]->video_header.height, height); } TEST_F(PacketBufferTest, FreeSlotsOnFrameCreation) { @@ -740,20 +601,20 @@ TEST_F(PacketBufferTest, FramesAfterClear) { Insert(9025, kDeltaFrame, kFirst, kLast); Insert(9024, kKeyFrame, kFirst, kLast); packet_buffer_.ClearTo(9025); - EXPECT_THAT(Insert(9057, kDeltaFrame, kFirst, kLast).frames, SizeIs(1)); - EXPECT_THAT(Insert(9026, kDeltaFrame, kFirst, kLast).frames, SizeIs(1)); + EXPECT_THAT(Insert(9057, kDeltaFrame, kFirst, kLast).packets, SizeIs(1)); + EXPECT_THAT(Insert(9026, kDeltaFrame, kFirst, kLast).packets, SizeIs(1)); } TEST_F(PacketBufferTest, SameFrameDifferentTimestamps) { Insert(0, kKeyFrame, kFirst, kNotLast, {}, 1000); - EXPECT_THAT(Insert(1, kKeyFrame, kNotFirst, kLast, {}, 1001).frames, + EXPECT_THAT(Insert(1, kKeyFrame, kNotFirst, kLast, {}, 1001).packets, IsEmpty()); } TEST_F(PacketBufferTest, ContinuousSeqNumDoubleMarkerBit) { Insert(2, kKeyFrame, kNotFirst, kNotLast); Insert(1, kKeyFrame, kFirst, kLast); - EXPECT_THAT(Insert(3, kKeyFrame, kNotFirst, kLast).frames, IsEmpty()); + EXPECT_THAT(Insert(3, kKeyFrame, kNotFirst, kLast).packets, IsEmpty()); } TEST_F(PacketBufferTest, PacketTimestamps) { @@ -826,7 +687,8 @@ TEST_F(PacketBufferTest, IncomingCodecChange) { packet->timestamp = 1; packet->seq_num = 1; packet->video_header.frame_type = VideoFrameType::kVideoFrameKey; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, SizeIs(1)); + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, + SizeIs(1)); packet = std::make_unique(); packet->video_header.is_first_packet_in_frame = true; @@ -838,7 +700,8 @@ TEST_F(PacketBufferTest, IncomingCodecChange) { packet->timestamp = 3; packet->seq_num = 3; packet->video_header.frame_type = VideoFrameType::kVideoFrameKey; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, IsEmpty()); + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, + IsEmpty()); packet = std::make_unique(); packet->video_header.is_first_packet_in_frame = true; @@ -848,7 +711,8 @@ TEST_F(PacketBufferTest, IncomingCodecChange) { packet->timestamp = 2; packet->seq_num = 2; packet->video_header.frame_type = VideoFrameType::kVideoFrameDelta; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, SizeIs(2)); + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, + SizeIs(2)); } TEST_F(PacketBufferTest, TooManyNalusInPacket) { @@ -862,7 +726,8 @@ TEST_F(PacketBufferTest, TooManyNalusInPacket) { auto& h264_header = packet->video_header.video_type_header.emplace(); h264_header.nalus_length = kMaxNalusPerPacket; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, IsEmpty()); + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, + IsEmpty()); } TEST_P(PacketBufferH264ParameterizedTest, OneFrameFillBuffer) { @@ -874,7 +739,7 @@ TEST_P(PacketBufferH264ParameterizedTest, OneFrameFillBuffer) { } TEST_P(PacketBufferH264ParameterizedTest, CreateFramesAfterFilledBuffer) { - EXPECT_THAT(InsertH264(kStartSize - 2, kKeyFrame, kFirst, kLast, 0).frames, + EXPECT_THAT(InsertH264(kStartSize - 2, kKeyFrame, kFirst, kLast, 0).packets, SizeIs(1)); InsertH264(kStartSize, kDeltaFrame, kFirst, kNotLast, 2000); @@ -882,7 +747,7 @@ TEST_P(PacketBufferH264ParameterizedTest, CreateFramesAfterFilledBuffer) { InsertH264(kStartSize + i, kDeltaFrame, kNotFirst, kNotLast, 2000); EXPECT_THAT( InsertH264(kStartSize + kStartSize, kDeltaFrame, kNotFirst, kLast, 2000) - .frames, + .packets, IsEmpty()); EXPECT_THAT(InsertH264(kStartSize - 1, kKeyFrame, kFirst, kLast, 1000), @@ -908,7 +773,7 @@ TEST_P(PacketBufferH264ParameterizedTest, ClearMissingPacketsOnKeyframe) { TEST_P(PacketBufferH264ParameterizedTest, FindFramesOnPadding) { EXPECT_THAT(InsertH264(0, kKeyFrame, kFirst, kLast, 1000), StartSeqNumsAre(0)); - EXPECT_THAT(InsertH264(2, kDeltaFrame, kFirst, kLast, 1000).frames, + EXPECT_THAT(InsertH264(2, kDeltaFrame, kFirst, kLast, 1000).packets, IsEmpty()); EXPECT_THAT(packet_buffer_.InsertPadding(1), StartSeqNumsAre(2)); @@ -945,7 +810,7 @@ TEST_F(PacketBufferH264IdrIsKeyframeTest, IdrIsKeyframe) { packet->video_header.video_type_header.emplace(); h264_header.nalus[0].type = H264::NaluType::kIdr; h264_header.nalus_length = 1; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, ElementsAre(KeyFrame())); } @@ -958,7 +823,7 @@ TEST_F(PacketBufferH264IdrIsKeyframeTest, SpsPpsIdrIsKeyframe) { h264_header.nalus[2].type = H264::NaluType::kIdr; h264_header.nalus_length = 3; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, ElementsAre(KeyFrame())); } @@ -976,7 +841,7 @@ TEST_F(PacketBufferH264SpsPpsIdrIsKeyframeTest, IdrIsNotKeyframe) { h264_header.nalus[0].type = H264::NaluType::kIdr; h264_header.nalus_length = 1; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, ElementsAre(DeltaFrame())); } @@ -988,7 +853,7 @@ TEST_F(PacketBufferH264SpsPpsIdrIsKeyframeTest, SpsPpsIsNotKeyframe) { h264_header.nalus[1].type = H264::NaluType::kPps; h264_header.nalus_length = 2; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, ElementsAre(DeltaFrame())); } @@ -1001,7 +866,7 @@ TEST_F(PacketBufferH264SpsPpsIdrIsKeyframeTest, SpsPpsIdrIsKeyframe) { h264_header.nalus[2].type = H264::NaluType::kIdr; h264_header.nalus_length = 3; - EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, + EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets, ElementsAre(KeyFrame())); } diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 576da68f9e..df060cfd4e 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -36,6 +36,7 @@ #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer_av1.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h" #include "modules/utility/include/process_thread.h" #include "modules/video_coding/frame_object.h" @@ -707,9 +708,80 @@ bool RtpVideoStreamReceiver::IsDecryptable() const { void RtpVideoStreamReceiver::OnInsertedPacket( video_coding::PacketBuffer::InsertResult result) { - for (std::unique_ptr& frame : result.frames) { - OnAssembledFrame(std::move(frame)); + video_coding::PacketBuffer::Packet* first_packet = nullptr; + int max_nack_count; + int64_t min_recv_time; + int64_t max_recv_time; + int frame_size; + std::vector> payloads; + RtpPacketInfos::vector_type packet_infos; + + bool frame_boundary = true; + for (auto& packet : result.packets) { + // PacketBuffer promisses frame boundaries are correctly set on each + // packet. Document that assumption with the DCHECKs. + RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame()); + if (packet->is_first_packet_in_frame()) { + first_packet = packet.get(); + max_nack_count = packet->times_nacked; + min_recv_time = packet->packet_info.receive_time_ms(); + max_recv_time = packet->packet_info.receive_time_ms(); + frame_size = packet->video_payload.size(); + payloads.clear(); + packet_infos.clear(); + } else { + max_nack_count = std::max(max_nack_count, packet->times_nacked); + min_recv_time = + std::min(min_recv_time, packet->packet_info.receive_time_ms()); + max_recv_time = + std::max(max_recv_time, packet->packet_info.receive_time_ms()); + frame_size += packet->video_payload.size(); + } + payloads.emplace_back(packet->video_payload); + packet_infos.push_back(packet->packet_info); + + frame_boundary = packet->is_last_packet_in_frame(); + if (packet->is_last_packet_in_frame()) { + rtc::scoped_refptr bitstream; + // TODO(danilchap): Hide codec-specific code paths behind an interface. + if (first_packet->codec() == VideoCodecType::kVideoCodecAV1) { + bitstream = VideoRtpDepacketizerAv1::AssembleFrame(payloads); + if (!bitstream) { + // Failed to assemble a frame. Discard and continue. + continue; + } + } else { + bitstream = EncodedImageBuffer::Create(frame_size); + + uint8_t* write_at = bitstream->data(); + for (rtc::ArrayView payload : payloads) { + memcpy(write_at, payload.data(), payload.size()); + write_at += payload.size(); + } + RTC_DCHECK_EQ(write_at - bitstream->data(), bitstream->size()); + } + const video_coding::PacketBuffer::Packet& last_packet = *packet; + OnAssembledFrame(std::make_unique( + first_packet->seq_num, // + last_packet.seq_num, // + last_packet.marker_bit, // + max_nack_count, // + min_recv_time, // + max_recv_time, // + first_packet->timestamp, // + first_packet->ntp_time_ms, // + last_packet.video_header.video_timing, // + first_packet->payload_type, // + first_packet->codec(), // + last_packet.video_header.rotation, // + last_packet.video_header.content_type, // + first_packet->video_header, // + last_packet.video_header.color_space, // + RtpPacketInfos(std::move(packet_infos)), // + std::move(bitstream))); + } } + RTC_DCHECK(frame_boundary); if (result.buffer_cleared) { RequestKeyFrame(); }