diff --git a/modules/rtp_rtcp/source/rtp_format.cc b/modules/rtp_rtcp/source/rtp_format.cc index 47838cb385..2448f82bd9 100644 --- a/modules/rtp_rtcp/source/rtp_format.cc +++ b/modules/rtp_rtcp/source/rtp_format.cc @@ -13,10 +13,12 @@ #include #include "absl/types/variant.h" +#include "modules/rtp_rtcp/source/rtp_depacketizer_av1.h" #include "modules/rtp_rtcp/source/rtp_format_h264.h" #include "modules/rtp_rtcp/source/rtp_format_video_generic.h" #include "modules/rtp_rtcp/source/rtp_format_vp8.h" #include "modules/rtp_rtcp/source/rtp_format_vp9.h" +#include "modules/rtp_rtcp/source/rtp_packetizer_av1.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" @@ -54,6 +56,9 @@ std::unique_ptr RtpPacketizer::Create( absl::get(rtp_video_header.video_type_header); return std::make_unique(payload, limits, vp9); } + case kVideoCodecAV1: + return std::make_unique(payload, limits, + rtp_video_header.frame_type); default: { return std::make_unique(payload, limits, rtp_video_header); @@ -151,6 +156,8 @@ RtpDepacketizer* RtpDepacketizer::Create(absl::optional type) { return new RtpDepacketizerVp8(); case kVideoCodecVP9: return new RtpDepacketizerVp9(); + case kVideoCodecAV1: + return new RtpDepacketizerAv1(); default: return new RtpDepacketizerGeneric(/*generic_header_enabled=*/true); } diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 9592b06f16..5f7735e597 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -366,10 +366,12 @@ void RTPSenderVideo::LogAndSendToNetwork( continue; } } - RTC_DCHECK_GE(packetized_payload_size, unpacketized_payload_size); - packetization_overhead_bitrate_.Update( - packetized_payload_size - unpacketized_payload_size, - clock_->TimeInMilliseconds()); + // AV1 packetizer may produce less packetized bytes than unpacketized. + if (packetized_payload_size >= unpacketized_payload_size) { + packetization_overhead_bitrate_.Update( + packetized_payload_size - unpacketized_payload_size, + clock_->TimeInMilliseconds()); + } } rtp_sender_->EnqueuePackets(std::move(packets)); diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 71e14fa780..ceee019e06 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -136,6 +136,7 @@ rtc_library("video_coding") { "../../api/video:video_bitrate_allocator", "../../api/video:video_frame", "../../api/video:video_frame_i420", + "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", "../../common_video", @@ -156,6 +157,7 @@ rtc_library("video_coding") { "../../rtc_base/third_party/base64", "../../rtc_base/time:timestamp_extrapolator", "../../system_wrappers", + "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/types:optional", diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index 3e8b11f4fd..30dfc21e41 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -22,7 +22,9 @@ #include "api/array_view.h" #include "api/rtp_packet_info.h" #include "api/video/encoded_frame.h" +#include "api/video/video_frame_type.h" #include "common_video/h264/h264_common.h" +#include "modules/rtp_rtcp/source/rtp_depacketizer_av1.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" @@ -398,7 +400,12 @@ std::vector> PacketBuffer::FindFrames( } } - found_frames.push_back(AssembleFrame(start_seq_num, seq_num)); + if (auto frame = AssembleFrame(start_seq_num, seq_num)) { + found_frames.push_back(std::move(frame)); + } else { + RTC_LOG(LS_ERROR) << "Failed to assemble frame from packets " + << start_seq_num << "-" << seq_num; + } missing_packets_.erase(missing_packets_.begin(), missing_packets_.upper_bound(seq_num)); @@ -437,16 +444,25 @@ std::unique_ptr PacketBuffer::AssembleFrame( packet_infos.push_back(packet.packet_info); } - auto bitstream = EncodedImageBuffer::Create(frame_size); - - uint8_t* write_at = bitstream->data(); - for (rtc::ArrayView payload : payloads) { - memcpy(write_at, payload.data(), payload.size()); - write_at += payload.size(); - } - RTC_DCHECK_EQ(write_at - bitstream->data(), bitstream->size()); - const Packet& first_packet = GetPacket(first_seq_num); + rtc::scoped_refptr bitstream; + // TODO(danilchap): Hide codec-specific code paths behind an interface. + if (first_packet.codec() == VideoCodecType::kVideoCodecAV1) { + bitstream = RtpDepacketizerAv1::AssembleFrame(payloads); + if (!bitstream) { + // Failed to assemble a frame. Discard and continue. + return nullptr; + } + } else { + bitstream = EncodedImageBuffer::Create(frame_size); + + uint8_t* write_at = bitstream->data(); + for (rtc::ArrayView payload : payloads) { + memcpy(write_at, payload.data(), payload.size()); + write_at += payload.size(); + } + RTC_DCHECK_EQ(write_at - bitstream->data(), bitstream->size()); + } const Packet& last_packet = GetPacket(last_seq_num); return std::make_unique( first_seq_num, // diff --git a/modules/video_coding/packet_buffer_unittest.cc b/modules/video_coding/packet_buffer_unittest.cc index 3385f78f1f..b1468537b3 100644 --- a/modules/video_coding/packet_buffer_unittest.cc +++ b/modules/video_coding/packet_buffer_unittest.cc @@ -444,6 +444,74 @@ TEST_F(PacketBufferTest, GetBitstreamOneFrameFullBuffer) { ElementsAreArray(expected)); } +TEST_F(PacketBufferTest, GetBitstreamAv1) { + const uint8_t data1[] = {0b01'01'0000, 0b0'0100'000, 'm', 'a', 'n', 'y', ' '}; + const uint8_t data2[] = {0b10'01'0000, 'b', 'i', 't', 's', 0}; + + uint8_t* new_data1 = new uint8_t[sizeof(data1)]; + memcpy(new_data1, data1, sizeof(data1)); + uint8_t* new_data2 = new uint8_t[sizeof(data2)]; + memcpy(new_data2, data2, sizeof(data2)); + + PacketBuffer::Packet packet1; + packet1.video_header.codec = kVideoCodecAV1; + packet1.seq_num = 13; + packet1.video_header.is_first_packet_in_frame = true; + packet1.video_header.is_last_packet_in_frame = false; + packet1.size_bytes = sizeof(data1); + packet1.data = new_data1; + auto frames = packet_buffer_.InsertPacket(&packet1).frames; + EXPECT_THAT(frames, IsEmpty()); + + PacketBuffer::Packet packet2; + packet2.video_header.codec = kVideoCodecAV1; + packet2.seq_num = 14; + packet2.video_header.is_first_packet_in_frame = false; + packet2.video_header.is_last_packet_in_frame = true; + packet2.size_bytes = sizeof(data2); + packet2.data = new_data2; + frames = packet_buffer_.InsertPacket(&packet2).frames; + + ASSERT_THAT(frames, SizeIs(1)); + EXPECT_EQ(frames[0]->first_seq_num(), 13); + EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), 2), + ElementsAre(0b0'0100'010, 10)); // obu_header and obu_size. + EXPECT_THAT(rtc::MakeArrayView(frames[0]->data() + 2, frames[0]->size() - 2), + ElementsAreArray("many bits")); +} + +TEST_F(PacketBufferTest, GetBitstreamInvalidAv1) { + // Two av1 payloads that can't be combined into proper frame. + const uint8_t data1[] = {0b01'01'0000, 0b0'0100'000, 'm', 'a', 'n', 'y', ' '}; + const uint8_t data2[] = {0b00'01'0000, 'b', 'i', 't', 's', 0}; + + uint8_t* new_data1 = new uint8_t[sizeof(data1)]; + memcpy(new_data1, data1, sizeof(data1)); + uint8_t* new_data2 = new uint8_t[sizeof(data2)]; + memcpy(new_data2, data2, sizeof(data2)); + + PacketBuffer::Packet packet1; + packet1.video_header.codec = kVideoCodecAV1; + packet1.seq_num = 13; + packet1.video_header.is_first_packet_in_frame = true; + packet1.video_header.is_last_packet_in_frame = false; + packet1.size_bytes = sizeof(data1); + packet1.data = new_data1; + auto frames = packet_buffer_.InsertPacket(&packet1).frames; + EXPECT_THAT(frames, IsEmpty()); + + PacketBuffer::Packet packet2; + packet2.video_header.codec = kVideoCodecAV1; + packet2.seq_num = 14; + packet2.video_header.is_first_packet_in_frame = false; + packet2.video_header.is_last_packet_in_frame = true; + packet2.size_bytes = sizeof(data2); + packet2.data = new_data2; + frames = packet_buffer_.InsertPacket(&packet2).frames; + + EXPECT_THAT(frames, IsEmpty()); +} + TEST_F(PacketBufferTest, InsertPacketAfterSequenceNumberWrapAround) { uint16_t kFirstSeqNum = 0; uint32_t kTimestampDelta = 100;