diff --git a/modules/rtp_rtcp/source/rtp_packet_h265_common.h b/modules/rtp_rtcp/source/rtp_packet_h265_common.h index 8655a02001..6f77ebf9e3 100644 --- a/modules/rtp_rtcp/source/rtp_packet_h265_common.h +++ b/modules/rtp_rtcp/source/rtp_packet_h265_common.h @@ -18,6 +18,8 @@ namespace webrtc { // fields (F, Type, LayerId and TID) as the NAL unit header. Refer to // section 4.4 in RFC 7798. constexpr size_t kH265PayloadHeaderSizeBytes = 2; +constexpr uint8_t kH265MaxLayerId = 127; +constexpr uint8_t kH265MaxTemporalId = 7; // Unlike H.264, H.265 NAL header is 2-bytes. constexpr size_t kH265NalHeaderSizeBytes = 2; // H.265's FU is constructed of 2-byte payload header, 1-byte FU header and FU diff --git a/modules/rtp_rtcp/source/rtp_packetizer_h265.cc b/modules/rtp_rtcp/source/rtp_packetizer_h265.cc index efc0d8aeae..599de8d485 100644 --- a/modules/rtp_rtcp/source/rtp_packetizer_h265.cc +++ b/modules/rtp_rtcp/source/rtp_packetizer_h265.cc @@ -10,6 +10,7 @@ #include "modules/rtp_rtcp/source/rtp_packetizer_h265.h" +#include #include #include @@ -160,13 +161,14 @@ int RtpPacketizerH265::PacketizeAp(size_t fragment_index) { return fragment_size; }; + uint16_t header = (fragment[0] << 8) | fragment[1]; while (payload_size_left >= payload_size_needed()) { RTC_CHECK_GT(fragment.size(), 0); packets_.push({.source_fragment = fragment, .first_fragment = (aggregated_fragments == 0), .last_fragment = false, .aggregated = true, - .header = fragment[0]}); + .header = header}); payload_size_left -= fragment.size(); payload_size_left -= fragment_headers_length; @@ -234,22 +236,25 @@ void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) { |F| Type | LayerId | TID | +-------------+-----------------+ */ - // Refer to section section 4.4.2 for aggregation packets and modify type to + // Refer to section 4.4.2 for aggregation packets and modify type to // 48 in PayloadHdr for aggregate packet. Do not support DONL for aggregation // packets, DONL field is not present. - uint8_t payload_hdr_h = packet->header >> 8; - uint8_t payload_hdr_l = packet->header & 0xFF; - uint8_t layer_id_h = payload_hdr_h & kH265LayerIDHMask; - payload_hdr_h = (payload_hdr_h & kH265TypeMaskN) | - (H265::NaluType::kAp << 1) | layer_id_h; - buffer[0] = payload_hdr_h; - buffer[1] = payload_hdr_l; - int index = kH265PayloadHeaderSizeBytes; bool is_last_fragment = packet->last_fragment; + + // Refer to section 4.4.2 for aggregation packets and calculate the lowest + // value of LayerId and TID of all the aggregated NAL units + uint8_t layer_id_min = kH265MaxLayerId; + uint8_t temporal_id_min = kH265MaxTemporalId; while (packet->aggregated) { // Add NAL unit length field. rtc::ArrayView fragment = packet->source_fragment; + uint8_t layer_id = ((fragment[0] & kH265LayerIDHMask) << 5) | + ((fragment[1] & kH265LayerIDLMask) >> 3); + layer_id_min = std::min(layer_id_min, layer_id); + uint8_t temporal_id = fragment[1] & kH265TIDMask; + temporal_id_min = std::min(temporal_id_min, temporal_id); + ByteWriter::WriteBigEndian(&buffer[index], fragment.size()); index += kH265LengthFieldSizeBytes; // Add NAL unit. @@ -263,6 +268,9 @@ void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) { packet = &packets_.front(); is_last_fragment = packet->last_fragment; } + + buffer[0] = (H265::NaluType::kAp << 1) | (layer_id_min >> 5); + buffer[1] = (layer_id_min << 3) | temporal_id_min; RTC_CHECK(is_last_fragment); rtp_packet->SetPayloadSize(index); } diff --git a/modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc b/modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc index 8f739e8618..5355c5dd09 100644 --- a/modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc @@ -37,13 +37,19 @@ constexpr RtpPacketizer::PayloadSizeLimits kNoLimits; constexpr size_t kFuHeaderSizeBytes = kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes; -// Creates Buffer that looks like nal unit of given size. -rtc::Buffer GenerateNalUnit(size_t size) { +struct NalUnitHeader { + uint8_t forbidden_zero_bit = 0; + uint8_t nal_unit_type = 0; + uint8_t nuh_layer_id = 0; + uint8_t nuh_temporal_id_plus1 = 0; +}; + +// Creates Buffer that looks like nal unit of given header and size. +rtc::Buffer GenerateNalUnit(NalUnitHeader header, size_t size) { RTC_CHECK_GT(size, 0); rtc::Buffer buffer(size); - // Set some valid header with type TRAIL_R and temporal id - buffer[0] = 2; - buffer[1] = 2; + buffer[0] = (header.nal_unit_type << 1) | (header.nuh_layer_id >> 5); + buffer[1] = (header.nuh_layer_id << 3) | header.nuh_temporal_id_plus1; for (size_t i = 2; i < size; ++i) { buffer[i] = static_cast(i); } @@ -123,8 +129,15 @@ TEST(RtpPacketizerH265Test, SingleNalu) { TEST(RtpPacketizerH265Test, SingleNaluTwoPackets) { RtpPacketizer::PayloadSizeLimits limits; limits.max_payload_len = kMaxPayloadSizeBytes; - rtc::Buffer nalus[] = {GenerateNalUnit(kMaxPayloadSizeBytes), - GenerateNalUnit(100)}; + rtc::Buffer nalus[] = { + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + kMaxPayloadSizeBytes), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + 100)}; rtc::Buffer frame = CreateFrame(nalus); RtpPacketizerH265 packetizer(frame, limits); @@ -140,9 +153,19 @@ TEST(RtpPacketizerH265Test, RtpPacketizer::PayloadSizeLimits limits; limits.max_payload_len = 200; limits.first_packet_reduction_len = 5; - rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/195), - GenerateNalUnit(/*size=*/200), - GenerateNalUnit(/*size=*/200)}; + rtc::Buffer nalus[] = { + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/195), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/200), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/200)}; rtc::Buffer frame = CreateFrame(nalus); RtpPacketizerH265 packetizer(frame, limits); @@ -159,9 +182,19 @@ TEST(RtpPacketizerH265Test, RtpPacketizer::PayloadSizeLimits limits; limits.max_payload_len = 200; limits.last_packet_reduction_len = 5; - rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/200), - GenerateNalUnit(/*size=*/200), - GenerateNalUnit(/*size=*/195)}; + rtc::Buffer nalus[] = { + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/200), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/200), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/195)}; rtc::Buffer frame = CreateFrame(nalus); RtpPacketizerH265 packetizer(frame, limits); @@ -189,9 +222,19 @@ TEST(RtpPacketizerH265Test, // Aggregation tests. TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) { - rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2), - GenerateNalUnit(/*size=*/2), - GenerateNalUnit(/*size=*/0x123)}; + rtc::Buffer nalus[] = { + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/0x123)}; rtc::Buffer frame = CreateFrame(nalus); RtpPacketizerH265 packetizer(frame, kNoLimits); @@ -201,22 +244,22 @@ TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) { auto payload = packets[0].payload(); int type = H265::ParseNaluType(payload[0]); EXPECT_EQ(payload.size(), kH265NalHeaderSizeBytes + - 3 * kH265LengthFieldSizeBytes + 2 + 2 + 0x123); + 3 * kH265LengthFieldSizeBytes + 3 + 3 + 0x123); EXPECT_EQ(type, H265::NaluType::kAp); payload = payload.subview(kH265NalHeaderSizeBytes); // 1st fragment. EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes), - ElementsAre(0, 2)); // Size. - EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 2), + ElementsAre(0, 3)); // Size. + EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 3), ElementsAreArray(nalus[0])); - payload = payload.subview(kH265LengthFieldSizeBytes + 2); + payload = payload.subview(kH265LengthFieldSizeBytes + 3); // 2nd fragment. EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes), - ElementsAre(0, 2)); // Size. - EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 2), + ElementsAre(0, 3)); // Size. + EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 3), ElementsAreArray(nalus[1])); - payload = payload.subview(kH265LengthFieldSizeBytes + 2); + payload = payload.subview(kH265LengthFieldSizeBytes + 3); // 3rd fragment. EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes), ElementsAre(0x1, 0x23)); // Size. @@ -224,15 +267,80 @@ TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) { ElementsAreArray(nalus[2])); } +TEST(RtpPacketizerH265Test, ApRespectsLayerIdAndTemporalId) { + // Generate 3 NALUs: NALU 1 with nuh_layer_id 2 and nuh_temporal_id_plus1 6, + // NALU 2 with nuh_layer_id 0 and nuh_temporal_id_plus1 1, + // NALU 3 with nuh_layer_id 32 and nuh_temporal_id_plus1 2, + // So in the AP packet header, nuh_layer_id should be 0 which is the lowest + // nuh_layer_id value of 3 NALUs and nuh_temporal_id_plus1 should be 1 which + // is the lowest nuh_temporal_id_plus1 value of 3 NALUs + rtc::Buffer nalus[] = { + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 2, + .nuh_temporal_id_plus1 = 6}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 0, + .nuh_temporal_id_plus1 = 1}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/0x123)}; + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, kNoLimits); + std::vector packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(1)); + auto payload = packets[0].payload(); + uint8_t type = H265::ParseNaluType(payload[0]); + uint8_t layer_id = ((payload[0] & kH265LayerIDHMask) << 5) | + ((payload[1] & kH265LayerIDLMask) >> 3); + uint8_t temporal_id = payload[1] & kH265TIDMask; + EXPECT_EQ(payload.size(), kH265NalHeaderSizeBytes + + 3 * kH265LengthFieldSizeBytes + 3 + 3 + 0x123); + + EXPECT_EQ(type, H265::NaluType::kAp); + EXPECT_EQ(layer_id, 0); + EXPECT_EQ(temporal_id, 1); + payload = payload.subview(kH265NalHeaderSizeBytes); + // 1st fragment. + EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes), ElementsAre(0, 3)); + EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 3), + ElementsAreArray(nalus[0])); + payload = payload.subview(kH265LengthFieldSizeBytes + 3); + // 2nd fragment. + EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes), ElementsAre(0, 3)); + EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 3), + ElementsAreArray(nalus[1])); + payload = payload.subview(kH265LengthFieldSizeBytes + 3); + // 3rd fragment. + EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes), + ElementsAre(0x1, 0x23)); + EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes), + ElementsAreArray(nalus[2])); +} + TEST(RtpPacketizerH265Test, ApRespectsFirstPacketReduction) { RtpPacketizer::PayloadSizeLimits limits; limits.max_payload_len = 1000; limits.first_packet_reduction_len = 100; const size_t kFirstFragmentSize = limits.max_payload_len - limits.first_packet_reduction_len; - rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/kFirstFragmentSize), - GenerateNalUnit(/*size=*/2), - GenerateNalUnit(/*size=*/2)}; + rtc::Buffer nalus[] = { + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/kFirstFragmentSize), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/3)}; rtc::Buffer frame = CreateFrame(nalus); RtpPacketizerH265 packetizer(frame, limits); @@ -243,13 +351,13 @@ TEST(RtpPacketizerH265Test, ApRespectsFirstPacketReduction) { EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0])); // Expect 2nd packet is aggregate of last two fragments. // The size of H265 nal_unit_header is 2 bytes, according to 7.3.1.2 - // in H265 spec. Aggregation packet type is 48, and nuh_temporal_id_plus1 - // is 2, so the nal_unit_header should be "01100000 00000010", - // which is 96 and 2. + // in H265 spec. Aggregation packet type is 48, nuh_layer_id is 32 and + // nuh_temporal_id_plus1 is 2, so the nal_unit_header should be "01100001 + // 00000010", which is 97 and 2. EXPECT_THAT(packets[1].payload(), - ElementsAre(96, 2, // - 0, 2, nalus[1][0], nalus[1][1], // - 0, 2, nalus[2][0], nalus[2][1])); + ElementsAre(97, 2, // + 0, 3, nalus[1][0], nalus[1][1], nalus[1][2], // + 0, 3, nalus[2][0], nalus[2][1], nalus[2][2])); } TEST(RtpPacketizerH265Test, ApRespectsLastPacketReduction) { @@ -258,9 +366,19 @@ TEST(RtpPacketizerH265Test, ApRespectsLastPacketReduction) { limits.last_packet_reduction_len = 100; const size_t kLastFragmentSize = limits.max_payload_len - limits.last_packet_reduction_len; - rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2), - GenerateNalUnit(/*size=*/2), - GenerateNalUnit(/*size=*/kLastFragmentSize)}; + rtc::Buffer nalus[] = { + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/kLastFragmentSize)}; rtc::Buffer frame = CreateFrame(nalus); RtpPacketizerH265 packetizer(frame, limits); @@ -269,9 +387,9 @@ TEST(RtpPacketizerH265Test, ApRespectsLastPacketReduction) { ASSERT_THAT(packets, SizeIs(2)); // Expect 1st packet is aggregate of 1st two fragments. EXPECT_THAT(packets[0].payload(), - ElementsAre(96, 2, // - 0, 2, nalus[0][0], nalus[0][1], // - 0, 2, nalus[1][0], nalus[1][1])); + ElementsAre(97, 2, // + 0, 3, nalus[0][0], nalus[0][1], nalus[0][2], // + 0, 3, nalus[1][0], nalus[1][1], nalus[1][2])); // Expect 2nd packet is single nalu. EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[2])); } @@ -281,9 +399,19 @@ TEST(RtpPacketizerH265Test, TooSmallForApHeaders) { limits.max_payload_len = 1000; const size_t kLastFragmentSize = limits.max_payload_len - 3 * kH265LengthFieldSizeBytes - 4; - rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2), - GenerateNalUnit(/*size=*/2), - GenerateNalUnit(/*size=*/kLastFragmentSize)}; + rtc::Buffer nalus[] = { + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/3), + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + /*size=*/kLastFragmentSize)}; rtc::Buffer frame = CreateFrame(nalus); RtpPacketizerH265 packetizer(frame, limits); @@ -292,9 +420,9 @@ TEST(RtpPacketizerH265Test, TooSmallForApHeaders) { ASSERT_THAT(packets, SizeIs(2)); // Expect 1st packet is aggregate of 1st two fragments. EXPECT_THAT(packets[0].payload(), - ElementsAre(96, 2, // - 0, 2, nalus[0][0], nalus[0][1], // - 0, 2, nalus[1][0], nalus[1][1])); + ElementsAre(97, 2, // + 0, 3, nalus[0][0], nalus[0][1], nalus[0][2], // + 0, 3, nalus[1][0], nalus[1][1], nalus[1][2])); // Expect 2nd packet is single nalu. EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[2])); } @@ -323,7 +451,10 @@ TEST(RtpPacketizerH265Test, LastFragmentFitsInSingleButNotLastPacket) { std::vector TestFu(size_t frame_payload_size, const RtpPacketizer::PayloadSizeLimits& limits) { rtc::Buffer nalu[] = { - GenerateNalUnit(kH265NalHeaderSizeBytes + frame_payload_size)}; + GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + kH265NalHeaderSizeBytes + frame_payload_size)}; rtc::Buffer frame = CreateFrame(nalu); RtpPacketizerH265 packetizer(frame, limits); @@ -433,7 +564,10 @@ TEST_P(RtpPacketizerH265ParametrizedTest, MixedApFu) { // Generate nalus according to size specified in paramters for (size_t index = 0; index < params.nalus.size(); index++) { - nalus.push_back(GenerateNalUnit(params.nalus[index])); + nalus.push_back(GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp, + .nuh_layer_id = 32, + .nuh_temporal_id_plus1 = 2}, + params.nalus[index])); } rtc::Buffer frame = CreateFrame(nalus); @@ -462,9 +596,9 @@ TEST_P(RtpPacketizerH265ParametrizedTest, MixedApFu) { uint8_t fu_header = 0; fu_header |= (expected_packet.first_fragment ? kH265SBitMask : 0); fu_header |= (expected_packet.last_fragment ? kH265EBitMask : 0); - fu_header |= H265::NaluType::kTrailR; + fu_header |= H265::NaluType::kIdrNLp; EXPECT_THAT(packets[i].payload().subview(0, kFuHeaderSizeBytes), - ElementsAre(98, 2, fu_header)); + ElementsAre(99, 2, fu_header)); EXPECT_THAT(packets[i].payload().subview(kFuHeaderSizeBytes), ElementsAreArray(nalus[expected_packet.nalu_index].data() + kH265NalHeaderSizeBytes +