admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix header length and set layer_id/temporal_id with lowest value of aggregated NALU for AP packet in H265 RTP packetizer

Browse Source 
Bug: webrtc:41480904
Change-Id: I56047b20933ba1f251ef88dc73a40c4967e8f89e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/362560
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Qiujiao Wu <qiujiao.wu@intel.com>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43220}
This commit is contained in:
qwu16 2024-09-23 16:29:52 +08:00 committed by WebRTC LUCI CQ
parent f1849d9034
commit 60c3fea7eb
3 changed files with 202 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/rtp_rtcp/source/rtp_packet_h265_common.h
View File

@ -18,6 +18,8 @@ namespace webrtc {
// fields (F, Type, LayerId and TID) as the NAL unit header. Refer to
// section 4.4 in RFC 7798.
constexpr size_t kH265PayloadHeaderSizeBytes = 2;
constexpr uint8_t kH265MaxLayerId = 127;
constexpr uint8_t kH265MaxTemporalId = 7;
// Unlike H.264, H.265 NAL header is 2-bytes.
constexpr size_t kH265NalHeaderSizeBytes = 2;
// H.265's FU is constructed of 2-byte payload header, 1-byte FU header and FU

28
modules/rtp_rtcp/source/rtp_packetizer_h265.cc
View File

@ -10,6 +10,7 @@
#include "modules/rtp_rtcp/source/rtp_packetizer_h265.h"
#include <algorithm>
#include <optional>
#include <vector>
@ -160,13 +161,14 @@ int RtpPacketizerH265::PacketizeAp(size_t fragment_index) {
return fragment_size;
};
uint16_t header = (fragment[0] << 8) | fragment[1];
while (payload_size_left >= payload_size_needed()) {
RTC_CHECK_GT(fragment.size(), 0);
packets_.push({.source_fragment = fragment,
.first_fragment = (aggregated_fragments == 0),
.last_fragment = false,
.aggregated = true,
.header = fragment[0]});
.header = header});
payload_size_left -= fragment.size();
payload_size_left -= fragment_headers_length;
@ -234,22 +236,25 @@ void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) {
|F| Type | LayerId | TID |
+-------------+-----------------+
*/
// Refer to section section 4.4.2 for aggregation packets and modify type to
// Refer to section 4.4.2 for aggregation packets and modify type to
// 48 in PayloadHdr for aggregate packet. Do not support DONL for aggregation
// packets, DONL field is not present.
uint8_t payload_hdr_h = packet->header >> 8;
uint8_t payload_hdr_l = packet->header & 0xFF;
uint8_t layer_id_h = payload_hdr_h & kH265LayerIDHMask;
payload_hdr_h = (payload_hdr_h & kH265TypeMaskN) |
(H265::NaluType::kAp << 1) | layer_id_h;
buffer[0] = payload_hdr_h;
buffer[1] = payload_hdr_l;
int index = kH265PayloadHeaderSizeBytes;
bool is_last_fragment = packet->last_fragment;
// Refer to section 4.4.2 for aggregation packets and calculate the lowest
// value of LayerId and TID of all the aggregated NAL units
uint8_t layer_id_min = kH265MaxLayerId;
uint8_t temporal_id_min = kH265MaxTemporalId;
while (packet->aggregated) {
// Add NAL unit length field.
rtc::ArrayView<const uint8_t> fragment = packet->source_fragment;
uint8_t layer_id = ((fragment[0] & kH265LayerIDHMask) << 5) |
((fragment[1] & kH265LayerIDLMask) >> 3);
layer_id_min = std::min(layer_id_min, layer_id);
uint8_t temporal_id = fragment[1] & kH265TIDMask;
temporal_id_min = std::min(temporal_id_min, temporal_id);
ByteWriter<uint16_t>::WriteBigEndian(&buffer[index], fragment.size());
index += kH265LengthFieldSizeBytes;
// Add NAL unit.
@ -263,6 +268,9 @@ void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) {
packet = &packets_.front();
is_last_fragment = packet->last_fragment;
}
buffer[0] = (H265::NaluType::kAp << 1) | (layer_id_min >> 5);
buffer[1] = (layer_id_min << 3) | temporal_id_min;
RTC_CHECK(is_last_fragment);
rtp_packet->SetPayloadSize(index);
}

230
modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc
View File

@ -37,13 +37,19 @@ constexpr RtpPacketizer::PayloadSizeLimits kNoLimits;
constexpr size_t kFuHeaderSizeBytes =
kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes;
// Creates Buffer that looks like nal unit of given size.
rtc::Buffer GenerateNalUnit(size_t size) {
struct NalUnitHeader {
uint8_t forbidden_zero_bit = 0;
uint8_t nal_unit_type = 0;
uint8_t nuh_layer_id = 0;
uint8_t nuh_temporal_id_plus1 = 0;
};
// Creates Buffer that looks like nal unit of given header and size.
rtc::Buffer GenerateNalUnit(NalUnitHeader header, size_t size) {
RTC_CHECK_GT(size, 0);
rtc::Buffer buffer(size);
// Set some valid header with type TRAIL_R and temporal id
buffer[0] = 2;
buffer[1] = 2;
buffer[0] = (header.nal_unit_type << 1) | (header.nuh_layer_id >> 5);
buffer[1] = (header.nuh_layer_id << 3) | header.nuh_temporal_id_plus1;
for (size_t i = 2; i < size; ++i) {
buffer[i] = static_cast<uint8_t>(i);
}
@ -123,8 +129,15 @@ TEST(RtpPacketizerH265Test, SingleNalu) {
TEST(RtpPacketizerH265Test, SingleNaluTwoPackets) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = kMaxPayloadSizeBytes;
rtc::Buffer nalus[] = {GenerateNalUnit(kMaxPayloadSizeBytes),
GenerateNalUnit(100)};
rtc::Buffer nalus[] = {
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
kMaxPayloadSizeBytes),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
100)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
@ -140,9 +153,19 @@ TEST(RtpPacketizerH265Test,
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 200;
limits.first_packet_reduction_len = 5;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/195),
GenerateNalUnit(/*size=*/200),
GenerateNalUnit(/*size=*/200)};
rtc::Buffer nalus[] = {
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/195),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/200),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/200)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
@ -159,9 +182,19 @@ TEST(RtpPacketizerH265Test,
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 200;
limits.last_packet_reduction_len = 5;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/200),
GenerateNalUnit(/*size=*/200),
GenerateNalUnit(/*size=*/195)};
rtc::Buffer nalus[] = {
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/200),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/200),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/195)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
@ -189,9 +222,19 @@ TEST(RtpPacketizerH265Test,
// Aggregation tests.
TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) {
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/0x123)};
rtc::Buffer nalus[] = {
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/0x123)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, kNoLimits);
@ -201,22 +244,22 @@ TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) {
auto payload = packets[0].payload();
int type = H265::ParseNaluType(payload[0]);
EXPECT_EQ(payload.size(), kH265NalHeaderSizeBytes +
3 * kH265LengthFieldSizeBytes + 2 + 2 + 0x123);
3 * kH265LengthFieldSizeBytes + 3 + 3 + 0x123);
EXPECT_EQ(type, H265::NaluType::kAp);
payload = payload.subview(kH265NalHeaderSizeBytes);
// 1st fragment.
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes),
ElementsAre(0, 2)); // Size.
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 2),
ElementsAre(0, 3)); // Size.
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 3),
ElementsAreArray(nalus[0]));
payload = payload.subview(kH265LengthFieldSizeBytes + 2);
payload = payload.subview(kH265LengthFieldSizeBytes + 3);
// 2nd fragment.
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes),
ElementsAre(0, 2)); // Size.
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 2),
ElementsAre(0, 3)); // Size.
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 3),
ElementsAreArray(nalus[1]));
payload = payload.subview(kH265LengthFieldSizeBytes + 2);
payload = payload.subview(kH265LengthFieldSizeBytes + 3);
// 3rd fragment.
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes),
ElementsAre(0x1, 0x23)); // Size.
@ -224,15 +267,80 @@ TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) {
ElementsAreArray(nalus[2]));
}
TEST(RtpPacketizerH265Test, ApRespectsLayerIdAndTemporalId) {
// Generate 3 NALUs: NALU 1 with nuh_layer_id 2 and nuh_temporal_id_plus1 6,
// NALU 2 with nuh_layer_id 0 and nuh_temporal_id_plus1 1,
// NALU 3 with nuh_layer_id 32 and nuh_temporal_id_plus1 2,
// So in the AP packet header, nuh_layer_id should be 0 which is the lowest
// nuh_layer_id value of 3 NALUs and nuh_temporal_id_plus1 should be 1 which
// is the lowest nuh_temporal_id_plus1 value of 3 NALUs
rtc::Buffer nalus[] = {
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 2,
.nuh_temporal_id_plus1 = 6},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 0,
.nuh_temporal_id_plus1 = 1},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/0x123)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, kNoLimits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(1));
auto payload = packets[0].payload();
uint8_t type = H265::ParseNaluType(payload[0]);
uint8_t layer_id = ((payload[0] & kH265LayerIDHMask) << 5) |
((payload[1] & kH265LayerIDLMask) >> 3);
uint8_t temporal_id = payload[1] & kH265TIDMask;
EXPECT_EQ(payload.size(), kH265NalHeaderSizeBytes +
3 * kH265LengthFieldSizeBytes + 3 + 3 + 0x123);
EXPECT_EQ(type, H265::NaluType::kAp);
EXPECT_EQ(layer_id, 0);
EXPECT_EQ(temporal_id, 1);
payload = payload.subview(kH265NalHeaderSizeBytes);
// 1st fragment.
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes), ElementsAre(0, 3));
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 3),
ElementsAreArray(nalus[0]));
payload = payload.subview(kH265LengthFieldSizeBytes + 3);
// 2nd fragment.
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes), ElementsAre(0, 3));
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 3),
ElementsAreArray(nalus[1]));
payload = payload.subview(kH265LengthFieldSizeBytes + 3);
// 3rd fragment.
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes),
ElementsAre(0x1, 0x23));
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes),
ElementsAreArray(nalus[2]));
}
TEST(RtpPacketizerH265Test, ApRespectsFirstPacketReduction) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1000;
limits.first_packet_reduction_len = 100;
const size_t kFirstFragmentSize =
limits.max_payload_len - limits.first_packet_reduction_len;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/kFirstFragmentSize),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2)};
rtc::Buffer nalus[] = {
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/kFirstFragmentSize),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/3)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
@ -243,13 +351,13 @@ TEST(RtpPacketizerH265Test, ApRespectsFirstPacketReduction) {
EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0]));
// Expect 2nd packet is aggregate of last two fragments.
// The size of H265 nal_unit_header is 2 bytes, according to 7.3.1.2
// in H265 spec. Aggregation packet type is 48, and nuh_temporal_id_plus1
// is 2, so the nal_unit_header should be "01100000 00000010",
// which is 96 and 2.
// in H265 spec. Aggregation packet type is 48, nuh_layer_id is 32 and
// nuh_temporal_id_plus1 is 2, so the nal_unit_header should be "01100001
// 00000010", which is 97 and 2.
EXPECT_THAT(packets[1].payload(),
ElementsAre(96, 2, //
0, 2, nalus[1][0], nalus[1][1], //
0, 2, nalus[2][0], nalus[2][1]));
ElementsAre(97, 2, //
0, 3, nalus[1][0], nalus[1][1], nalus[1][2], //
0, 3, nalus[2][0], nalus[2][1], nalus[2][2]));
}
TEST(RtpPacketizerH265Test, ApRespectsLastPacketReduction) {
@ -258,9 +366,19 @@ TEST(RtpPacketizerH265Test, ApRespectsLastPacketReduction) {
limits.last_packet_reduction_len = 100;
const size_t kLastFragmentSize =
limits.max_payload_len - limits.last_packet_reduction_len;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/kLastFragmentSize)};
rtc::Buffer nalus[] = {
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/kLastFragmentSize)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
@ -269,9 +387,9 @@ TEST(RtpPacketizerH265Test, ApRespectsLastPacketReduction) {
ASSERT_THAT(packets, SizeIs(2));
// Expect 1st packet is aggregate of 1st two fragments.
EXPECT_THAT(packets[0].payload(),
ElementsAre(96, 2, //
0, 2, nalus[0][0], nalus[0][1], //
0, 2, nalus[1][0], nalus[1][1]));
ElementsAre(97, 2, //
0, 3, nalus[0][0], nalus[0][1], nalus[0][2], //
0, 3, nalus[1][0], nalus[1][1], nalus[1][2]));
// Expect 2nd packet is single nalu.
EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[2]));
}
@ -281,9 +399,19 @@ TEST(RtpPacketizerH265Test, TooSmallForApHeaders) {
limits.max_payload_len = 1000;
const size_t kLastFragmentSize =
limits.max_payload_len - 3 * kH265LengthFieldSizeBytes - 4;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/kLastFragmentSize)};
rtc::Buffer nalus[] = {
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/3),
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
/*size=*/kLastFragmentSize)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
@ -292,9 +420,9 @@ TEST(RtpPacketizerH265Test, TooSmallForApHeaders) {
ASSERT_THAT(packets, SizeIs(2));
// Expect 1st packet is aggregate of 1st two fragments.
EXPECT_THAT(packets[0].payload(),
ElementsAre(96, 2, //
0, 2, nalus[0][0], nalus[0][1], //
0, 2, nalus[1][0], nalus[1][1]));
ElementsAre(97, 2, //
0, 3, nalus[0][0], nalus[0][1], nalus[0][2], //
0, 3, nalus[1][0], nalus[1][1], nalus[1][2]));
// Expect 2nd packet is single nalu.
EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[2]));
}
@ -323,7 +451,10 @@ TEST(RtpPacketizerH265Test, LastFragmentFitsInSingleButNotLastPacket) {
std::vector<int> TestFu(size_t frame_payload_size,
const RtpPacketizer::PayloadSizeLimits& limits) {
rtc::Buffer nalu[] = {
GenerateNalUnit(kH265NalHeaderSizeBytes + frame_payload_size)};
GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
kH265NalHeaderSizeBytes + frame_payload_size)};
rtc::Buffer frame = CreateFrame(nalu);
RtpPacketizerH265 packetizer(frame, limits);
@ -433,7 +564,10 @@ TEST_P(RtpPacketizerH265ParametrizedTest, MixedApFu) {
// Generate nalus according to size specified in paramters
for (size_t index = 0; index < params.nalus.size(); index++) {
nalus.push_back(GenerateNalUnit(params.nalus[index]));
nalus.push_back(GenerateNalUnit({.nal_unit_type = H265::NaluType::kIdrNLp,
.nuh_layer_id = 32,
.nuh_temporal_id_plus1 = 2},
params.nalus[index]));
}
rtc::Buffer frame = CreateFrame(nalus);
@ -462,9 +596,9 @@ TEST_P(RtpPacketizerH265ParametrizedTest, MixedApFu) {
uint8_t fu_header = 0;
fu_header |= (expected_packet.first_fragment ? kH265SBitMask : 0);
fu_header |= (expected_packet.last_fragment ? kH265EBitMask : 0);
fu_header |= H265::NaluType::kTrailR;
fu_header |= H265::NaluType::kIdrNLp;
EXPECT_THAT(packets[i].payload().subview(0, kFuHeaderSizeBytes),
ElementsAre(98, 2, fu_header));
ElementsAre(99, 2, fu_header));
EXPECT_THAT(packets[i].payload().subview(kFuHeaderSizeBytes),
ElementsAreArray(nalus[expected_packet.nalu_index].data() +
kH265NalHeaderSizeBytes +