admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add RTP depacketizer for H265

Browse Source 
1. Depacketize single nalu packet/AP/FU

2. Insert start code before each nalu

Bug: webrtc:13485
Change-Id: I8346f9c31e61e5d3c2c7e1bf5fdaae4018a1ff78
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/325660
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41628}
This commit is contained in:
qwu16 2024-01-19 11:32:17 +08:00 committed by WebRTC LUCI CQ
parent 6adf2243b5
commit f43e8ebab9
12 changed files with 830 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
common_video/h265/h265_common.h
View File

@ -55,11 +55,15 @@ enum NaluType : uint8_t {
kAud = 35,
kPrefixSei = 39,
kSuffixSei = 40,
// Aggregation packets, refer to section 4.4.2 in RFC 7798.
kAp = 48,
kFu = 49
// Fragmentation units, refer to section 4.4.3 in RFC 7798.
kFu = 49,
// PACI packets, refer to section 4.4.4 in RFC 7798.
kPaci = 50
};
// Slice type definition. See table 7-7 of the H265 spec
// Slice type definition. See table 7-7 of the H.265 spec
enum SliceType : uint8_t { kB = 0, kP = 1, kI = 2 };
struct NaluIndex {
@ -78,7 +82,7 @@ std::vector<NaluIndex> FindNaluIndices(const uint8_t* buffer,
// Get the NAL type from the header byte immediately following start sequence.
NaluType ParseNaluType(uint8_t data);
// Methods for parsing and writing RBSP. See section 7.4.2 of the H265 spec.
// Methods for parsing and writing RBSP. See section 7.4.2 of the H.265 spec.
//
// The following sequences are illegal, and need to be escaped when encoding:
// 00 00 00 -> 00 00 03 00

8
modules/rtp_rtcp/BUILD.gn
View File

@ -260,8 +260,11 @@ rtc_library("rtp_rtcp") {
if (rtc_use_h265) {
sources += [
"source/rtp_packet_h265_common.h",
"source/rtp_packetizer_h265.cc",
"source/rtp_packetizer_h265.h",
"source/video_rtp_depacketizer_h265.cc",
"source/video_rtp_depacketizer_h265.h",
]
}
@ -632,7 +635,10 @@ if (rtc_include_tests) {
"source/video_rtp_depacketizer_vp9_unittest.cc",
]
if (rtc_use_h265) {
sources += [ "source/rtp_packetizer_h265_unittest.cc" ]
sources += [
"source/rtp_packetizer_h265_unittest.cc",
"source/video_rtp_depacketizer_h265_unittest.cc",
]
}
deps = [

8
modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc
View File

@ -19,6 +19,9 @@
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_h264.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp8.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h"
#ifdef RTC_ENABLE_H265
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_h265.h"
#endif
namespace webrtc {
@ -34,8 +37,11 @@ std::unique_ptr<VideoRtpDepacketizer> CreateVideoRtpDepacketizer(
case kVideoCodecAV1:
return std::make_unique<VideoRtpDepacketizerAv1>();
case kVideoCodecH265:
// TODO(bugs.webrtc.org/13485): Implement VideoRtpDepacketizerH265.
#ifdef RTC_ENABLE_H265
return std::make_unique<VideoRtpDepacketizerH265>();
#else
return nullptr;
#endif
case kVideoCodecGeneric:
case kVideoCodecMultiplex:
return std::make_unique<VideoRtpDepacketizerGeneric>();

54
modules/rtp_rtcp/source/rtp_packet_h265_common.h Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_RTP_RTCP_SOURCE_RTP_PACKET_H265_COMMON_H_
#define MODULES_RTP_RTCP_SOURCE_RTP_PACKET_H265_COMMON_H_
#include <string>
#include <vector>
namespace webrtc {
// The payload header consists of the same
// fields (F, Type, LayerId and TID) as the NAL unit header. Refer to
// section 4.4 in RFC 7798.
constexpr size_t kH265PayloadHeaderSizeBytes = 2;
// Unlike H.264, H.265 NAL header is 2-bytes.
constexpr size_t kH265NalHeaderSizeBytes = 2;
// H.265's FU is constructed of 2-byte payload header, 1-byte FU header and FU
// payload.
constexpr size_t kH265FuHeaderSizeBytes = 1;
// The NALU size for H.265 RTP aggregated packet indicates the size of the NAL
// unit is 2-bytes.
constexpr size_t kH265LengthFieldSizeBytes = 2;
constexpr size_t kH265ApHeaderSizeBytes =
kH265NalHeaderSizeBytes + kH265LengthFieldSizeBytes;
// Bit masks for NAL headers.
enum NalHdrMasks {
kH265FBit = 0x80,
kH265TypeMask = 0x7E,
kH265LayerIDHMask = 0x1,
kH265LayerIDLMask = 0xF8,
kH265TIDMask = 0x7,
kH265TypeMaskN = 0x81,
kH265TypeMaskInFuHeader = 0x3F
};
// Bit masks for FU headers.
enum FuBitmasks {
kH265SBitMask = 0x80,
kH265EBitMask = 0x40,
kH265FuTypeBitMask = 0x3F
};
constexpr uint8_t kStartCode[] = {0, 0, 0, 1};
} // namespace webrtc
#endif // MODULES_RTP_RTCP_SOURCE_RTP_PACKET_H265_COMMON_H_

58
modules/rtp_rtcp/source/rtp_packetizer_h265.cc
View File

@ -16,42 +16,10 @@
#include "common_video/h264/h264_common.h"
#include "common_video/h265/h265_common.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_packet_h265_common.h"
#include "rtc_base/logging.h"
namespace webrtc {
namespace {
// The payload header consists of the same
// fields (F, Type, LayerId and TID) as the NAL unit header. Refer to
// section 4.2 in RFC 7798.
constexpr size_t kH265PayloadHeaderSize = 2;
// Unlike H.264, H265 NAL header is 2-bytes.
constexpr size_t kH265NalHeaderSize = 2;
// H265's FU is constructed of 2-byte payload header, 1-byte FU header and FU
// payload.
constexpr size_t kH265FuHeaderSize = 1;
// The NALU size for H265 RTP aggregated packet indicates the size of the NAL
// unit is 2-bytes.
constexpr size_t kH265LengthFieldSize = 2;
enum H265NalHdrMasks {
kH265FBit = 0x80,
kH265TypeMask = 0x7E,
kH265LayerIDHMask = 0x1,
kH265LayerIDLMask = 0xF8,
kH265TIDMask = 0x7,
kH265TypeMaskN = 0x81,
kH265TypeMaskInFuHeader = 0x3F
};
// Bit masks for FU headers.
enum H265FuBitmasks {
kH265SBitMask = 0x80,
kH265EBitMask = 0x40,
kH265FuTypeBitMask = 0x3F
};
} // namespace
RtpPacketizerH265::RtpPacketizerH265(rtc::ArrayView<const uint8_t> payload,
PayloadSizeLimits limits)
@ -112,7 +80,8 @@ bool RtpPacketizerH265::PacketizeFu(size_t fragment_index) {
// Refer to section 4.4.3 in RFC7798, each FU fragment will have a 2-bytes
// payload header and a one-byte FU header. DONL is not supported so ignore
// its size when calculating max_payload_len.
limits.max_payload_len -= kH265FuHeaderSize + kH265PayloadHeaderSize;
limits.max_payload_len -=
kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes;
// Update single/first/last packet reductions unless it is single/first/last
// fragment.
@ -135,8 +104,8 @@ bool RtpPacketizerH265::PacketizeFu(size_t fragment_index) {
}
// Strip out the original header.
size_t payload_left = fragment.size() - kH265NalHeaderSize;
int offset = kH265NalHeaderSize;
size_t payload_left = fragment.size() - kH265NalHeaderSizeBytes;
int offset = kH265NalHeaderSizeBytes;
std::vector<int> payload_sizes = SplitAboutEqually(payload_left, limits);
if (payload_sizes.empty()) {
@ -198,12 +167,13 @@ int RtpPacketizerH265::PacketizeAp(size_t fragment_index) {
payload_size_left -= fragment.size();
payload_size_left -= fragment_headers_length;
fragment_headers_length = kH265LengthFieldSize;
fragment_headers_length = kH265LengthFieldSizeBytes;
// If we are going to try to aggregate more fragments into this packet
// we need to add the AP NALU header and a length field for the first
// NALU of this packet.
if (aggregated_fragments == 0) {
fragment_headers_length += kH265PayloadHeaderSize + kH265LengthFieldSize;
fragment_headers_length +=
kH265PayloadHeaderSizeBytes + kH265LengthFieldSizeBytes;
}
++aggregated_fragments;
@ -248,7 +218,7 @@ bool RtpPacketizerH265::NextPacket(RtpPacketToSend* rtp_packet) {
void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) {
size_t payload_capacity = rtp_packet->FreeCapacity();
RTC_CHECK_GE(payload_capacity, kH265PayloadHeaderSize);
RTC_CHECK_GE(payload_capacity, kH265PayloadHeaderSizeBytes);
uint8_t* buffer = rtp_packet->AllocatePayload(payload_capacity);
RTC_CHECK(buffer);
PacketUnit* packet = &packets_.front();
@ -272,13 +242,13 @@ void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) {
buffer[0] = payload_hdr_h;
buffer[1] = payload_hdr_l;
int index = kH265PayloadHeaderSize;
int index = kH265PayloadHeaderSizeBytes;
bool is_last_fragment = packet->last_fragment;
while (packet->aggregated) {
// Add NAL unit length field.
rtc::ArrayView<const uint8_t> fragment = packet->source_fragment;
ByteWriter<uint16_t>::WriteBigEndian(&buffer[index], fragment.size());
index += kH265LengthFieldSize;
index += kH265LengthFieldSizeBytes;
// Add NAL unit.
memcpy(&buffer[index], fragment.data(), fragment.size());
index += fragment.size();
@ -332,15 +302,15 @@ void RtpPacketizerH265::NextFragmentPacket(RtpPacketToSend* rtp_packet) {
(H265::NaluType::kFu << 1) | layer_id_h;
rtc::ArrayView<const uint8_t> fragment = packet->source_fragment;
uint8_t* buffer = rtp_packet->AllocatePayload(
kH265FuHeaderSize + kH265PayloadHeaderSize + fragment.size());
kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes + fragment.size());
RTC_CHECK(buffer);
buffer[0] = payload_hdr_h;
buffer[1] = payload_hdr_l;
buffer[2] = fu_header;
// Do not support DONL for fragmentation units, DONL field is not present.
memcpy(buffer + kH265FuHeaderSize + kH265PayloadHeaderSize, fragment.data(),
fragment.size());
memcpy(buffer + kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes,
fragment.data(), fragment.size());
if (packet->last_fragment) {
input_fragments_.pop_front();
}

85
modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc
View File

@ -15,6 +15,7 @@
#include "common_video/h265/h265_common.h"
#include "modules/rtp_rtcp/mocks/mock_rtp_rtcp.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_packet_h265_common.h"
#include "test/gmock.h"
#include "test/gtest.h"
@ -29,18 +30,12 @@ using ::testing::IsEmpty;
using ::testing::SizeIs;
constexpr RtpPacketToSend::ExtensionManager* kNoExtensions = nullptr;
constexpr size_t kMaxPayloadSize = 1200;
constexpr size_t kLengthFieldLength = 2;
constexpr size_t kMaxPayloadSizeBytes = 1200;
constexpr size_t kH265LengthFieldSizeBytes = 2;
constexpr RtpPacketizer::PayloadSizeLimits kNoLimits;
constexpr size_t kNalHeaderSize = 2;
constexpr size_t kFuHeaderSize = 3;
constexpr uint8_t kNaluTypeMask = 0x7E;
// Bit masks for FU headers.
constexpr uint8_t kH265SBit = 0x80;
constexpr uint8_t kH265EBit = 0x40;
constexpr size_t kFuHeaderSizeBytes =
kH265FuHeaderSizeBytes + kH265PayloadHeaderSizeBytes;
// Creates Buffer that looks like nal unit of given size.
rtc::Buffer GenerateNalUnit(size_t size) {
@ -127,8 +122,8 @@ TEST(RtpPacketizerH265Test, SingleNalu) {
TEST(RtpPacketizerH265Test, SingleNaluTwoPackets) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = kMaxPayloadSize;
rtc::Buffer nalus[] = {GenerateNalUnit(kMaxPayloadSize),
limits.max_payload_len = kMaxPayloadSizeBytes;
rtc::Buffer nalus[] = {GenerateNalUnit(kMaxPayloadSizeBytes),
GenerateNalUnit(100)};
rtc::Buffer frame = CreateFrame(nalus);
@ -205,27 +200,28 @@ TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) {
ASSERT_THAT(packets, SizeIs(1));
auto payload = packets[0].payload();
int type = H265::ParseNaluType(payload[0]);
EXPECT_EQ(payload.size(),
kNalHeaderSize + 3 * kLengthFieldLength + 2 + 2 + 0x123);
EXPECT_EQ(payload.size(), kH265NalHeaderSizeBytes +
3 * kH265LengthFieldSizeBytes + 2 + 2 + 0x123);
EXPECT_EQ(type, H265::NaluType::kAp);
payload = payload.subview(kNalHeaderSize);
payload = payload.subview(kH265NalHeaderSizeBytes);
// 1st fragment.
EXPECT_THAT(payload.subview(0, kLengthFieldLength),
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes),
ElementsAre(0, 2)); // Size.
EXPECT_THAT(payload.subview(kLengthFieldLength, 2),
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 2),
ElementsAreArray(nalus[0]));
payload = payload.subview(kLengthFieldLength + 2);
payload = payload.subview(kH265LengthFieldSizeBytes + 2);
// 2nd fragment.
EXPECT_THAT(payload.subview(0, kLengthFieldLength),
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes),
ElementsAre(0, 2)); // Size.
EXPECT_THAT(payload.subview(kLengthFieldLength, 2),
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes, 2),
ElementsAreArray(nalus[1]));
payload = payload.subview(kLengthFieldLength + 2);
payload = payload.subview(kH265LengthFieldSizeBytes + 2);
// 3rd fragment.
EXPECT_THAT(payload.subview(0, kLengthFieldLength),
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes),
ElementsAre(0x1, 0x23)); // Size.
EXPECT_THAT(payload.subview(kLengthFieldLength), ElementsAreArray(nalus[2]));
EXPECT_THAT(payload.subview(kH265LengthFieldSizeBytes),
ElementsAreArray(nalus[2]));
}
TEST(RtpPacketizerH265Test, ApRespectsFirstPacketReduction) {
@ -284,7 +280,7 @@ TEST(RtpPacketizerH265Test, TooSmallForApHeaders) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1000;
const size_t kLastFragmentSize =
limits.max_payload_len - 3 * kLengthFieldLength - 4;
limits.max_payload_len - 3 * kH265LengthFieldSizeBytes - 4;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/kLastFragmentSize)};
@ -326,7 +322,8 @@ TEST(RtpPacketizerH265Test, LastFragmentFitsInSingleButNotLastPacket) {
// Returns sizes of the payloads excluding FU headers.
std::vector<int> TestFu(size_t frame_payload_size,
const RtpPacketizer::PayloadSizeLimits& limits) {
rtc::Buffer nalu[] = {GenerateNalUnit(kNalHeaderSize + frame_payload_size)};
rtc::Buffer nalu[] = {
GenerateNalUnit(kH265NalHeaderSizeBytes + frame_payload_size)};
rtc::Buffer frame = CreateFrame(nalu);
RtpPacketizerH265 packetizer(frame, limits);
@ -338,18 +335,18 @@ std::vector<int> TestFu(size_t frame_payload_size,
for (const RtpPacketToSend& packet : packets) {
auto payload = packet.payload();
EXPECT_GT(payload.size(), kFuHeaderSize);
EXPECT_GT(payload.size(), kFuHeaderSizeBytes);
// FU header is after the 2-bytes size PayloadHdr according to 4.4.3 in spec
fu_header.push_back(payload[2]);
payload_sizes.push_back(payload.size() - kFuHeaderSize);
payload_sizes.push_back(payload.size() - kFuHeaderSizeBytes);
}
EXPECT_TRUE(fu_header.front() & kH265SBit);
EXPECT_TRUE(fu_header.back() & kH265EBit);
EXPECT_TRUE(fu_header.front() & kH265SBitMask);
EXPECT_TRUE(fu_header.back() & kH265EBitMask);
// Clear S and E bits before testing all are duplicating same original header.
fu_header.front() &= ~kH265SBit;
fu_header.back() &= ~kH265EBit;
uint8_t nalu_type = (nalu[0][0] & kNaluTypeMask) >> 1;
fu_header.front() &= ~kH265SBitMask;
fu_header.back() &= ~kH265EBitMask;
uint8_t nalu_type = (nalu[0][0] & kH265TypeMask) >> 1;
EXPECT_THAT(fu_header, Each(Eq(nalu_type)));
return payload_sizes;
@ -403,7 +400,7 @@ TEST(RtpPacketizerH265Test, FuBig) {
limits.max_payload_len = 1200;
// Generate 10 full sized packets, leave room for FU headers.
EXPECT_THAT(
TestFu(10 * (1200 - kFuHeaderSize), limits),
TestFu(10 * (1200 - kFuHeaderSizeBytes), limits),
ElementsAre(1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197));
}
@ -449,29 +446,29 @@ TEST_P(RtpPacketizerH265ParametrizedTest, MixedApFu) {
if (expected_packet.aggregated) {
int type = H265::ParseNaluType(packets[i].payload()[0]);
EXPECT_THAT(type, H265::NaluType::kAp);
auto payload = packets[i].payload().subview(kNalHeaderSize);
auto payload = packets[i].payload().subview(kH265NalHeaderSizeBytes);
int offset = 0;
// Generated AP packet header and payload align
for (int j = expected_packet.nalu_index; j < expected_packet.nalu_number;
j++) {
EXPECT_THAT(payload.subview(0, kLengthFieldLength),
EXPECT_THAT(payload.subview(0, kH265LengthFieldSizeBytes),
ElementsAre(0, nalus[j].size()));
EXPECT_THAT(
payload.subview(offset + kLengthFieldLength, nalus[j].size()),
EXPECT_THAT(payload.subview(offset + kH265LengthFieldSizeBytes,
nalus[j].size()),
ElementsAreArray(nalus[j]));
offset += kLengthFieldLength + nalus[j].size();
offset += kH265LengthFieldSizeBytes + nalus[j].size();
}
} else {
uint8_t fu_header = 0;
fu_header |= (expected_packet.first_fragment ? kH265SBit : 0);
fu_header |= (expected_packet.last_fragment ? kH265EBit : 0);
fu_header |= (expected_packet.first_fragment ? kH265SBitMask : 0);
fu_header |= (expected_packet.last_fragment ? kH265EBitMask : 0);
fu_header |= H265::NaluType::kTrailR;
EXPECT_THAT(packets[i].payload().subview(0, kFuHeaderSize),
EXPECT_THAT(packets[i].payload().subview(0, kFuHeaderSizeBytes),
ElementsAre(98, 2, fu_header));
EXPECT_THAT(
packets[i].payload().subview(kFuHeaderSize),
EXPECT_THAT(packets[i].payload().subview(kFuHeaderSizeBytes),
ElementsAreArray(nalus[expected_packet.nalu_index].data() +
kNalHeaderSize + expected_packet.start_offset,
kH265NalHeaderSizeBytes +
expected_packet.start_offset,
expected_packet.payload_size));
}
}

244
modules/rtp_rtcp/source/video_rtp_depacketizer_h265.cc Normal file
View File

@ -0,0 +1,244 @@
/*
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_h265.h"
#include <cstddef>
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
#include "absl/base/macros.h"
#include "absl/types/optional.h"
#include "absl/types/variant.h"
#include "api/video/video_codec_type.h"
#include "common_video/h264/h264_common.h"
#include "common_video/h265/h265_bitstream_parser.h"
#include "common_video/h265/h265_common.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_packet_h265_common.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
namespace {
bool ParseApStartOffsets(const uint8_t* nalu_ptr,
size_t length_remaining,
std::vector<size_t>* offsets) {
size_t offset = 0;
while (length_remaining > 0) {
// Buffer doesn't contain room for additional NALU length.
if (length_remaining < kH265LengthFieldSizeBytes)
return false;
// Read 16-bit NALU size defined in RFC7798 section 4.4.2.
uint16_t nalu_size = ByteReader<uint16_t>::ReadBigEndian(nalu_ptr);
nalu_ptr += kH265LengthFieldSizeBytes;
length_remaining -= kH265LengthFieldSizeBytes;
if (nalu_size > length_remaining)
return false;
nalu_ptr += nalu_size;
length_remaining -= nalu_size;
offsets->push_back(offset + kH265ApHeaderSizeBytes);
offset += kH265LengthFieldSizeBytes + nalu_size;
}
return true;
}
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> ProcessApOrSingleNalu(
rtc::CopyOnWriteBuffer rtp_payload) {
// Skip the single NALU header (payload header), aggregated packet case will
// be checked later.
if (rtp_payload.size() <= kH265PayloadHeaderSizeBytes) {
RTC_LOG(LS_ERROR) << "Single NALU header truncated.";
return absl::nullopt;
}
const uint8_t* const payload_data = rtp_payload.cdata();
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload(
absl::in_place);
parsed_payload->video_header.width = 0;
parsed_payload->video_header.height = 0;
parsed_payload->video_header.codec = kVideoCodecH265;
parsed_payload->video_header.is_first_packet_in_frame = true;
const uint8_t* nalu_start = payload_data + kH265PayloadHeaderSizeBytes;
const size_t nalu_length = rtp_payload.size() - kH265PayloadHeaderSizeBytes;
uint8_t nal_type = (payload_data[0] & kH265TypeMask) >> 1;
std::vector<size_t> nalu_start_offsets;
rtc::CopyOnWriteBuffer video_payload;
if (nal_type == H265::NaluType::kAp) {
// Skip the aggregated packet header (Aggregated packet NAL type + length).
if (rtp_payload.size() <= kH265ApHeaderSizeBytes) {
RTC_LOG(LS_ERROR) << "Aggregated packet header truncated.";
return absl::nullopt;
}
if (!ParseApStartOffsets(nalu_start, nalu_length, &nalu_start_offsets)) {
RTC_LOG(LS_ERROR)
<< "Aggregated packet with incorrect NALU packet lengths.";
return absl::nullopt;
}
nal_type = (payload_data[kH265ApHeaderSizeBytes] & kH265TypeMask) >> 1;
} else {
nalu_start_offsets.push_back(0);
}
parsed_payload->video_header.frame_type = VideoFrameType::kVideoFrameDelta;
nalu_start_offsets.push_back(rtp_payload.size() +
kH265LengthFieldSizeBytes); // End offset.
for (size_t i = 0; i < nalu_start_offsets.size() - 1; ++i) {
size_t start_offset = nalu_start_offsets[i];
// End offset is actually start offset for next unit, excluding length field
// so remove that from this units length.
size_t end_offset = nalu_start_offsets[i + 1] - kH265LengthFieldSizeBytes;
if (end_offset - start_offset < kH265NalHeaderSizeBytes) {
RTC_LOG(LS_ERROR) << "Aggregated packet too short";
return absl::nullopt;
}
// Insert start code before each NALU in aggregated packet.
video_payload.AppendData(kStartCode);
video_payload.AppendData(&payload_data[start_offset],
end_offset - start_offset);
uint8_t nalu_type = (payload_data[start_offset] & kH265TypeMask) >> 1;
start_offset += kH265NalHeaderSizeBytes;
switch (nalu_type) {
case H265::NaluType::kBlaWLp:
case H265::NaluType::kBlaWRadl:
case H265::NaluType::kBlaNLp:
case H265::NaluType::kIdrWRadl:
case H265::NaluType::kIdrNLp:
case H265::NaluType::kCra:
case H265::NaluType::kRsvIrapVcl23:
parsed_payload->video_header.frame_type =
VideoFrameType::kVideoFrameKey;
ABSL_FALLTHROUGH_INTENDED;
case H265::NaluType::kSps: {
// Copy any previous data first (likely just the first header).
std::unique_ptr<rtc::Buffer> output_buffer(new rtc::Buffer());
if (start_offset)
output_buffer->AppendData(payload_data, start_offset);
absl::optional<H265SpsParser::SpsState> sps = H265SpsParser::ParseSps(
&payload_data[start_offset], end_offset - start_offset);
if (sps) {
// TODO(bugs.webrtc.org/13485): Implement the size calculation taking
// VPS->vui_parameters.def_disp_win_xx_offset into account.
parsed_payload->video_header.width = sps->width;
parsed_payload->video_header.height = sps->height;
} else {
RTC_LOG(LS_WARNING) << "Failed to parse SPS from SPS slice.";
}
}
ABSL_FALLTHROUGH_INTENDED;
case H265::NaluType::kVps:
case H265::NaluType::kPps:
case H265::NaluType::kTrailN:
case H265::NaluType::kTrailR:
// Slices below don't contain SPS or PPS ids.
case H265::NaluType::kAud:
case H265::NaluType::kTsaN:
case H265::NaluType::kTsaR:
case H265::NaluType::kStsaN:
case H265::NaluType::kStsaR:
case H265::NaluType::kRadlN:
case H265::NaluType::kRadlR:
case H265::NaluType::kPrefixSei:
case H265::NaluType::kSuffixSei:
break;
case H265::NaluType::kAp:
case H265::NaluType::kFu:
case H265::NaluType::kPaci:
RTC_LOG(LS_WARNING) << "Unexpected AP, FU or PACI received.";
return absl::nullopt;
}
}
parsed_payload->video_payload = video_payload;
return parsed_payload;
}
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> ParseFuNalu(
rtc::CopyOnWriteBuffer rtp_payload) {
if (rtp_payload.size() < kH265FuHeaderSizeBytes + kH265NalHeaderSizeBytes) {
RTC_LOG(LS_ERROR) << "FU NAL units truncated.";
return absl::nullopt;
}
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload(
absl::in_place);
uint8_t f = rtp_payload.cdata()[0] & kH265FBit;
uint8_t layer_id_h = rtp_payload.cdata()[0] & kH265LayerIDHMask;
uint8_t layer_id_l_unshifted = rtp_payload.cdata()[1] & kH265LayerIDLMask;
uint8_t tid = rtp_payload.cdata()[1] & kH265TIDMask;
uint8_t original_nal_type = rtp_payload.cdata()[2] & kH265TypeMaskInFuHeader;
bool first_fragment = rtp_payload.cdata()[2] & kH265SBitMask;
if (first_fragment) {
rtp_payload = rtp_payload.Slice(
kH265FuHeaderSizeBytes, rtp_payload.size() - kH265FuHeaderSizeBytes);
rtp_payload.MutableData()[0] = f | original_nal_type << 1 | layer_id_h;
rtp_payload.MutableData()[1] = layer_id_l_unshifted | tid;
rtc::CopyOnWriteBuffer video_payload;
// Insert start code before the first fragment in FU.
video_payload.AppendData(kStartCode);
video_payload.AppendData(rtp_payload);
parsed_payload->video_payload = video_payload;
} else {
parsed_payload->video_payload = rtp_payload.Slice(
kH265NalHeaderSizeBytes + kH265FuHeaderSizeBytes,
rtp_payload.size() - kH265NalHeaderSizeBytes - kH265FuHeaderSizeBytes);
}
if (original_nal_type == H265::NaluType::kIdrWRadl ||
original_nal_type == H265::NaluType::kIdrNLp ||
original_nal_type == H265::NaluType::kCra) {
parsed_payload->video_header.frame_type = VideoFrameType::kVideoFrameKey;
} else {
parsed_payload->video_header.frame_type = VideoFrameType::kVideoFrameDelta;
}
parsed_payload->video_header.width = 0;
parsed_payload->video_header.height = 0;
parsed_payload->video_header.codec = kVideoCodecH265;
parsed_payload->video_header.is_first_packet_in_frame = first_fragment;
return parsed_payload;
}
} // namespace
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload>
VideoRtpDepacketizerH265::Parse(rtc::CopyOnWriteBuffer rtp_payload) {
if (rtp_payload.empty()) {
RTC_LOG(LS_ERROR) << "Empty payload.";
return absl::nullopt;
}
uint8_t nal_type = (rtp_payload.cdata()[0] & kH265TypeMask) >> 1;
if (nal_type == H265::NaluType::kFu) {
// Fragmented NAL units (FU).
return ParseFuNalu(std::move(rtp_payload));
} else if (nal_type == H265::NaluType::kPaci) {
// TODO(bugs.webrtc.org/13485): Implement PACI parse for H265
RTC_LOG(LS_ERROR) << "Not support type:" << nal_type;
return absl::nullopt;
} else {
// Single NAL unit packet or Aggregated packets (AP).
return ProcessApOrSingleNalu(std::move(rtp_payload));
}
}
} // namespace webrtc

28
modules/rtp_rtcp/source/video_rtp_depacketizer_h265.h Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_RTP_RTCP_SOURCE_VIDEO_RTP_DEPACKETIZER_H265_H_
#define MODULES_RTP_RTCP_SOURCE_VIDEO_RTP_DEPACKETIZER_H265_H_
#include "absl/types/optional.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
#include "rtc_base/copy_on_write_buffer.h"
namespace webrtc {
class VideoRtpDepacketizerH265 : public VideoRtpDepacketizer {
public:
~VideoRtpDepacketizerH265() override = default;
absl::optional<ParsedRtpPayload> Parse(
rtc::CopyOnWriteBuffer rtp_payload) override;
};
} // namespace webrtc
#endif // MODULES_RTP_RTCP_SOURCE_VIDEO_RTP_DEPACKETIZER_H265_H_

400
modules/rtp_rtcp/source/video_rtp_depacketizer_h265_unittest.cc Normal file
View File

@ -0,0 +1,400 @@
/*
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_h265.h"
#include <cstdint>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "common_video/h265/h265_common.h"
#include "modules/rtp_rtcp/mocks/mock_rtp_rtcp.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_packet_h265_common.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
using ::testing::Each;
using ::testing::ElementsAre;
using ::testing::ElementsAreArray;
using ::testing::Eq;
using ::testing::IsEmpty;
using ::testing::SizeIs;
TEST(VideoRtpDepacketizerH265Test, SingleNalu) {
uint8_t packet[3] = {0x26, 0x02,
0xFF}; // F=0, Type=19 (Idr), LayerId=0, TID=2.
uint8_t expected_packet[] = {0x00, 0x00, 0x00, 0x01, 0x26, 0x02, 0xff};
rtc::CopyOnWriteBuffer rtp_payload(packet);
VideoRtpDepacketizerH265 depacketizer;
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed =
depacketizer.Parse(rtp_payload);
ASSERT_TRUE(parsed);
EXPECT_THAT(rtc::MakeArrayView(parsed->video_payload.cdata(),
parsed->video_payload.size()),
ElementsAreArray(expected_packet));
EXPECT_EQ(parsed->video_header.frame_type, VideoFrameType::kVideoFrameKey);
EXPECT_EQ(parsed->video_header.codec, kVideoCodecH265);
EXPECT_TRUE(parsed->video_header.is_first_packet_in_frame);
}
TEST(VideoRtpDepacketizerH265Test, SingleNaluSpsWithResolution) {
// SPS for a 1280x720 camera capture from ffmpeg on linux. Contains
// emulation bytes but no cropping. This buffer is generated
// with following command:
// 1) ffmpeg -i /dev/video0 -r 30 -c:v libx265 -s 1280x720 camera.h265
//
// 2) Open camera.h265 and find the SPS, generally everything between the
// second and third start codes (0 0 0 1 or 0 0 1). The first two bytes
// 0x42 and 0x02 shows the nal header of SPS.
uint8_t packet[] = {0x42, 0x02, 0x01, 0x04, 0x08, 0x00, 0x00, 0x03,
0x00, 0x9d, 0x08, 0x00, 0x00, 0x03, 0x00, 0x00,
0x5d, 0xb0, 0x02, 0x80, 0x80, 0x2d, 0x16, 0x59,
0x59, 0xa4, 0x93, 0x2b, 0x80, 0x40, 0x00, 0x00,
0x03, 0x00, 0x40, 0x00, 0x00, 0x07, 0x82};
uint8_t expected_packet[] = {
0x00, 0x00, 0x00, 0x01, 0x42, 0x02, 0x01, 0x04, 0x08, 0x00, 0x00,
0x03, 0x00, 0x9d, 0x08, 0x00, 0x00, 0x03, 0x00, 0x00, 0x5d, 0xb0,
0x02, 0x80, 0x80, 0x2d, 0x16, 0x59, 0x59, 0xa4, 0x93, 0x2b, 0x80,
0x40, 0x00, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x07, 0x82};
rtc::CopyOnWriteBuffer rtp_payload(packet);
VideoRtpDepacketizerH265 depacketizer;
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed =
depacketizer.Parse(rtp_payload);
ASSERT_TRUE(parsed);
EXPECT_THAT(rtc::MakeArrayView(parsed->video_payload.cdata(),
parsed->video_payload.size()),
ElementsAreArray(expected_packet));
EXPECT_EQ(parsed->video_header.codec, kVideoCodecH265);
EXPECT_TRUE(parsed->video_header.is_first_packet_in_frame);
EXPECT_EQ(parsed->video_header.width, 1280u);
EXPECT_EQ(parsed->video_header.height, 720u);
}
TEST(VideoRtpDepacketizerH265Test, PaciPackets) {
uint8_t packet[2] = {0x64, 0x02}; // F=0, Type=50 (PACI), LayerId=0, TID=2.
rtc::CopyOnWriteBuffer rtp_payload(packet);
VideoRtpDepacketizerH265 depacketizer;
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed =
depacketizer.Parse(rtp_payload);
ASSERT_FALSE(parsed);
}
TEST(VideoRtpDepacketizerH265Test, ApKey) {
uint8_t payload_header[] = {0x60, 0x02};
uint8_t vps_nalu_size[] = {0, 0x17};
uint8_t sps_nalu_size[] = {0, 0x27};
uint8_t pps_nalu_size[] = {0, 0x32};
uint8_t slice_nalu_size[] = {0, 0xa};
uint8_t start_code[] = {0x00, 0x00, 0x00, 0x01};
// VPS/SPS/PPS/IDR for a 1280x720 camera capture from ffmpeg on linux.
// Contains emulation bytes but no cropping. This buffer is generated with
// following command: 1) ffmpeg -i /dev/video0 -r 30 -c:v libx265 -s 1280x720
// camera.h265
//
// 2) Open camera.h265 and find:
// VPS - generally everything between the first and second start codes (0 0 0
// 1 or 0 0 1). The first two bytes 0x40 and 0x02 shows the nal header of VPS.
// SPS - generally everything between the
// second and third start codes (0 0 0 1 or 0 0 1). The first two bytes
// 0x42 and 0x02 shows the nal header of SPS.
// PPS - generally everything between the third and fourth start codes (0 0 0
// 1 or 0 0 1). The first two bytes 0x44 and 0x02 shows the nal header of PPS.
// IDR - Part of the keyframe bitstream (no need to show all the bytes for
// depacketizer testing). The first two bytes 0x26 and 0x02 shows the nal
// header of IDR frame.
uint8_t vps[] = {
0x40, 0x02, 0x1c, 0x01, 0xff, 0xff, 0x04, 0x08, 0x00, 0x00, 0x03, 0x00,
0x9d, 0x08, 0x00, 0x00, 0x03, 0x00, 0x00, 0x78, 0x95, 0x98, 0x09,
};
uint8_t sps[] = {0x42, 0x02, 0x01, 0x04, 0x08, 0x00, 0x00, 0x03, 0x00, 0x9d,
0x08, 0x00, 0x00, 0x03, 0x00, 0x00, 0x5d, 0xb0, 0x02, 0x80,
0x80, 0x2d, 0x16, 0x59, 0x59, 0xa4, 0x93, 0x2b, 0x80, 0x40,
0x00, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x07, 0x82};
uint8_t pps[] = {0x44, 0x02, 0xa4, 0x04, 0x55, 0xa2, 0x6d, 0xce, 0xc0, 0xc3,
0xed, 0x0b, 0xac, 0xbc, 0x00, 0xc4, 0x44, 0x2e, 0xf7, 0x55,
0xfd, 0x05, 0x86, 0x92, 0x19, 0xdf, 0x58, 0xec, 0x38, 0x36,
0xb7, 0x7c, 0x00, 0x15, 0x33, 0x78, 0x03, 0x67, 0x26, 0x0f,
0x7b, 0x30, 0x1c, 0xd7, 0xd4, 0x3a, 0xec, 0xad, 0xef, 0x73};
uint8_t idr[] = {0x26, 0x02, 0xaf, 0x08, 0x4a, 0x31, 0x11, 0x15, 0xe5, 0xc0};
rtc::Buffer packet;
packet.AppendData(payload_header);
packet.AppendData(vps_nalu_size);
packet.AppendData(vps);
packet.AppendData(sps_nalu_size);
packet.AppendData(sps);
packet.AppendData(pps_nalu_size);
packet.AppendData(pps);
packet.AppendData(slice_nalu_size);
packet.AppendData(idr);
rtc::Buffer expected_packet;
expected_packet.AppendData(start_code);
expected_packet.AppendData(vps);
expected_packet.AppendData(start_code);
expected_packet.AppendData(sps);
expected_packet.AppendData(start_code);
expected_packet.AppendData(pps);
expected_packet.AppendData(start_code);
expected_packet.AppendData(idr);
// clang-format on
rtc::CopyOnWriteBuffer rtp_payload(packet);
VideoRtpDepacketizerH265 depacketizer;
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed =
depacketizer.Parse(rtp_payload);
ASSERT_TRUE(parsed);
EXPECT_THAT(rtc::MakeArrayView(parsed->video_payload.cdata(),
parsed->video_payload.size()),
ElementsAreArray(expected_packet));
EXPECT_EQ(parsed->video_header.frame_type, VideoFrameType::kVideoFrameKey);
EXPECT_EQ(parsed->video_header.codec, kVideoCodecH265);
EXPECT_TRUE(parsed->video_header.is_first_packet_in_frame);
}
TEST(VideoRtpDepacketizerH265Test, ApNaluSpsWithResolution) {
uint8_t payload_header[] = {0x60, 0x02};
uint8_t vps_nalu_size[] = {0, 0x17};
uint8_t sps_nalu_size[] = {0, 0x27};
uint8_t pps_nalu_size[] = {0, 0x32};
uint8_t slice_nalu_size[] = {0, 0xa};
uint8_t start_code[] = {0x00, 0x00, 0x00, 0x01};
// The VPS/SPS/PPS/IDR bytes are generated using the same way as above case.
uint8_t vps[] = {
0x40, 0x02, 0x1c, 0x01, 0xff, 0xff, 0x04, 0x08, 0x00, 0x00, 0x03, 0x00,
0x9d, 0x08, 0x00, 0x00, 0x03, 0x00, 0x00, 0x78, 0x95, 0x98, 0x09,
};
uint8_t sps[] = {0x42, 0x02, 0x01, 0x04, 0x08, 0x00, 0x00, 0x03, 0x00, 0x9d,
0x08, 0x00, 0x00, 0x03, 0x00, 0x00, 0x5d, 0xb0, 0x02, 0x80,
0x80, 0x2d, 0x16, 0x59, 0x59, 0xa4, 0x93, 0x2b, 0x80, 0x40,
0x00, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x07, 0x82};
uint8_t pps[] = {0x44, 0x02, 0xa4, 0x04, 0x55, 0xa2, 0x6d, 0xce, 0xc0, 0xc3,
0xed, 0x0b, 0xac, 0xbc, 0x00, 0xc4, 0x44, 0x2e, 0xf7, 0x55,
0xfd, 0x05, 0x86, 0x92, 0x19, 0xdf, 0x58, 0xec, 0x38, 0x36,
0xb7, 0x7c, 0x00, 0x15, 0x33, 0x78, 0x03, 0x67, 0x26, 0x0f,
0x7b, 0x30, 0x1c, 0xd7, 0xd4, 0x3a, 0xec, 0xad, 0xef, 0x73};
uint8_t idr[] = {0x26, 0x02, 0xaf, 0x08, 0x4a, 0x31, 0x11, 0x15, 0xe5, 0xc0};
rtc::Buffer packet;
packet.AppendData(payload_header);
packet.AppendData(vps_nalu_size);
packet.AppendData(vps);
packet.AppendData(sps_nalu_size);
packet.AppendData(sps);
packet.AppendData(pps_nalu_size);
packet.AppendData(pps);
packet.AppendData(slice_nalu_size);
packet.AppendData(idr);
rtc::Buffer expected_packet;
expected_packet.AppendData(start_code);
expected_packet.AppendData(vps);
expected_packet.AppendData(start_code);
expected_packet.AppendData(sps);
expected_packet.AppendData(start_code);
expected_packet.AppendData(pps);
expected_packet.AppendData(start_code);
expected_packet.AppendData(idr);
rtc::CopyOnWriteBuffer rtp_payload(packet);
VideoRtpDepacketizerH265 depacketizer;
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed =
depacketizer.Parse(rtp_payload);
ASSERT_TRUE(parsed);
EXPECT_THAT(rtc::MakeArrayView(parsed->video_payload.cdata(),
parsed->video_payload.size()),
ElementsAreArray(expected_packet));
EXPECT_EQ(parsed->video_header.frame_type, VideoFrameType::kVideoFrameKey);
EXPECT_EQ(parsed->video_header.codec, kVideoCodecH265);
EXPECT_TRUE(parsed->video_header.is_first_packet_in_frame);
EXPECT_EQ(parsed->video_header.width, 1280u);
EXPECT_EQ(parsed->video_header.height, 720u);
}
TEST(VideoRtpDepacketizerH265Test, EmptyApRejected) {
uint8_t lone_empty_packet[] = {0x60, 0x02, // F=0, Type=48 (kH265Ap).
0x00, 0x00};
uint8_t leading_empty_packet[] = {0x60, 0x02, // F=0, Type=48 (kH265Ap).
0x00, 0x00, 0x00, 0x05, 0x26,
0x02, 0xFF, 0x00, 0x11}; // kIdrWRadl
uint8_t middle_empty_packet[] = {0x60, 0x02, // F=0, Type=48 (kH265Ap).
0x00, 0x04, 0x26, 0x02, 0xFF,
0x00, 0x00, 0x00, 0x00, 0x05,
0x26, 0x02, 0xFF, 0x00, 0x11}; // kIdrWRadl
uint8_t trailing_empty_packet[] = {0x60, 0x02, // F=0, Type=48 (kH265Ap).
0x00, 0x04, 0x26,
0x02, 0xFF, 0x00, // kIdrWRadl
0x00, 0x00};
VideoRtpDepacketizerH265 depacketizer;
EXPECT_FALSE(depacketizer.Parse(rtc::CopyOnWriteBuffer(lone_empty_packet)));
EXPECT_FALSE(
depacketizer.Parse(rtc::CopyOnWriteBuffer(leading_empty_packet)));
EXPECT_FALSE(depacketizer.Parse(rtc::CopyOnWriteBuffer(middle_empty_packet)));
EXPECT_FALSE(
depacketizer.Parse(rtc::CopyOnWriteBuffer(trailing_empty_packet)));
}
TEST(VideoRtpDepacketizerH265Test, ApDelta) {
uint8_t packet[20] = {0x60, 0x02, // F=0, Type=48 (kH265Ap).
// Length, nal header, payload.
0, 0x03, 0x02, 0x02, 0xFF, // TrailR
0, 0x04, 0x02, 0x02, 0xFF, 0x00, // TrailR
0, 0x05, 0x02, 0x02, 0xFF, 0x00, 0x11}; // TrailR
uint8_t expected_packet[] = {
0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0xFF, // TrailR
0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0xFF, 0x00, // TrailR
0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0xFF, 0x00, 0x11}; // TrailR
rtc::CopyOnWriteBuffer rtp_payload(packet);
VideoRtpDepacketizerH265 depacketizer;
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed =
depacketizer.Parse(rtp_payload);
ASSERT_TRUE(parsed);
EXPECT_THAT(rtc::MakeArrayView(parsed->video_payload.cdata(),
parsed->video_payload.size()),
ElementsAreArray(expected_packet));
EXPECT_EQ(parsed->video_header.frame_type, VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(parsed->video_header.codec, kVideoCodecH265);
EXPECT_TRUE(parsed->video_header.is_first_packet_in_frame);
}
TEST(VideoRtpDepacketizerH265Test, Fu) {
// clang-format off
uint8_t packet1[] = {
0x62, 0x02, // F=0, Type=49 (kH265Fu).
0x93, // FU header kH265SBitMask | H265::kIdrWRadl.
0xaf, 0x08, 0x4a, 0x31, 0x11, 0x15, 0xe5, 0xc0 // Payload.
};
// clang-format on
// F=0, Type=19, (kIdrWRadl), tid=1, nalu header: 00100110 00000010, which is
// 0x26, 0x02
const uint8_t kExpected1[] = {0x00, 0x00, 0x00, 0x01, 0x26, 0x02, 0xaf,
0x08, 0x4a, 0x31, 0x11, 0x15, 0xe5, 0xc0};
uint8_t packet2[] = {
0x62, 0x02, // F=0, Type=49 (kH265Fu).
H265::kIdrWRadl, // FU header.
0x02 // Payload.
};
const uint8_t kExpected2[] = {0x02};
uint8_t packet3[] = {
0x62, 0x02, // F=0, Type=49 (kH265Fu).
0x33, // FU header kH265EBitMask | H265::kIdrWRadl.
0x03 // Payload.
};
const uint8_t kExpected3[] = {0x03};
VideoRtpDepacketizerH265 depacketizer;
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed1 =
depacketizer.Parse(rtc::CopyOnWriteBuffer(packet1));
ASSERT_TRUE(parsed1);
// We expect that the first packet is one byte shorter since the FU header
// has been replaced by the original nal header.
EXPECT_THAT(rtc::MakeArrayView(parsed1->video_payload.cdata(),
parsed1->video_payload.size()),
ElementsAreArray(kExpected1));
EXPECT_EQ(parsed1->video_header.frame_type, VideoFrameType::kVideoFrameKey);
EXPECT_EQ(parsed1->video_header.codec, kVideoCodecH265);
EXPECT_TRUE(parsed1->video_header.is_first_packet_in_frame);
// Following packets will be 2 bytes shorter since they will only be appended
// onto the first packet.
auto parsed2 = depacketizer.Parse(rtc::CopyOnWriteBuffer(packet2));
EXPECT_THAT(rtc::MakeArrayView(parsed2->video_payload.cdata(),
parsed2->video_payload.size()),
ElementsAreArray(kExpected2));
EXPECT_FALSE(parsed2->video_header.is_first_packet_in_frame);
EXPECT_EQ(parsed2->video_header.codec, kVideoCodecH265);
auto parsed3 = depacketizer.Parse(rtc::CopyOnWriteBuffer(packet3));
EXPECT_THAT(rtc::MakeArrayView(parsed3->video_payload.cdata(),
parsed3->video_payload.size()),
ElementsAreArray(kExpected3));
EXPECT_FALSE(parsed3->video_header.is_first_packet_in_frame);
EXPECT_EQ(parsed3->video_header.codec, kVideoCodecH265);
}
TEST(VideoRtpDepacketizerH265Test, EmptyPayload) {
rtc::CopyOnWriteBuffer empty;
VideoRtpDepacketizerH265 depacketizer;
EXPECT_FALSE(depacketizer.Parse(empty));
}
TEST(VideoRtpDepacketizerH265Test, TruncatedFuNalu) {
const uint8_t kPayload[] = {0x62};
VideoRtpDepacketizerH265 depacketizer;
EXPECT_FALSE(depacketizer.Parse(rtc::CopyOnWriteBuffer(kPayload)));
}
TEST(VideoRtpDepacketizerH265Test, TruncatedSingleApNalu) {
const uint8_t kPayload[] = {0xe0, 0x02, 0x40};
VideoRtpDepacketizerH265 depacketizer;
EXPECT_FALSE(depacketizer.Parse(rtc::CopyOnWriteBuffer(kPayload)));
}
TEST(VideoRtpDepacketizerH265Test, ApPacketWithTruncatedNalUnits) {
const uint8_t kPayload[] = {0x60, 0x02, 0xED, 0xDF};
VideoRtpDepacketizerH265 depacketizer;
EXPECT_FALSE(depacketizer.Parse(rtc::CopyOnWriteBuffer(kPayload)));
}
TEST(VideoRtpDepacketizerH265Test, TruncationJustAfterSingleApNalu) {
const uint8_t kPayload[] = {0x60, 0x02, 0x40, 0x40};
VideoRtpDepacketizerH265 depacketizer;
EXPECT_FALSE(depacketizer.Parse(rtc::CopyOnWriteBuffer(kPayload)));
}
TEST(VideoRtpDepacketizerH265Test, ShortSpsPacket) {
const uint8_t kPayload[] = {0x40, 0x80, 0x00};
VideoRtpDepacketizerH265 depacketizer;
EXPECT_TRUE(depacketizer.Parse(rtc::CopyOnWriteBuffer(kPayload)));
}
TEST(VideoRtpDepacketizerH265Test, InvalidNaluSizeApNalu) {
const uint8_t kPayload[] = {0x60, 0x02, // F=0, Type=48 (kH265Ap).
// Length, nal header, payload.
0, 0xff, 0x02, 0x02, 0xFF, // TrailR
0, 0x05, 0x02, 0x02, 0xFF, 0x00,
0x11}; // TrailR;
VideoRtpDepacketizerH265 depacketizer;
EXPECT_FALSE(depacketizer.Parse(rtc::CopyOnWriteBuffer(kPayload)));
}
TEST(VideoRtpDepacketizerH265Test, SeiPacket) {
const uint8_t kPayload[] = {
0x4e, 0x02, // F=0, Type=39 (kPrefixSei).
0x03, 0x03, 0x03, 0x03 // Payload.
};
VideoRtpDepacketizerH265 depacketizer;
auto parsed = depacketizer.Parse(rtc::CopyOnWriteBuffer(kPayload));
ASSERT_TRUE(parsed);
}
} // namespace
} // namespace webrtc

5
test/fuzzers/BUILD.gn
View File

@ -132,6 +132,11 @@ if (rtc_use_h265) {
"../../modules/video_coding/",
]
}
webrtc_fuzzer_test("h265_depacketizer_fuzzer") {
sources = [ "h265_depacketizer_fuzzer.cc" ]
deps = [ "../../modules/rtp_rtcp" ]
}
}
webrtc_fuzzer_test("forward_error_correction_fuzzer") {

19
test/fuzzers/h265_depacketizer_fuzzer.cc Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_h265.h"
namespace webrtc {
void FuzzOneInput(const uint8_t* data, size_t size) {
if (size > 200000)
return;
VideoRtpDepacketizerH265 depacketizer;
depacketizer.Parse(rtc::CopyOnWriteBuffer(data, size));
}
} // namespace webrtc

1
video/rtp_video_stream_receiver2.cc
View File

@ -803,6 +803,7 @@ void RtpVideoStreamReceiver2::OnInsertedPacket(
if (packet->is_last_packet_in_frame()) {
auto depacketizer_it = payload_type_map_.find(first_packet->payload_type);
RTC_CHECK(depacketizer_it != payload_type_map_.end());
RTC_CHECK(depacketizer_it->second);
rtc::scoped_refptr<EncodedImageBuffer> bitstream =
depacketizer_it->second->AssembleFrame(payloads);