admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add rtp packetizer for H265

Browse Source 
Bug: webrtc:13485
Change-Id: I4e7e29a7661d51e12bb2ee12e319f6cef49482d4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/318005
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41107}
This commit is contained in:
qwu16 2023-10-26 16:52:22 +08:00 committed by WebRTC LUCI CQ
parent 916ec25eed
commit bd396fdffa
7 changed files with 964 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
common_video/h265/h265_bitstream_parser.cc
View File

@ -475,8 +475,8 @@ void H265BitstreamParser::ParseSlice(const uint8_t* slice, size_t length) {
case H265::NaluType::kAud:
case H265::NaluType::kPrefixSei:
case H265::NaluType::kSuffixSei:
case H265::NaluType::kAP:
case H265::NaluType::kFU:
case H265::NaluType::kAp:
case H265::NaluType::kFu:
break;
default:
Result res = ParseNonParameterSetNalu(slice, length, nalu_type);

4
common_video/h265/h265_common.h
View File

@ -55,8 +55,8 @@ enum NaluType : uint8_t {
kAud = 35,
kPrefixSei = 39,
kSuffixSei = 40,
kAP = 48,
kFU = 49
kAp = 48,
kFu = 49
};
// Slice type definition. See table 7-7 of the H265 spec

11
modules/rtp_rtcp/BUILD.gn
View File

@ -258,6 +258,13 @@ rtc_library("rtp_rtcp") {
"source/video_rtp_depacketizer_vp9.h",
]
if (rtc_use_h265) {
sources += [
"source/rtp_packetizer_h265.cc",
"source/rtp_packetizer_h265.h",
]
}
if (rtc_enable_bwe_test_logging) {
defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=1" ]
} else {
@ -624,6 +631,10 @@ if (rtc_include_tests) {
"source/video_rtp_depacketizer_vp8_unittest.cc",
"source/video_rtp_depacketizer_vp9_unittest.cc",
]
if (rtc_use_h265) {
sources += [ "source/rtp_packetizer_h265_unittest.cc" ]
}
deps = [
":fec_test_helper",
":frame_transformer_factory_unittest",

9
modules/rtp_rtcp/source/rtp_format.cc
View File

@ -22,6 +22,9 @@
#include "modules/video_coding/codecs/vp8/include/vp8_globals.h"
#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
#include "rtc_base/checks.h"
#ifdef RTC_ENABLE_H265
#include "modules/rtp_rtcp/source/rtp_packetizer_h265.h"
#endif
namespace webrtc {
@ -57,7 +60,11 @@ std::unique_ptr<RtpPacketizer> RtpPacketizer::Create(
return std::make_unique<RtpPacketizerAv1>(
payload, limits, rtp_video_header.frame_type,
rtp_video_header.is_last_frame_in_picture);
// TODO(bugs.webrtc.org/13485): Implement RtpPacketizerH265.
#ifdef RTC_ENABLE_H265
case kVideoCodecH265: {
return std::make_unique<RtpPacketizerH265>(payload, limits);
}
#endif
default: {
return std::make_unique<RtpPacketizerGeneric>(payload, limits,
rtp_video_header);

350
modules/rtp_rtcp/source/rtp_packetizer_h265.cc Normal file
View File

@ -0,0 +1,350 @@
/*
* Copyright (c) 2023 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/rtp_packetizer_h265.h"
#include <vector>
#include "absl/types/optional.h"
#include "common_video/h264/h264_common.h"
#include "common_video/h265/h265_common.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "rtc_base/logging.h"
namespace webrtc {
namespace {
// The payload header consists of the same
// fields (F, Type, LayerId and TID) as the NAL unit header. Refer to
// section 4.2 in RFC 7798.
constexpr size_t kH265PayloadHeaderSize = 2;
// Unlike H.264, H265 NAL header is 2-bytes.
constexpr size_t kH265NalHeaderSize = 2;
// H265's FU is constructed of 2-byte payload header, 1-byte FU header and FU
// payload.
constexpr size_t kH265FuHeaderSize = 1;
// The NALU size for H265 RTP aggregated packet indicates the size of the NAL
// unit is 2-bytes.
constexpr size_t kH265LengthFieldSize = 2;
enum H265NalHdrMasks {
kH265FBit = 0x80,
kH265TypeMask = 0x7E,
kH265LayerIDHMask = 0x1,
kH265LayerIDLMask = 0xF8,
kH265TIDMask = 0x7,
kH265TypeMaskN = 0x81,
kH265TypeMaskInFuHeader = 0x3F
};
// Bit masks for FU headers.
enum H265FuBitmasks {
kH265SBitMask = 0x80,
kH265EBitMask = 0x40,
kH265FuTypeBitMask = 0x3F
};
} // namespace
RtpPacketizerH265::RtpPacketizerH265(rtc::ArrayView<const uint8_t> payload,
PayloadSizeLimits limits)
: limits_(limits), num_packets_left_(0) {
for (const auto& nalu :
H264::FindNaluIndices(payload.data(), payload.size())) {
input_fragments_.push_back(
payload.subview(nalu.payload_start_offset, nalu.payload_size));
}
if (!GeneratePackets()) {
// If failed to generate all the packets, discard already generated
// packets in case the caller would ignore return value and still try to
// call NextPacket().
num_packets_left_ = 0;
while (!packets_.empty()) {
packets_.pop();
}
}
}
RtpPacketizerH265::~RtpPacketizerH265() = default;
size_t RtpPacketizerH265::NumPackets() const {
return num_packets_left_;
}
bool RtpPacketizerH265::GeneratePackets() {
for (size_t i = 0; i < input_fragments_.size();) {
int fragment_len = input_fragments_[i].size();
int single_packet_capacity = limits_.max_payload_len;
if (input_fragments_.size() == 1) {
single_packet_capacity -= limits_.single_packet_reduction_len;
} else if (i == 0) {
single_packet_capacity -= limits_.first_packet_reduction_len;
} else if (i + 1 == input_fragments_.size()) {
// Pretend that last fragment is larger instead of making last packet
// smaller.
single_packet_capacity -= limits_.last_packet_reduction_len;
}
if (fragment_len > single_packet_capacity) {
if (!PacketizeFu(i)) {
return false;
}
++i;
} else {
i = PacketizeAp(i);
}
}
return true;
}
bool RtpPacketizerH265::PacketizeFu(size_t fragment_index) {
// Fragment payload into packets (FU).
// Strip out the original header and leave room for the FU header.
rtc::ArrayView<const uint8_t> fragment = input_fragments_[fragment_index];
PayloadSizeLimits limits = limits_;
// Refer to section 4.4.3 in RFC7798, each FU fragment will have a 2-bytes
// payload header and a one-byte FU header. DONL is not supported so ignore
// its size when calculating max_payload_len.
limits.max_payload_len -= kH265FuHeaderSize + kH265PayloadHeaderSize;
// Update single/first/last packet reductions unless it is single/first/last
// fragment.
if (input_fragments_.size() != 1) {
// if this fragment is put into a single packet, it might still be the
// first or the last packet in the whole sequence of packets.
if (fragment_index == input_fragments_.size() - 1) {
limits.single_packet_reduction_len = limits_.last_packet_reduction_len;
} else if (fragment_index == 0) {
limits.single_packet_reduction_len = limits_.first_packet_reduction_len;
} else {
limits.single_packet_reduction_len = 0;
}
}
if (fragment_index != 0) {
limits.first_packet_reduction_len = 0;
}
if (fragment_index != input_fragments_.size() - 1) {
limits.last_packet_reduction_len = 0;
}
// Strip out the original header.
size_t payload_left = fragment.size() - kH265NalHeaderSize;
int offset = kH265NalHeaderSize;
std::vector<int> payload_sizes = SplitAboutEqually(payload_left, limits);
if (payload_sizes.empty()) {
return false;
}
for (size_t i = 0; i < payload_sizes.size(); ++i) {
int packet_length = payload_sizes[i];
RTC_CHECK_GT(packet_length, 0);
uint16_t header = (fragment[0] << 8) | fragment[1];
packets_.push({.source_fragment = fragment.subview(offset, packet_length),
.first_fragment = (i == 0),
.last_fragment = (i == payload_sizes.size() - 1),
.aggregated = false,
.header = header});
offset += packet_length;
payload_left -= packet_length;
}
num_packets_left_ += payload_sizes.size();
RTC_CHECK_EQ(payload_left, 0);
return true;
}
int RtpPacketizerH265::PacketizeAp(size_t fragment_index) {
// Aggregate fragments into one packet.
size_t payload_size_left = limits_.max_payload_len;
if (input_fragments_.size() == 1) {
payload_size_left -= limits_.single_packet_reduction_len;
} else if (fragment_index == 0) {
payload_size_left -= limits_.first_packet_reduction_len;
}
int aggregated_fragments = 0;
size_t fragment_headers_length = 0;
rtc::ArrayView<const uint8_t> fragment = input_fragments_[fragment_index];
RTC_CHECK_GE(payload_size_left, fragment.size());
++num_packets_left_;
auto payload_size_needed = [&] {
size_t fragment_size = fragment.size() + fragment_headers_length;
if (input_fragments_.size() == 1) {
// Single fragment, single packet, payload_size_left already adjusted
// with limits_.single_packet_reduction_len.
return fragment_size;
}
if (fragment_index == input_fragments_.size() - 1) {
// Last fragment, so this might be the last packet.
return fragment_size + limits_.last_packet_reduction_len;
}
return fragment_size;
};
while (payload_size_left >= payload_size_needed()) {
RTC_CHECK_GT(fragment.size(), 0);
packets_.push({.source_fragment = fragment,
.first_fragment = (aggregated_fragments == 0),
.last_fragment = false,
.aggregated = true,
.header = fragment[0]});
payload_size_left -= fragment.size();
payload_size_left -= fragment_headers_length;
fragment_headers_length = kH265LengthFieldSize;
// If we are going to try to aggregate more fragments into this packet
// we need to add the AP NALU header and a length field for the first
// NALU of this packet.
if (aggregated_fragments == 0) {
fragment_headers_length += kH265PayloadHeaderSize + kH265LengthFieldSize;
}
++aggregated_fragments;
// Next fragment.
++fragment_index;
if (fragment_index == input_fragments_.size()) {
break;
}
fragment = input_fragments_[fragment_index];
}
RTC_CHECK_GT(aggregated_fragments, 0);
packets_.back().last_fragment = true;
return fragment_index;
}
bool RtpPacketizerH265::NextPacket(RtpPacketToSend* rtp_packet) {
RTC_DCHECK(rtp_packet);
if (packets_.empty()) {
return false;
}
PacketUnit packet = packets_.front();
if (packet.first_fragment && packet.last_fragment) {
// Single NAL unit packet. Do not support DONL for single NAL unit packets,
// DONL field is not present.
size_t bytes_to_send = packet.source_fragment.size();
uint8_t* buffer = rtp_packet->AllocatePayload(bytes_to_send);
memcpy(buffer, packet.source_fragment.data(), bytes_to_send);
packets_.pop();
input_fragments_.pop_front();
} else if (packet.aggregated) {
NextAggregatePacket(rtp_packet);
} else {
NextFragmentPacket(rtp_packet);
}
rtp_packet->SetMarker(packets_.empty());
--num_packets_left_;
return true;
}
void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) {
size_t payload_capacity = rtp_packet->FreeCapacity();
RTC_CHECK_GE(payload_capacity, kH265PayloadHeaderSize);
uint8_t* buffer = rtp_packet->AllocatePayload(payload_capacity);
RTC_CHECK(buffer);
PacketUnit* packet = &packets_.front();
RTC_CHECK(packet->first_fragment);
/*
+---------------+---------------+
|0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7|
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|F| Type | LayerId | TID |
+-------------+-----------------+
*/
// Refer to section section 4.4.2 for aggregation packets and modify type to
// 48 in PayloadHdr for aggregate packet. Do not support DONL for aggregation
// packets, DONL field is not present.
uint8_t payload_hdr_h = packet->header >> 8;
uint8_t payload_hdr_l = packet->header & 0xFF;
uint8_t layer_id_h = payload_hdr_h & kH265LayerIDHMask;
payload_hdr_h = (payload_hdr_h & kH265TypeMaskN) |
(H265::NaluType::kAp << 1) | layer_id_h;
buffer[0] = payload_hdr_h;
buffer[1] = payload_hdr_l;
int index = kH265PayloadHeaderSize;
bool is_last_fragment = packet->last_fragment;
while (packet->aggregated) {
// Add NAL unit length field.
rtc::ArrayView<const uint8_t> fragment = packet->source_fragment;
ByteWriter<uint16_t>::WriteBigEndian(&buffer[index], fragment.size());
index += kH265LengthFieldSize;
// Add NAL unit.
memcpy(&buffer[index], fragment.data(), fragment.size());
index += fragment.size();
packets_.pop();
input_fragments_.pop_front();
if (is_last_fragment) {
break;
}
packet = &packets_.front();
is_last_fragment = packet->last_fragment;
}
RTC_CHECK(is_last_fragment);
rtp_packet->SetPayloadSize(index);
}
void RtpPacketizerH265::NextFragmentPacket(RtpPacketToSend* rtp_packet) {
PacketUnit* packet = &packets_.front();
// NAL unit fragmented over multiple packets (FU).
// We do not send original NALU header, so it will be replaced by the
// PayloadHdr of the first packet.
/*
+---------------+---------------+
|0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7|
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|F| Type | LayerId | TID |
+-------------+-----------------+
*/
// Refer to section section 4.4.3 for aggregation packets and modify type to
// 49 in PayloadHdr for aggregate packet.
uint8_t payload_hdr_h =
packet->header >> 8; // 1-bit F, 6-bit type, 1-bit layerID highest-bit
uint8_t payload_hdr_l = packet->header & 0xFF;
uint8_t layer_id_h = payload_hdr_h & kH265LayerIDHMask;
uint8_t fu_header = 0;
/*
+---------------+
|0|1|2|3|4|5|6|7|
+-+-+-+-+-+-+-+-+
|S|E| FuType |
+---------------+
*/
// S bit indicates the start of a fragmented NAL unit.
// E bit indicates the end of a fragmented NAL unit.
// FuType must be equal to the field type value of the fragmented NAL unit.
fu_header |= (packet->first_fragment ? kH265SBitMask : 0);
fu_header |= (packet->last_fragment ? kH265EBitMask : 0);
uint8_t type = (payload_hdr_h & kH265TypeMask) >> 1;
fu_header |= type;
// Now update payload_hdr_h with FU type.
payload_hdr_h = (payload_hdr_h & kH265TypeMaskN) |
(H265::NaluType::kFu << 1) | layer_id_h;
rtc::ArrayView<const uint8_t> fragment = packet->source_fragment;
uint8_t* buffer = rtp_packet->AllocatePayload(
kH265FuHeaderSize + kH265PayloadHeaderSize + fragment.size());
RTC_CHECK(buffer);
buffer[0] = payload_hdr_h;
buffer[1] = payload_hdr_l;
buffer[2] = fu_header;
// Do not support DONL for fragmentation units, DONL field is not present.
memcpy(buffer + kH265FuHeaderSize + kH265PayloadHeaderSize, fragment.data(),
fragment.size());
if (packet->last_fragment) {
input_fragments_.pop_front();
}
packets_.pop();
}
} // namespace webrtc

66
modules/rtp_rtcp/source/rtp_packetizer_h265.h Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright (c) 2023 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_RTP_RTCP_SOURCE_RTP_PACKETIZER_H265_H_
#define MODULES_RTP_RTCP_SOURCE_RTP_PACKETIZER_H265_H_
#include <deque>
#include <queue>
#include <string>
#include "api/array_view.h"
#include "modules/rtp_rtcp/source/rtp_format.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
namespace webrtc {
class RtpPacketizerH265 : public RtpPacketizer {
public:
// Initialize with payload from encoder.
// The payload_data must be exactly one encoded H.265 frame.
// For H265 we only support tx-mode SRST.
RtpPacketizerH265(rtc::ArrayView<const uint8_t> payload,
PayloadSizeLimits limits);
RtpPacketizerH265(const RtpPacketizerH265&) = delete;
RtpPacketizerH265& operator=(const RtpPacketizerH265&) = delete;
~RtpPacketizerH265() override;
size_t NumPackets() const override;
// Get the next payload with H.265 payload header.
// Write payload and set marker bit of the `packet`.
// Returns true on success or false if there was no payload to packetize.
bool NextPacket(RtpPacketToSend* rtp_packet) override;
private:
struct PacketUnit {
rtc::ArrayView<const uint8_t> source_fragment;
bool first_fragment = false;
bool last_fragment = false;
bool aggregated = false;
uint16_t header = 0;
};
std::deque<rtc::ArrayView<const uint8_t>> input_fragments_;
std::queue<PacketUnit> packets_;
bool GeneratePackets();
bool PacketizeFu(size_t fragment_index);
int PacketizeAp(size_t fragment_index);
void NextAggregatePacket(RtpPacketToSend* rtp_packet);
void NextFragmentPacket(RtpPacketToSend* rtp_packet);
const PayloadSizeLimits limits_;
size_t num_packets_left_ = 0;
};
} // namespace webrtc
#endif // MODULES_RTP_RTCP_SOURCE_RTP_PACKETIZER_H265_H_

525
modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc Normal file
View File

@ -0,0 +1,525 @@
/*
* Copyright (c) 2023 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/rtp_packetizer_h265.h"
#include <vector>
#include "common_video/h265/h265_common.h"
#include "modules/rtp_rtcp/mocks/mock_rtp_rtcp.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
using ::testing::Each;
using ::testing::ElementsAre;
using ::testing::ElementsAreArray;
using ::testing::Eq;
using ::testing::IsEmpty;
using ::testing::SizeIs;
constexpr RtpPacketToSend::ExtensionManager* kNoExtensions = nullptr;
constexpr size_t kMaxPayloadSize = 1200;
constexpr size_t kLengthFieldLength = 2;
constexpr RtpPacketizer::PayloadSizeLimits kNoLimits;
constexpr size_t kNalHeaderSize = 2;
constexpr size_t kFuHeaderSize = 3;
constexpr uint8_t kNaluTypeMask = 0x7E;
// Bit masks for FU headers.
constexpr uint8_t kH265SBit = 0x80;
constexpr uint8_t kH265EBit = 0x40;
// Creates Buffer that looks like nal unit of given size.
rtc::Buffer GenerateNalUnit(size_t size) {
RTC_CHECK_GT(size, 0);
rtc::Buffer buffer(size);
// Set some valid header with type TRAIL_R and temporal id
buffer[0] = 2;
buffer[1] = 2;
for (size_t i = 2; i < size; ++i) {
buffer[i] = static_cast<uint8_t>(i);
}
// Last byte shouldn't be 0, or it may be counted as part of next 4-byte start
// sequence.
buffer[size - 1] |= 0x10;
return buffer;
}
// Create frame consisting of nalus of given size.
rtc::Buffer CreateFrame(std::initializer_list<size_t> nalu_sizes) {
static constexpr int kStartCodeSize = 3;
rtc::Buffer frame(absl::c_accumulate(nalu_sizes, size_t{0}) +
kStartCodeSize * nalu_sizes.size());
size_t offset = 0;
for (size_t nalu_size : nalu_sizes) {
EXPECT_GE(nalu_size, 1u);
// Insert nalu start code
frame[offset] = 0;
frame[offset + 1] = 0;
frame[offset + 2] = 1;
// Set some valid header.
frame[offset + 3] = 2;
// Fill payload avoiding accidental start codes
if (nalu_size > 1) {
memset(frame.data() + offset + 4, 0x3f, nalu_size - 1);
}
offset += (kStartCodeSize + nalu_size);
}
return frame;
}
// Create frame consisting of given nalus.
rtc::Buffer CreateFrame(rtc::ArrayView<const rtc::Buffer> nalus) {
static constexpr int kStartCodeSize = 3;
int frame_size = 0;
for (const rtc::Buffer& nalu : nalus) {
frame_size += (kStartCodeSize + nalu.size());
}
rtc::Buffer frame(frame_size);
size_t offset = 0;
for (const rtc::Buffer& nalu : nalus) {
// Insert nalu start code
frame[offset] = 0;
frame[offset + 1] = 0;
frame[offset + 2] = 1;
// Copy the nalu unit.
memcpy(frame.data() + offset + 3, nalu.data(), nalu.size());
offset += (kStartCodeSize + nalu.size());
}
return frame;
}
std::vector<RtpPacketToSend> FetchAllPackets(RtpPacketizerH265* packetizer) {
std::vector<RtpPacketToSend> result;
size_t num_packets = packetizer->NumPackets();
result.reserve(num_packets);
RtpPacketToSend packet(kNoExtensions);
while (packetizer->NextPacket(&packet)) {
result.push_back(packet);
}
EXPECT_THAT(result, SizeIs(num_packets));
return result;
}
// Single nalu tests.
TEST(RtpPacketizerH265Test, SingleNalu) {
const uint8_t frame[] = {0, 0, 1, H265::kIdrWRadl, 0xFF};
RtpPacketizerH265 packetizer(frame, kNoLimits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(1));
EXPECT_THAT(packets[0].payload(), ElementsAre(H265::kIdrWRadl, 0xFF));
}
TEST(RtpPacketizerH265Test, SingleNaluTwoPackets) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = kMaxPayloadSize;
rtc::Buffer nalus[] = {GenerateNalUnit(kMaxPayloadSize),
GenerateNalUnit(100)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(2));
EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0]));
EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[1]));
}
TEST(RtpPacketizerH265Test,
SingleNaluFirstPacketReductionAppliesOnlyToFirstFragment) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 200;
limits.first_packet_reduction_len = 5;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/195),
GenerateNalUnit(/*size=*/200),
GenerateNalUnit(/*size=*/200)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(3));
EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0]));
EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[1]));
EXPECT_THAT(packets[2].payload(), ElementsAreArray(nalus[2]));
}
TEST(RtpPacketizerH265Test,
SingleNaluLastPacketReductionAppliesOnlyToLastFragment) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 200;
limits.last_packet_reduction_len = 5;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/200),
GenerateNalUnit(/*size=*/200),
GenerateNalUnit(/*size=*/195)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(3));
EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0]));
EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[1]));
EXPECT_THAT(packets[2].payload(), ElementsAreArray(nalus[2]));
}
TEST(RtpPacketizerH265Test,
SingleNaluFirstAndLastPacketReductionSumsForSinglePacket) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 200;
limits.first_packet_reduction_len = 20;
limits.last_packet_reduction_len = 30;
rtc::Buffer frame = CreateFrame({150});
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
EXPECT_THAT(packets, SizeIs(1));
}
// Aggregation tests.
TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) {
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/0x123)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, kNoLimits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(1));
auto payload = packets[0].payload();
int type = H265::ParseNaluType(payload[0]);
EXPECT_EQ(payload.size(),
kNalHeaderSize + 3 * kLengthFieldLength + 2 + 2 + 0x123);
EXPECT_EQ(type, H265::NaluType::kAp);
payload = payload.subview(kNalHeaderSize);
// 1st fragment.
EXPECT_THAT(payload.subview(0, kLengthFieldLength),
ElementsAre(0, 2)); // Size.
EXPECT_THAT(payload.subview(kLengthFieldLength, 2),
ElementsAreArray(nalus[0]));
payload = payload.subview(kLengthFieldLength + 2);
// 2nd fragment.
EXPECT_THAT(payload.subview(0, kLengthFieldLength),
ElementsAre(0, 2)); // Size.
EXPECT_THAT(payload.subview(kLengthFieldLength, 2),
ElementsAreArray(nalus[1]));
payload = payload.subview(kLengthFieldLength + 2);
// 3rd fragment.
EXPECT_THAT(payload.subview(0, kLengthFieldLength),
ElementsAre(0x1, 0x23)); // Size.
EXPECT_THAT(payload.subview(kLengthFieldLength), ElementsAreArray(nalus[2]));
}
TEST(RtpPacketizerH265Test, ApRespectsFirstPacketReduction) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1000;
limits.first_packet_reduction_len = 100;
const size_t kFirstFragmentSize =
limits.max_payload_len - limits.first_packet_reduction_len;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/kFirstFragmentSize),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(2));
// Expect 1st packet is single nalu.
EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0]));
// Expect 2nd packet is aggregate of last two fragments.
// The size of H265 nal_unit_header is 2 bytes, according to 7.3.1.2
// in H265 spec. Aggregation packet type is 48, and nuh_temporal_id_plus1
// is 2, so the nal_unit_header should be "01100000 00000010",
// which is 96 and 2.
EXPECT_THAT(packets[1].payload(),
ElementsAre(96, 2, //
0, 2, nalus[1][0], nalus[1][1], //
0, 2, nalus[2][0], nalus[2][1]));
}
TEST(RtpPacketizerH265Test, ApRespectsLastPacketReduction) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1000;
limits.last_packet_reduction_len = 100;
const size_t kLastFragmentSize =
limits.max_payload_len - limits.last_packet_reduction_len;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/kLastFragmentSize)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(2));
// Expect 1st packet is aggregate of 1st two fragments.
EXPECT_THAT(packets[0].payload(),
ElementsAre(96, 2, //
0, 2, nalus[0][0], nalus[0][1], //
0, 2, nalus[1][0], nalus[1][1]));
// Expect 2nd packet is single nalu.
EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[2]));
}
TEST(RtpPacketizerH265Test, TooSmallForApHeaders) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1000;
const size_t kLastFragmentSize =
limits.max_payload_len - 3 * kLengthFieldLength - 4;
rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/2),
GenerateNalUnit(/*size=*/kLastFragmentSize)};
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(2));
// Expect 1st packet is aggregate of 1st two fragments.
EXPECT_THAT(packets[0].payload(),
ElementsAre(96, 2, //
0, 2, nalus[0][0], nalus[0][1], //
0, 2, nalus[1][0], nalus[1][1]));
// Expect 2nd packet is single nalu.
EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[2]));
}
TEST(RtpPacketizerH265Test, LastFragmentFitsInSingleButNotLastPacket) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1178;
limits.first_packet_reduction_len = 0;
limits.last_packet_reduction_len = 20;
limits.single_packet_reduction_len = 20;
// Actual sizes, which triggered this bug.
rtc::Buffer frame = CreateFrame({20, 8, 18, 1161});
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
// Last packet has to be of correct size.
// Incorrect implementation might miss this constraint and not split the last
// fragment in two packets.
EXPECT_LE(static_cast<int>(packets.back().payload_size()),
limits.max_payload_len - limits.last_packet_reduction_len);
}
// Splits frame with payload size `frame_payload_size` without fragmentation,
// Returns sizes of the payloads excluding FU headers.
std::vector<int> TestFu(size_t frame_payload_size,
const RtpPacketizer::PayloadSizeLimits& limits) {
rtc::Buffer nalu[] = {GenerateNalUnit(kNalHeaderSize + frame_payload_size)};
rtc::Buffer frame = CreateFrame(nalu);
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
EXPECT_GE(packets.size(), 2u); // Single packet indicates it is not FU.
std::vector<uint16_t> fu_header;
std::vector<int> payload_sizes;
for (const RtpPacketToSend& packet : packets) {
auto payload = packet.payload();
EXPECT_GT(payload.size(), kFuHeaderSize);
// FU header is after the 2-bytes size PayloadHdr according to 4.4.3 in spec
fu_header.push_back(payload[2]);
payload_sizes.push_back(payload.size() - kFuHeaderSize);
}
EXPECT_TRUE(fu_header.front() & kH265SBit);
EXPECT_TRUE(fu_header.back() & kH265EBit);
// Clear S and E bits before testing all are duplicating same original header.
fu_header.front() &= ~kH265SBit;
fu_header.back() &= ~kH265EBit;
uint8_t nalu_type = (nalu[0][0] & kNaluTypeMask) >> 1;
EXPECT_THAT(fu_header, Each(Eq(nalu_type)));
return payload_sizes;
}
// Fragmentation tests.
TEST(RtpPacketizerH265Test, FuOddSize) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1200;
EXPECT_THAT(TestFu(1200, limits), ElementsAre(600, 600));
}
TEST(RtpPacketizerH265Test, FuWithFirstPacketReduction) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1200;
limits.first_packet_reduction_len = 4;
limits.single_packet_reduction_len = 4;
EXPECT_THAT(TestFu(1198, limits), ElementsAre(597, 601));
}
TEST(RtpPacketizerH265Test, FuWithLastPacketReduction) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1200;
limits.last_packet_reduction_len = 4;
limits.single_packet_reduction_len = 4;
EXPECT_THAT(TestFu(1198, limits), ElementsAre(601, 597));
}
TEST(RtpPacketizerH265Test, FuWithSinglePacketReduction) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1199;
limits.single_packet_reduction_len = 200;
EXPECT_THAT(TestFu(1000, limits), ElementsAre(500, 500));
}
TEST(RtpPacketizerH265Test, FuEvenSize) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1200;
EXPECT_THAT(TestFu(1201, limits), ElementsAre(600, 601));
}
TEST(RtpPacketizerH265Test, FuRounding) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1448;
EXPECT_THAT(TestFu(10123, limits),
ElementsAre(1265, 1265, 1265, 1265, 1265, 1266, 1266, 1266));
}
TEST(RtpPacketizerH265Test, FuBig) {
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = 1200;
// Generate 10 full sized packets, leave room for FU headers.
EXPECT_THAT(
TestFu(10 * (1200 - kFuHeaderSize), limits),
ElementsAre(1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197));
}
struct PacketInfo {
bool first_fragment = false;
bool last_fragment = false;
bool aggregated = false;
int nalu_index = 0;
int nalu_number = 0;
int payload_size = 0;
int start_offset = 0;
};
struct MixedApFuTestParams {
std::vector<int> nalus;
int expect_packetsSize = 0;
std::vector<PacketInfo> expected_packets;
};
class RtpPacketizerH265ParametrizedTest
: public ::testing::TestWithParam<MixedApFuTestParams> {};
// Fragmentation + aggregation mixed testing.
TEST_P(RtpPacketizerH265ParametrizedTest, MixedApFu) {
RtpPacketizer::PayloadSizeLimits limits;
const MixedApFuTestParams params = GetParam();
limits.max_payload_len = 100;
std::vector<rtc::Buffer> nalus;
nalus.reserve(params.nalus.size());
// Generate nalus according to size specified in paramters
for (size_t index = 0; index < params.nalus.size(); index++) {
nalus.push_back(GenerateNalUnit(params.nalus[index]));
}
rtc::Buffer frame = CreateFrame(nalus);
RtpPacketizerH265 packetizer(frame, limits);
std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer);
ASSERT_THAT(packets, SizeIs(params.expect_packetsSize));
for (int i = 0; i < params.expect_packetsSize; i++) {
PacketInfo expected_packet = params.expected_packets[i];
if (expected_packet.aggregated) {
int type = H265::ParseNaluType(packets[i].payload()[0]);
EXPECT_THAT(type, H265::NaluType::kAp);
auto payload = packets[i].payload().subview(kNalHeaderSize);
int offset = 0;
// Generated AP packet header and payload align
for (int j = expected_packet.nalu_index; j < expected_packet.nalu_number;
j++) {
EXPECT_THAT(payload.subview(0, kLengthFieldLength),
ElementsAre(0, nalus[j].size()));
EXPECT_THAT(
payload.subview(offset + kLengthFieldLength, nalus[j].size()),
ElementsAreArray(nalus[j]));
offset += kLengthFieldLength + nalus[j].size();
}
} else {
uint8_t fu_header = 0;
fu_header |= (expected_packet.first_fragment ? kH265SBit : 0);
fu_header |= (expected_packet.last_fragment ? kH265EBit : 0);
fu_header |= H265::NaluType::kTrailR;
EXPECT_THAT(packets[i].payload().subview(0, kFuHeaderSize),
ElementsAre(98, 2, fu_header));
EXPECT_THAT(
packets[i].payload().subview(kFuHeaderSize),
ElementsAreArray(nalus[expected_packet.nalu_index].data() +
kNalHeaderSize + expected_packet.start_offset,
expected_packet.payload_size));
}
}
}
INSTANTIATE_TEST_SUITE_P(
RtpPacketizerH265Test,
RtpPacketizerH265ParametrizedTest,
testing::Values(
// FU + AP + FU.
// GenerateNalUnit will include 2 bytes nalu header, for FU packet split
// calculation, this 2-byte nalu header length should be excluded.
MixedApFuTestParams{.nalus = {140, 20, 20, 160},
.expect_packetsSize = 5,
.expected_packets = {{.first_fragment = true,
.nalu_index = 0,
.payload_size = 69,
.start_offset = 0},
{.last_fragment = true,
.nalu_index = 0,
.payload_size = 69,
.start_offset = 69},
{.aggregated = true,
.nalu_index = 1,
.nalu_number = 2},
{.first_fragment = true,
.nalu_index = 3,
.payload_size = 79,
.start_offset = 0},
{.last_fragment = true,
.nalu_index = 3,
.payload_size = 79,
.start_offset = 79}}},
// AP + FU + AP
MixedApFuTestParams{
.nalus = {20, 20, 160, 30, 30},
.expect_packetsSize = 4,
.expected_packets = {
{.aggregated = true, .nalu_index = 0, .nalu_number = 2},
{.first_fragment = true,
.nalu_index = 2,
.payload_size = 79,
.start_offset = 0},
{.last_fragment = true,
.nalu_index = 2,
.payload_size = 79,
.start_offset = 79},
{.aggregated = true, .nalu_index = 3, .nalu_number = 2}}}));
} // namespace
} // namespace webrtc