admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add HEVC support for h264_packet_buffer.

Browse Source 
Renamed to h26x_packet_buffer as it also supports HEVC now. For HEVC,
start code is added by depacktizer, and remote endpoint must send
sequence and picture information in-band.

Co-authored-by: Qiujiao Wu <qiujiao.wu@intel.com>

Bug: webrtc:13485
Change-Id: I321cb223357d8d15da95cec80ec0c3a43c26734e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/333863
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41739}
This commit is contained in:
Jianjun Zhu 2024-02-08 16:59:18 +08:00 committed by WebRTC LUCI CQ
parent 4efc830e53
commit a2655449ee
5 changed files with 1157 additions and 828 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/video_coding/BUILD.gn
View File

@ -124,10 +124,10 @@ rtc_library("packet_buffer") {
]
}
rtc_library("h264_packet_buffer") {
rtc_library("h26x_packet_buffer") {
sources = [
"h264_packet_buffer.cc",
"h264_packet_buffer.h",
"h26x_packet_buffer.cc",
"h26x_packet_buffer.h",
]
deps = [
":codec_globals_headers",
@ -1164,9 +1164,9 @@ if (rtc_include_tests) {
"frame_dependencies_calculator_unittest.cc",
"frame_helpers_unittest.cc",
"generic_decoder_unittest.cc",
"h264_packet_buffer_unittest.cc",
"h264_sprop_parameter_sets_unittest.cc",
"h264_sps_pps_tracker_unittest.cc",
"h26x_packet_buffer_unittest.cc",
"histogram_unittest.cc",
"loss_notification_controller_unittest.cc",
"nack_requester_unittest.cc",
@ -1201,7 +1201,7 @@ if (rtc_include_tests) {
":encoded_frame",
":frame_dependencies_calculator",
":frame_helpers",
":h264_packet_buffer",
":h26x_packet_buffer",
":nack_requester",
":packet_buffer",
":simulcast_test_fixture_impl",

778
modules/video_coding/h264_packet_buffer_unittest.cc
View File

@ -1,778 +0,0 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/h264_packet_buffer.h"
#include <cstring>
#include <limits>
#include <ostream>
#include <string>
#include <utility>
#include "api/array_view.h"
#include "api/video/render_resolution.h"
#include "common_video/h264/h264_common.h"
#include "rtc_base/system/unused.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
using ::testing::ElementsAreArray;
using ::testing::Eq;
using ::testing::IsEmpty;
using ::testing::SizeIs;
using H264::NaluType::kAud;
using H264::NaluType::kFuA;
using H264::NaluType::kIdr;
using H264::NaluType::kPps;
using H264::NaluType::kSlice;
using H264::NaluType::kSps;
using H264::NaluType::kStapA;
constexpr int kBufferSize = 2048;
std::vector<uint8_t> StartCode() {
return {0, 0, 0, 1};
}
NaluInfo MakeNaluInfo(uint8_t type) {
NaluInfo res;
res.type = type;
res.sps_id = -1;
res.pps_id = -1;
return res;
}
class Packet {
public:
explicit Packet(H264PacketizationTypes type);
Packet& Idr(std::vector<uint8_t> payload = {9, 9, 9});
Packet& Slice(std::vector<uint8_t> payload = {9, 9, 9});
Packet& Sps(std::vector<uint8_t> payload = {9, 9, 9});
Packet& SpsWithResolution(RenderResolution resolution,
std::vector<uint8_t> payload = {9, 9, 9});
Packet& Pps(std::vector<uint8_t> payload = {9, 9, 9});
Packet& Aud();
Packet& Marker();
Packet& AsFirstFragment();
Packet& Time(uint32_t rtp_timestamp);
Packet& SeqNum(uint16_t rtp_seq_num);
std::unique_ptr<H264PacketBuffer::Packet> Build();
private:
rtc::CopyOnWriteBuffer BuildFuaPayload() const;
rtc::CopyOnWriteBuffer BuildSingleNaluPayload() const;
rtc::CopyOnWriteBuffer BuildStapAPayload() const;
RTPVideoHeaderH264& H264Header() {
return absl::get<RTPVideoHeaderH264>(video_header_.video_type_header);
}
const RTPVideoHeaderH264& H264Header() const {
return absl::get<RTPVideoHeaderH264>(video_header_.video_type_header);
}
H264PacketizationTypes type_;
RTPVideoHeader video_header_;
bool first_fragment_ = false;
bool marker_bit_ = false;
uint32_t rtp_timestamp_ = 0;
uint16_t rtp_seq_num_ = 0;
std::vector<std::vector<uint8_t>> nalu_payloads_;
};
Packet::Packet(H264PacketizationTypes type) : type_(type) {
video_header_.video_type_header.emplace<RTPVideoHeaderH264>();
}
Packet& Packet::Idr(std::vector<uint8_t> payload) {
auto& h264_header = H264Header();
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kIdr);
nalu_payloads_.push_back(std::move(payload));
return *this;
}
Packet& Packet::Slice(std::vector<uint8_t> payload) {
auto& h264_header = H264Header();
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSlice);
nalu_payloads_.push_back(std::move(payload));
return *this;
}
Packet& Packet::Sps(std::vector<uint8_t> payload) {
auto& h264_header = H264Header();
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps);
nalu_payloads_.push_back(std::move(payload));
return *this;
}
Packet& Packet::SpsWithResolution(RenderResolution resolution,
std::vector<uint8_t> payload) {
auto& h264_header = H264Header();
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps);
video_header_.width = resolution.Width();
video_header_.height = resolution.Height();
nalu_payloads_.push_back(std::move(payload));
return *this;
}
Packet& Packet::Pps(std::vector<uint8_t> payload) {
auto& h264_header = H264Header();
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kPps);
nalu_payloads_.push_back(std::move(payload));
return *this;
}
Packet& Packet::Aud() {
auto& h264_header = H264Header();
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kAud);
nalu_payloads_.push_back({});
return *this;
}
Packet& Packet::Marker() {
marker_bit_ = true;
return *this;
}
Packet& Packet::AsFirstFragment() {
first_fragment_ = true;
return *this;
}
Packet& Packet::Time(uint32_t rtp_timestamp) {
rtp_timestamp_ = rtp_timestamp;
return *this;
}
Packet& Packet::SeqNum(uint16_t rtp_seq_num) {
rtp_seq_num_ = rtp_seq_num;
return *this;
}
std::unique_ptr<H264PacketBuffer::Packet> Packet::Build() {
auto res = std::make_unique<H264PacketBuffer::Packet>();
auto& h264_header = H264Header();
switch (type_) {
case kH264FuA: {
RTC_CHECK_EQ(h264_header.nalus_length, 1);
res->video_payload = BuildFuaPayload();
break;
}
case kH264SingleNalu: {
RTC_CHECK_EQ(h264_header.nalus_length, 1);
res->video_payload = BuildSingleNaluPayload();
break;
}
case kH264StapA: {
RTC_CHECK_GT(h264_header.nalus_length, 1);
RTC_CHECK_LE(h264_header.nalus_length, kMaxNalusPerPacket);
res->video_payload = BuildStapAPayload();
break;
}
}
if (type_ == kH264FuA && !first_fragment_) {
h264_header.nalus_length = 0;
}
h264_header.packetization_type = type_;
res->marker_bit = marker_bit_;
res->video_header = video_header_;
res->timestamp = rtp_timestamp_;
res->seq_num = rtp_seq_num_;
res->video_header.codec = kVideoCodecH264;
return res;
}
rtc::CopyOnWriteBuffer Packet::BuildFuaPayload() const {
return rtc::CopyOnWriteBuffer(nalu_payloads_[0]);
}
rtc::CopyOnWriteBuffer Packet::BuildSingleNaluPayload() const {
rtc::CopyOnWriteBuffer res;
auto& h264_header = H264Header();
res.AppendData(&h264_header.nalus[0].type, 1);
res.AppendData(nalu_payloads_[0]);
return res;
}
rtc::CopyOnWriteBuffer Packet::BuildStapAPayload() const {
rtc::CopyOnWriteBuffer res;
const uint8_t indicator = H264::NaluType::kStapA;
res.AppendData(&indicator, 1);
auto& h264_header = H264Header();
for (size_t i = 0; i < h264_header.nalus_length; ++i) {
// The two first bytes indicates the nalu segment size.
uint8_t length_as_array[2] = {
0, static_cast<uint8_t>(nalu_payloads_[i].size() + 1)};
res.AppendData(length_as_array);
res.AppendData(&h264_header.nalus[i].type, 1);
res.AppendData(nalu_payloads_[i]);
}
return res;
}
rtc::ArrayView<const uint8_t> PacketPayload(
const std::unique_ptr<H264PacketBuffer::Packet>& packet) {
return packet->video_payload;
}
std::vector<uint8_t> FlatVector(
const std::vector<std::vector<uint8_t>>& elems) {
std::vector<uint8_t> res;
for (const auto& elem : elems) {
res.insert(res.end(), elem.begin(), elem.end());
}
return res;
}
TEST(H264PacketBufferTest, IdrIsKeyframe) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/true);
EXPECT_THAT(
packet_buffer.InsertPacket(Packet(kH264SingleNalu).Idr().Marker().Build())
.packets,
SizeIs(1));
}
TEST(H264PacketBufferTest, IdrIsNotKeyframe) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(
packet_buffer.InsertPacket(Packet(kH264SingleNalu).Idr().Marker().Build())
.packets,
IsEmpty());
}
TEST(H264PacketBufferTest, IdrIsKeyframeFuaRequiresFirstFragmet) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/true);
// Not marked as the first fragment
EXPECT_THAT(
packet_buffer
.InsertPacket(Packet(kH264FuA).Idr().SeqNum(0).Time(0).Build())
.packets,
IsEmpty());
EXPECT_THAT(packet_buffer
.InsertPacket(
Packet(kH264FuA).Idr().SeqNum(1).Time(0).Marker().Build())
.packets,
IsEmpty());
// Marked as first fragment
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264FuA)
.Idr()
.SeqNum(2)
.Time(1)
.AsFirstFragment()
.Build())
.packets,
IsEmpty());
EXPECT_THAT(packet_buffer
.InsertPacket(
Packet(kH264FuA).Idr().SeqNum(3).Time(1).Marker().Build())
.packets,
SizeIs(2));
}
TEST(H264PacketBufferTest, SpsPpsIdrIsKeyframeSingleNalus) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Sps().SeqNum(0).Time(0).Build()));
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Pps().SeqNum(1).Time(0).Build()));
EXPECT_THAT(
packet_buffer
.InsertPacket(
Packet(kH264SingleNalu).Idr().SeqNum(2).Time(0).Marker().Build())
.packets,
SizeIs(3));
}
TEST(H264PacketBufferTest, PpsIdrIsNotKeyframeSingleNalus) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Pps().SeqNum(0).Time(0).Build()));
EXPECT_THAT(
packet_buffer
.InsertPacket(
Packet(kH264SingleNalu).Idr().SeqNum(1).Time(0).Marker().Build())
.packets,
IsEmpty());
}
TEST(H264PacketBufferTest, SpsIdrIsNotKeyframeSingleNalus) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Sps().SeqNum(0).Time(0).Build()));
EXPECT_THAT(
packet_buffer
.InsertPacket(
Packet(kH264SingleNalu).Idr().SeqNum(1).Time(0).Marker().Build())
.packets,
IsEmpty());
}
TEST(H264PacketBufferTest, SpsPpsIdrIsKeyframeStapA) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps()
.Pps()
.Idr()
.SeqNum(0)
.Time(0)
.Marker()
.Build())
.packets,
SizeIs(1));
}
TEST(H264PacketBufferTest, PpsIdrIsNotKeyframeStapA) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(
packet_buffer
.InsertPacket(
Packet(kH264StapA).Pps().Idr().SeqNum(0).Time(0).Marker().Build())
.packets,
IsEmpty());
}
TEST(H264PacketBufferTest, SpsIdrIsNotKeyframeStapA) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(
packet_buffer
.InsertPacket(
Packet(kH264StapA).Sps().Idr().SeqNum(2).Time(2).Marker().Build())
.packets,
IsEmpty());
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps()
.Pps()
.Idr()
.SeqNum(3)
.Time(3)
.Marker()
.Build())
.packets,
SizeIs(1));
}
TEST(H264PacketBufferTest, InsertingSpsPpsLastCompletesKeyframe) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Idr().SeqNum(2).Time(1).Marker().Build()));
EXPECT_THAT(packet_buffer
.InsertPacket(
Packet(kH264StapA).Sps().Pps().SeqNum(1).Time(1).Build())
.packets,
SizeIs(2));
}
TEST(H264PacketBufferTest, InsertingMidFuaCompletesFrame) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps()
.Pps()
.Idr()
.SeqNum(0)
.Time(0)
.Marker()
.Build())
.packets,
SizeIs(1));
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264FuA).Slice().SeqNum(1).Time(1).AsFirstFragment().Build()));
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264FuA).Slice().SeqNum(3).Time(1).Marker().Build()));
EXPECT_THAT(
packet_buffer
.InsertPacket(Packet(kH264FuA).Slice().SeqNum(2).Time(1).Build())
.packets,
SizeIs(3));
}
TEST(H264PacketBufferTest, SeqNumJumpDoesNotCompleteFrame) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps()
.Pps()
.Idr()
.SeqNum(0)
.Time(0)
.Marker()
.Build())
.packets,
SizeIs(1));
EXPECT_THAT(
packet_buffer
.InsertPacket(Packet(kH264FuA).Slice().SeqNum(1).Time(1).Build())
.packets,
IsEmpty());
// Add `kBufferSize` to make the index of the sequence number wrap and end up
// where the packet with sequence number 2 would have ended up.
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264FuA)
.Slice()
.SeqNum(2 + kBufferSize)
.Time(3)
.Marker()
.Build())
.packets,
IsEmpty());
}
TEST(H264PacketBufferTest, OldFramesAreNotCompletedAfterBufferWrap) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264SingleNalu)
.Slice()
.SeqNum(1)
.Time(1)
.Marker()
.Build())
.packets,
IsEmpty());
// New keyframe, preceedes packet with sequence number 1 in the buffer.
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps()
.Pps()
.Idr()
.SeqNum(kBufferSize)
.Time(kBufferSize)
.Marker()
.Build())
.packets,
SizeIs(1));
}
TEST(H264PacketBufferTest, OldPacketsDontBlockNewPackets) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps()
.Pps()
.Idr()
.SeqNum(kBufferSize)
.Time(kBufferSize)
.Marker()
.Build())
.packets,
SizeIs(1));
RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA)
.Slice()
.SeqNum(kBufferSize + 1)
.Time(kBufferSize + 1)
.AsFirstFragment()
.Build()));
RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA)
.Slice()
.SeqNum(kBufferSize + 3)
.Time(kBufferSize + 1)
.Marker()
.Build()));
EXPECT_THAT(
packet_buffer
.InsertPacket(Packet(kH264FuA).Slice().SeqNum(2).Time(2).Build())
.packets,
IsEmpty());
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264FuA)
.Slice()
.SeqNum(kBufferSize + 2)
.Time(kBufferSize + 1)
.Build())
.packets,
SizeIs(3));
}
TEST(H264PacketBufferTest, OldPacketDoesntCompleteFrame) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps()
.Pps()
.Idr()
.SeqNum(kBufferSize)
.Time(kBufferSize)
.Marker()
.Build())
.packets,
SizeIs(1));
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264FuA)
.Slice()
.SeqNum(kBufferSize + 3)
.Time(kBufferSize + 1)
.Marker()
.Build())
.packets,
IsEmpty());
EXPECT_THAT(
packet_buffer
.InsertPacket(
Packet(kH264FuA).Slice().SeqNum(2).Time(2).Marker().Build())
.packets,
IsEmpty());
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264FuA)
.Slice()
.SeqNum(kBufferSize + 1)
.Time(kBufferSize + 1)
.AsFirstFragment()
.Build())
.packets,
IsEmpty());
}
TEST(H264PacketBufferTest, FrameBoundariesAreSet) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
auto key = packet_buffer.InsertPacket(
Packet(kH264StapA).Sps().Pps().Idr().SeqNum(1).Time(1).Marker().Build());
ASSERT_THAT(key.packets, SizeIs(1));
EXPECT_TRUE(key.packets[0]->video_header.is_first_packet_in_frame);
EXPECT_TRUE(key.packets[0]->video_header.is_last_packet_in_frame);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264FuA).Slice().SeqNum(2).Time(2).Build()));
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264FuA).Slice().SeqNum(3).Time(2).Build()));
auto delta = packet_buffer.InsertPacket(
Packet(kH264FuA).Slice().SeqNum(4).Time(2).Marker().Build());
ASSERT_THAT(delta.packets, SizeIs(3));
EXPECT_TRUE(delta.packets[0]->video_header.is_first_packet_in_frame);
EXPECT_FALSE(delta.packets[0]->video_header.is_last_packet_in_frame);
EXPECT_FALSE(delta.packets[1]->video_header.is_first_packet_in_frame);
EXPECT_FALSE(delta.packets[1]->video_header.is_last_packet_in_frame);
EXPECT_FALSE(delta.packets[2]->video_header.is_first_packet_in_frame);
EXPECT_TRUE(delta.packets[2]->video_header.is_last_packet_in_frame);
}
TEST(H264PacketBufferTest, ResolutionSetOnFirstPacket) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Aud().SeqNum(1).Time(1).Build()));
auto res = packet_buffer.InsertPacket(Packet(kH264StapA)
.SpsWithResolution({320, 240})
.Pps()
.Idr()
.SeqNum(2)
.Time(1)
.Marker()
.Build());
ASSERT_THAT(res.packets, SizeIs(2));
EXPECT_THAT(res.packets[0]->video_header.width, Eq(320));
EXPECT_THAT(res.packets[0]->video_header.height, Eq(240));
}
TEST(H264PacketBufferTest, KeyframeAndDeltaFrameSetOnFirstPacket) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Aud().SeqNum(1).Time(1).Build()));
auto key = packet_buffer.InsertPacket(
Packet(kH264StapA).Sps().Pps().Idr().SeqNum(2).Time(1).Marker().Build());
auto delta = packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Slice().SeqNum(3).Time(2).Marker().Build());
ASSERT_THAT(key.packets, SizeIs(2));
EXPECT_THAT(key.packets[0]->video_header.frame_type,
Eq(VideoFrameType::kVideoFrameKey));
ASSERT_THAT(delta.packets, SizeIs(1));
EXPECT_THAT(delta.packets[0]->video_header.frame_type,
Eq(VideoFrameType::kVideoFrameDelta));
}
TEST(H264PacketBufferTest, RtpSeqNumWrap) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264StapA).Sps().Pps().SeqNum(0xffff).Time(0).Build()));
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264FuA).Idr().SeqNum(0).Time(0).Build()));
EXPECT_THAT(packet_buffer
.InsertPacket(
Packet(kH264FuA).Idr().SeqNum(1).Time(0).Marker().Build())
.packets,
SizeIs(3));
}
TEST(H264PacketBufferTest, StapAFixedBitstream) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
auto packets = packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps({1, 2, 3})
.Pps({4, 5, 6})
.Idr({7, 8, 9})
.SeqNum(0)
.Time(0)
.Marker()
.Build())
.packets;
ASSERT_THAT(packets, SizeIs(1));
EXPECT_THAT(PacketPayload(packets[0]),
ElementsAreArray(FlatVector({StartCode(),
{kSps, 1, 2, 3},
StartCode(),
{kPps, 4, 5, 6},
StartCode(),
{kIdr, 7, 8, 9}})));
}
TEST(H264PacketBufferTest, SingleNaluFixedBitstream) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Sps({1, 2, 3}).SeqNum(0).Time(0).Build()));
RTC_UNUSED(packet_buffer.InsertPacket(
Packet(kH264SingleNalu).Pps({4, 5, 6}).SeqNum(1).Time(0).Build()));
auto packets = packet_buffer
.InsertPacket(Packet(kH264SingleNalu)
.Idr({7, 8, 9})
.SeqNum(2)
.Time(0)
.Marker()
.Build())
.packets;
ASSERT_THAT(packets, SizeIs(3));
EXPECT_THAT(PacketPayload(packets[0]),
ElementsAreArray(FlatVector({StartCode(), {kSps, 1, 2, 3}})));
EXPECT_THAT(PacketPayload(packets[1]),
ElementsAreArray(FlatVector({StartCode(), {kPps, 4, 5, 6}})));
EXPECT_THAT(PacketPayload(packets[2]),
ElementsAreArray(FlatVector({StartCode(), {kIdr, 7, 8, 9}})));
}
TEST(H264PacketBufferTest, StapaAndFuaFixedBitstream) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264StapA)
.Sps({1, 2, 3})
.Pps({4, 5, 6})
.SeqNum(0)
.Time(0)
.Build()));
RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA)
.Idr({8, 8, 8})
.SeqNum(1)
.Time(0)
.AsFirstFragment()
.Build()));
auto packets = packet_buffer
.InsertPacket(Packet(kH264FuA)
.Idr({9, 9, 9})
.SeqNum(2)
.Time(0)
.Marker()
.Build())
.packets;
ASSERT_THAT(packets, SizeIs(3));
EXPECT_THAT(
PacketPayload(packets[0]),
ElementsAreArray(FlatVector(
{StartCode(), {kSps, 1, 2, 3}, StartCode(), {kPps, 4, 5, 6}})));
EXPECT_THAT(PacketPayload(packets[1]),
ElementsAreArray(FlatVector({StartCode(), {8, 8, 8}})));
// Third is a continuation of second, so only the payload is expected.
EXPECT_THAT(PacketPayload(packets[2]),
ElementsAreArray(FlatVector({{9, 9, 9}})));
}
TEST(H264PacketBufferTest, FullPacketBufferDoesNotBlockKeyframe) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
for (int i = 0; i < kBufferSize; ++i) {
EXPECT_THAT(
packet_buffer
.InsertPacket(
Packet(kH264SingleNalu).Slice().SeqNum(i).Time(0).Build())
.packets,
IsEmpty());
}
EXPECT_THAT(packet_buffer
.InsertPacket(Packet(kH264StapA)
.Sps()
.Pps()
.Idr()
.SeqNum(kBufferSize)
.Time(1)
.Marker()
.Build())
.packets,
SizeIs(1));
}
TEST(H264PacketBufferTest, TooManyNalusInPacket) {
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
std::unique_ptr<H264PacketBuffer::Packet> packet(
Packet(kH264StapA).Sps().Pps().Idr().SeqNum(1).Time(1).Marker().Build());
auto& h264_header =
absl::get<RTPVideoHeaderH264>(packet->video_header.video_type_header);
h264_header.nalus_length = kMaxNalusPerPacket + 1;
EXPECT_THAT(packet_buffer.InsertPacket(std::move(packet)).packets, IsEmpty());
}
} // namespace
} // namespace webrtc

122
modules/video_coding/h264_packet_buffer.cc → modules/video_coding/h26x_packet_buffer.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/h264_packet_buffer.h"
#include "modules/video_coding/h26x_packet_buffer.h"
#include <algorithm>
#include <cstdint>
@ -27,9 +27,13 @@
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/sequence_number_util.h"
#ifdef RTC_ENABLE_H265
#include "common_video/h265/h265_common.h"
#endif
namespace webrtc {
namespace {
int64_t EuclideanMod(int64_t n, int64_t div) {
RTC_DCHECK_GT(div, 0);
return (n %= div) < 0 ? n + div : n;
@ -48,7 +52,7 @@ bool IsFirstPacketOfFragment(const RTPVideoHeaderH264& h264_header) {
return h264_header.nalus_length > 0;
}
bool BeginningOfIdr(const H264PacketBuffer::Packet& packet) {
bool BeginningOfIdr(const H26xPacketBuffer::Packet& packet) {
const auto& h264_header =
absl::get<RTPVideoHeaderH264>(packet.video_header.video_type_header);
const bool contains_idr_nalu =
@ -66,7 +70,7 @@ bool BeginningOfIdr(const H264PacketBuffer::Packet& packet) {
}
}
bool HasSps(const H264PacketBuffer::Packet& packet) {
bool HasSps(const H26xPacketBuffer::Packet& packet) {
auto& h264_header =
absl::get<RTPVideoHeaderH264>(packet.video_header.video_type_header);
return absl::c_any_of(GetNaluInfos(h264_header), [](const auto& nalu_info) {
@ -74,10 +78,24 @@ bool HasSps(const H264PacketBuffer::Packet& packet) {
});
}
#ifdef RTC_ENABLE_H265
bool HasVps(const H26xPacketBuffer::Packet& packet) {
std::vector<H265::NaluIndex> nalu_indices = H265::FindNaluIndices(
packet.video_payload.cdata(), packet.video_payload.size());
return absl::c_any_of((nalu_indices), [&packet](
const H265::NaluIndex& nalu_index) {
return H265::ParseNaluType(
packet.video_payload.cdata()[nalu_index.payload_start_offset]) ==
H265::NaluType::kVps;
});
}
#endif
// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to
// fiddle with the payload at this point.
rtc::CopyOnWriteBuffer FixVideoPayload(rtc::ArrayView<const uint8_t> payload,
const RTPVideoHeader& video_header) {
rtc::CopyOnWriteBuffer FixH264VideoPayload(
rtc::ArrayView<const uint8_t> payload,
const RTPVideoHeader& video_header) {
constexpr uint8_t kStartCode[] = {0, 0, 0, 1};
const auto& h264_header =
@ -124,18 +142,15 @@ rtc::CopyOnWriteBuffer FixVideoPayload(rtc::ArrayView<const uint8_t> payload,
} // namespace
H264PacketBuffer::H264PacketBuffer(bool idr_only_keyframes_allowed)
: idr_only_keyframes_allowed_(idr_only_keyframes_allowed) {}
H26xPacketBuffer::H26xPacketBuffer(bool h264_idr_only_keyframes_allowed)
: h264_idr_only_keyframes_allowed_(h264_idr_only_keyframes_allowed) {}
H264PacketBuffer::InsertResult H264PacketBuffer::InsertPacket(
H26xPacketBuffer::InsertResult H26xPacketBuffer::InsertPacket(
std::unique_ptr<Packet> packet) {
RTC_DCHECK(packet->video_header.codec == kVideoCodecH264);
RTC_DCHECK(packet->video_header.codec == kVideoCodecH264 ||
packet->video_header.codec == kVideoCodecH265);
InsertResult result;
if (!absl::holds_alternative<RTPVideoHeaderH264>(
packet->video_header.video_type_header)) {
return result;
}
int64_t unwrapped_seq_num = seq_num_unwrapper_.Unwrap(packet->seq_num);
auto& packet_slot = GetPacket(unwrapped_seq_num);
@ -151,19 +166,27 @@ H264PacketBuffer::InsertResult H264PacketBuffer::InsertPacket(
return result;
}
std::unique_ptr<H264PacketBuffer::Packet>& H264PacketBuffer::GetPacket(
std::unique_ptr<H26xPacketBuffer::Packet>& H26xPacketBuffer::GetPacket(
int64_t unwrapped_seq_num) {
return buffer_[EuclideanMod(unwrapped_seq_num, kBufferSize)];
}
bool H264PacketBuffer::BeginningOfStream(
const H264PacketBuffer::Packet& packet) const {
return HasSps(packet) ||
(idr_only_keyframes_allowed_ && BeginningOfIdr(packet));
bool H26xPacketBuffer::BeginningOfStream(
const H26xPacketBuffer::Packet& packet) const {
if (packet.codec() == kVideoCodecH264) {
return HasSps(packet) ||
(h264_idr_only_keyframes_allowed_ && BeginningOfIdr(packet));
#ifdef RTC_ENABLE_H265
} else if (packet.codec() == kVideoCodecH265) {
return HasVps(packet);
#endif
}
RTC_DCHECK_NOTREACHED();
return false;
}
std::vector<std::unique_ptr<H264PacketBuffer::Packet>>
H264PacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
std::vector<std::unique_ptr<H26xPacketBuffer::Packet>>
H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
std::vector<std::unique_ptr<Packet>> found_frames;
Packet* packet = GetPacket(unwrapped_seq_num).get();
@ -223,13 +246,17 @@ H264PacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
return found_frames;
}
bool H264PacketBuffer::MaybeAssembleFrame(
bool H26xPacketBuffer::MaybeAssembleFrame(
int64_t start_seq_num_unwrapped,
int64_t end_sequence_number_unwrapped,
std::vector<std::unique_ptr<Packet>>& frames) {
#ifdef RTC_ENABLE_H265
bool has_vps = false;
#endif
bool has_sps = false;
bool has_pps = false;
bool has_idr = false;
bool has_irap = false;
int width = -1;
int height = -1;
@ -237,24 +264,44 @@ bool H264PacketBuffer::MaybeAssembleFrame(
for (int64_t seq_num = start_seq_num_unwrapped;
seq_num <= end_sequence_number_unwrapped; ++seq_num) {
const auto& packet = GetPacket(seq_num);
const auto& h264_header =
absl::get<RTPVideoHeaderH264>(packet->video_header.video_type_header);
for (const auto& nalu : GetNaluInfos(h264_header)) {
has_idr |= nalu.type == H264::NaluType::kIdr;
has_sps |= nalu.type == H264::NaluType::kSps;
has_pps |= nalu.type == H264::NaluType::kPps;
if (packet->codec() == kVideoCodecH264) {
const auto& h264_header =
absl::get<RTPVideoHeaderH264>(packet->video_header.video_type_header);
for (const auto& nalu : GetNaluInfos(h264_header)) {
has_idr |= nalu.type == H264::NaluType::kIdr;
has_sps |= nalu.type == H264::NaluType::kSps;
has_pps |= nalu.type == H264::NaluType::kPps;
}
if (has_idr) {
if (!h264_idr_only_keyframes_allowed_ && (!has_sps || !has_pps)) {
return false;
}
}
#ifdef RTC_ENABLE_H265
} else if (packet->codec() == kVideoCodecH265) {
std::vector<H265::NaluIndex> nalu_indices = H265::FindNaluIndices(
packet->video_payload.cdata(), packet->video_payload.size());
for (const auto& nalu_index : nalu_indices) {
uint8_t nalu_type = H265::ParseNaluType(
packet->video_payload.cdata()[nalu_index.payload_start_offset]);
has_irap |= (nalu_type >= H265::NaluType::kBlaWLp &&
nalu_type <= H265::NaluType::kRsvIrapVcl23);
has_vps |= nalu_type == H265::NaluType::kVps;
has_sps |= nalu_type == H265::NaluType::kSps;
has_pps |= nalu_type == H265::NaluType::kPps;
}
if (has_irap) {
if (!has_vps || !has_sps || !has_pps) {
return false;
}
}
#endif // RTC_ENABLE_H265
}
width = std::max<int>(packet->video_header.width, width);
height = std::max<int>(packet->video_header.height, height);
}
if (has_idr) {
if (!idr_only_keyframes_allowed_ && (!has_sps || !has_pps)) {
return false;
}
}
for (int64_t seq_num = start_seq_num_unwrapped;
seq_num <= end_sequence_number_unwrapped; ++seq_num) {
auto& packet = GetPacket(seq_num);
@ -270,13 +317,16 @@ bool H264PacketBuffer::MaybeAssembleFrame(
packet->video_header.height = height;
}
packet->video_header.frame_type = has_idr
packet->video_header.frame_type = has_idr || has_irap
? VideoFrameType::kVideoFrameKey
: VideoFrameType::kVideoFrameDelta;
}
packet->video_payload =
FixVideoPayload(packet->video_payload, packet->video_header);
// Start code is inserted by depacktizer for H.265.
if (packet->codec() == kVideoCodecH264) {
packet->video_payload =
FixH264VideoPayload(packet->video_payload, packet->video_header);
}
frames.push_back(std::move(packet));
}

19
modules/video_coding/h264_packet_buffer.h → modules/video_coding/h26x_packet_buffer.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_
#define MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_
#ifndef MODULES_VIDEO_CODING_H26X_PACKET_BUFFER_H_
#define MODULES_VIDEO_CODING_H26X_PACKET_BUFFER_H_
#include <array>
#include <memory>
@ -22,15 +22,16 @@
namespace webrtc {
class H264PacketBuffer {
class H26xPacketBuffer {
public:
// The H264PacketBuffer does the same job as the PacketBuffer but for H264
// only. To make it fit in with surronding code the PacketBuffer input/output
// classes are used.
// The H26xPacketBuffer does the same job as the PacketBuffer but for H264 and
// H265 only. To make it fit in with surronding code the PacketBuffer
// input/output classes are used.
using Packet = video_coding::PacketBuffer::Packet;
using InsertResult = video_coding::PacketBuffer::InsertResult;
explicit H264PacketBuffer(bool idr_only_keyframes_allowed);
// |h264_idr_only_keyframes_allowed| is ignored if H.265 is used.
explicit H26xPacketBuffer(bool h264_idr_only_keyframes_allowed);
ABSL_MUST_USE_RESULT InsertResult
InsertPacket(std::unique_ptr<Packet> packet);
@ -45,7 +46,7 @@ class H264PacketBuffer {
int64_t end_sequence_number_unwrapped,
std::vector<std::unique_ptr<Packet>>& packets);
const bool idr_only_keyframes_allowed_;
const bool h264_idr_only_keyframes_allowed_;
std::array<std::unique_ptr<Packet>, kBufferSize> buffer_;
absl::optional<int64_t> last_continuous_unwrapped_seq_num_;
SeqNumUnwrapper<uint16_t> seq_num_unwrapper_;
@ -53,4 +54,4 @@ class H264PacketBuffer {
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_
#endif // MODULES_VIDEO_CODING_H26X_PACKET_BUFFER_H_

1056
modules/video_coding/h26x_packet_buffer_unittest.cc Normal file
View File

File diff suppressed because it is too large Load Diff