From 9728241e6a57b0ac6c994cded1b3e87bafd241f1 Mon Sep 17 00:00:00 2001 From: Noah Richards Date: Thu, 23 Apr 2015 11:15:08 -0700 Subject: [PATCH] Record H264 NALU type in the h264 header. BUG= R=niklas.enbom@webrtc.org, stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/48999004 Cr-Commit-Position: refs/heads/master@{#9072} --- .../modules/interface/module_common_types.h | 20 +++++++++- .../rtp_rtcp/source/rtp_format_h264.cc | 11 ++--- .../source/rtp_format_h264_unittest.cc | 40 ++++++++++--------- .../video_coding/main/source/packet.cc | 2 +- .../video_coding/main/source/session_info.cc | 3 +- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/webrtc/modules/interface/module_common_types.h b/webrtc/modules/interface/module_common_types.h index dc8617631b..825d7e6c4c 100644 --- a/webrtc/modules/interface/module_common_types.h +++ b/webrtc/modules/interface/module_common_types.h @@ -61,9 +61,25 @@ struct RTPVideoHeaderVP8 { // in a VP8 partition. Otherwise false }; +// The packetization types that we support: single, aggregated, and fragmented. +enum H264PacketizationTypes { + kH264SingleNalu, // This packet contains a single NAL unit. + kH264StapA, // This packet contains STAP-A (single time + // aggregation) packets. If this packet has an + // associated NAL unit type, it'll be for the + // first such aggregated packet. + kH264FuA, // This packet contains a FU-A (fragmentation + // unit) packet, meaning it is a part of a frame + // that was too large to fit into a single packet. +}; + struct RTPVideoHeaderH264 { - bool stap_a; - bool single_nalu; + uint8_t nalu_type; // The NAL unit type. If this is a header for a + // fragmented packet, it's the NAL unit type of + // the original data. If this is the header for an + // aggregated packet, it's the NAL unit type of + // the first NAL unit in the packet. + H264PacketizationTypes packetization_type; }; union RTPVideoTypeHeader { diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc index 6568c38335..e297b7ce77 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -46,14 +46,15 @@ void ParseSingleNalu(RtpDepacketizer::ParsedPayload* parsed_payload, parsed_payload->type.Video.isFirstPacket = true; RTPVideoHeaderH264* h264_header = &parsed_payload->type.Video.codecHeader.H264; - h264_header->single_nalu = true; - h264_header->stap_a = false; uint8_t nal_type = payload_data[0] & kTypeMask; if (nal_type == kStapA) { nal_type = payload_data[3] & kTypeMask; - h264_header->stap_a = true; + h264_header->packetization_type = kH264StapA; + } else { + h264_header->packetization_type = kH264SingleNalu; } + h264_header->nalu_type = nal_type; switch (nal_type) { case kSps: @@ -95,8 +96,8 @@ void ParseFuaNalu(RtpDepacketizer::ParsedPayload* parsed_payload, parsed_payload->type.Video.isFirstPacket = first_fragment; RTPVideoHeaderH264* h264_header = &parsed_payload->type.Video.codecHeader.H264; - h264_header->single_nalu = false; - h264_header->stap_a = false; + h264_header->packetization_type = kH264FuA; + h264_header->nalu_type = original_nal_type; } } // namespace diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc index caae400550..5ccd85357a 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc @@ -207,9 +207,9 @@ TEST(RtpPacketizerH264Test, TestSingleNaluTwoPackets) { TEST(RtpPacketizerH264Test, TestStapA) { const size_t kFrameSize = kMaxPayloadSize - 3 * kLengthFieldLength - kNalHeaderSize; - uint8_t frame[kFrameSize] = {0x07, 0xFF, // F=0, NRI=0, Type=7. - 0x08, 0xFF, // F=0, NRI=0, Type=8. - 0x05}; // F=0, NRI=0, Type=5. + uint8_t frame[kFrameSize] = {0x07, 0xFF, // F=0, NRI=0, Type=7 (SPS). + 0x08, 0xFF, // F=0, NRI=0, Type=8 (PPS). + 0x05}; // F=0, NRI=0, Type=5 (IDR). const size_t kPayloadOffset = 5; for (size_t i = 0; i < kFrameSize - kPayloadOffset; ++i) frame[i + kPayloadOffset] = i; @@ -398,7 +398,7 @@ class RtpDepacketizerH264Test : public ::testing::Test { }; TEST_F(RtpDepacketizerH264Test, TestSingleNalu) { - uint8_t packet[2] = {0x05, 0xFF}; // F=0, NRI=0, Type=5. + uint8_t packet[2] = {0x05, 0xFF}; // F=0, NRI=0, Type=5 (IDR). RtpDepacketizer::ParsedPayload payload; ASSERT_TRUE(depacketizer_->Parse(&payload, packet, sizeof(packet))); @@ -406,15 +406,17 @@ TEST_F(RtpDepacketizerH264Test, TestSingleNalu) { EXPECT_EQ(kVideoFrameKey, payload.frame_type); EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec); EXPECT_TRUE(payload.type.Video.isFirstPacket); - EXPECT_TRUE(payload.type.Video.codecHeader.H264.single_nalu); - EXPECT_FALSE(payload.type.Video.codecHeader.H264.stap_a); + EXPECT_EQ(kH264SingleNalu, + payload.type.Video.codecHeader.H264.packetization_type); + EXPECT_EQ(kIdr, payload.type.Video.codecHeader.H264.nalu_type); } TEST_F(RtpDepacketizerH264Test, TestStapAKey) { uint8_t packet[16] = {kStapA, // F=0, NRI=0, Type=24. // Length, nal header, payload. - 0, 0x02, kIdr, 0xFF, 0, 0x03, kIdr, 0xFF, - 0x00, 0, 0x04, kIdr, 0xFF, 0x00, 0x11}; + 0, 0x02, kSps, 0xFF, + 0, 0x03, kPps, 0xFF, 0x00, + 0, 0x04, kIdr, 0xFF, 0x00, 0x11}; RtpDepacketizer::ParsedPayload payload; ASSERT_TRUE(depacketizer_->Parse(&payload, packet, sizeof(packet))); @@ -422,8 +424,9 @@ TEST_F(RtpDepacketizerH264Test, TestStapAKey) { EXPECT_EQ(kVideoFrameKey, payload.frame_type); EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec); EXPECT_TRUE(payload.type.Video.isFirstPacket); - EXPECT_TRUE(payload.type.Video.codecHeader.H264.single_nalu); - EXPECT_TRUE(payload.type.Video.codecHeader.H264.stap_a); + EXPECT_EQ(kH264StapA, payload.type.Video.codecHeader.H264.packetization_type); + // NALU type for aggregated packets is the type of the first packet only. + EXPECT_EQ(kSps, payload.type.Video.codecHeader.H264.nalu_type); } TEST_F(RtpDepacketizerH264Test, TestStapADelta) { @@ -438,8 +441,9 @@ TEST_F(RtpDepacketizerH264Test, TestStapADelta) { EXPECT_EQ(kVideoFrameDelta, payload.frame_type); EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec); EXPECT_TRUE(payload.type.Video.isFirstPacket); - EXPECT_TRUE(payload.type.Video.codecHeader.H264.single_nalu); - EXPECT_TRUE(payload.type.Video.codecHeader.H264.stap_a); + EXPECT_EQ(kH264StapA, payload.type.Video.codecHeader.H264.packetization_type); + // NALU type for aggregated packets is the type of the first packet only. + EXPECT_EQ(kSlice, payload.type.Video.codecHeader.H264.nalu_type); } TEST_F(RtpDepacketizerH264Test, TestFuA) { @@ -473,8 +477,8 @@ TEST_F(RtpDepacketizerH264Test, TestFuA) { EXPECT_EQ(kVideoFrameKey, payload.frame_type); EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec); EXPECT_TRUE(payload.type.Video.isFirstPacket); - EXPECT_FALSE(payload.type.Video.codecHeader.H264.single_nalu); - EXPECT_FALSE(payload.type.Video.codecHeader.H264.stap_a); + EXPECT_EQ(kH264FuA, payload.type.Video.codecHeader.H264.packetization_type); + EXPECT_EQ(kIdr, payload.type.Video.codecHeader.H264.nalu_type); // Following packets will be 2 bytes shorter since they will only be appended // onto the first packet. @@ -484,8 +488,8 @@ TEST_F(RtpDepacketizerH264Test, TestFuA) { EXPECT_EQ(kVideoFrameKey, payload.frame_type); EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec); EXPECT_FALSE(payload.type.Video.isFirstPacket); - EXPECT_FALSE(payload.type.Video.codecHeader.H264.single_nalu); - EXPECT_FALSE(payload.type.Video.codecHeader.H264.stap_a); + EXPECT_EQ(kH264FuA, payload.type.Video.codecHeader.H264.packetization_type); + EXPECT_EQ(kIdr, payload.type.Video.codecHeader.H264.nalu_type); payload = RtpDepacketizer::ParsedPayload(); ASSERT_TRUE(depacketizer_->Parse(&payload, packet3, sizeof(packet3))); @@ -493,7 +497,7 @@ TEST_F(RtpDepacketizerH264Test, TestFuA) { EXPECT_EQ(kVideoFrameKey, payload.frame_type); EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec); EXPECT_FALSE(payload.type.Video.isFirstPacket); - EXPECT_FALSE(payload.type.Video.codecHeader.H264.single_nalu); - EXPECT_FALSE(payload.type.Video.codecHeader.H264.stap_a); + EXPECT_EQ(kH264FuA, payload.type.Video.codecHeader.H264.packetization_type); + EXPECT_EQ(kIdr, payload.type.Video.codecHeader.H264.nalu_type); } } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/packet.cc b/webrtc/modules/video_coding/main/source/packet.cc index fb815c45b9..c9eb482ed0 100644 --- a/webrtc/modules/video_coding/main/source/packet.cc +++ b/webrtc/modules/video_coding/main/source/packet.cc @@ -123,7 +123,7 @@ void VCMPacket::CopyCodecSpecifics(const RTPVideoHeader& videoHeader) { if (isFirstPacket) insertStartCode = true; - if (videoHeader.codecHeader.H264.single_nalu) { + if (isFirstPacket && markerBit) { completeNALU = kNaluComplete; } else if (isFirstPacket) { completeNALU = kNaluStart; diff --git a/webrtc/modules/video_coding/main/source/session_info.cc b/webrtc/modules/video_coding/main/source/session_info.cc index 361c0a1a22..8eba432643 100644 --- a/webrtc/modules/video_coding/main/source/session_info.cc +++ b/webrtc/modules/video_coding/main/source/session_info.cc @@ -136,7 +136,8 @@ size_t VCMSessionInfo::InsertBuffer(uint8_t* frame_buffer, const size_t kH264NALHeaderLengthInBytes = 1; const size_t kLengthFieldLength = 2; if (packet.codecSpecificHeader.codec == kRtpVideoH264 && - packet.codecSpecificHeader.codecHeader.H264.stap_a) { + packet.codecSpecificHeader.codecHeader.H264.packetization_type == + kH264StapA) { size_t required_length = 0; const uint8_t* nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes; while (nalu_ptr < packet_buffer + packet.sizeBytes) {