diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index fa58999572..a7ea1191ff 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -491,6 +491,7 @@ if (rtc_include_tests) { "video_coding/codecs/vp8/simulcast_unittest.h", "video_coding/decoding_state_unittest.cc", "video_coding/frame_buffer2_unittest.cc", + "video_coding/h264_sps_pps_tracker_unittest.cc", "video_coding/histogram_unittest.cc", "video_coding/include/mock/mock_vcm_callbacks.h", "video_coding/jitter_buffer_unittest.cc", diff --git a/webrtc/modules/include/module_common_types.h b/webrtc/modules/include/module_common_types.h index 4e43110596..a5ea5c8e2d 100644 --- a/webrtc/modules/include/module_common_types.h +++ b/webrtc/modules/include/module_common_types.h @@ -264,6 +264,10 @@ struct NaluInfo { uint8_t type; int sps_id; int pps_id; + + // Offset and size are only valid for non-FuA packets. + size_t offset; + size_t size; }; const size_t kMaxNalusPerPacket = 10; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc index 2747945831..b32e78ef9a 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -411,6 +411,8 @@ bool RtpDepacketizerH264::ProcessStapAOrSingleNalu( NaluInfo nalu; nalu.type = payload_data[start_offset] & kTypeMask; + nalu.offset = start_offset; + nalu.size = end_offset - start_offset; nalu.sps_id = -1; nalu.pps_id = -1; start_offset += H264::kNaluTypeSize; diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index a21a79bbd9..e567b89327 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -29,6 +29,8 @@ rtc_static_library("video_coding") { "generic_decoder.h", "generic_encoder.cc", "generic_encoder.h", + "h264_sps_pps_tracker.cc", + "h264_sps_pps_tracker.h", "histogram.cc", "histogram.h", "include/video_coding.h", diff --git a/webrtc/modules/video_coding/h264_sps_pps_tracker.cc b/webrtc/modules/video_coding/h264_sps_pps_tracker.cc new file mode 100644 index 0000000000..b44f477427 --- /dev/null +++ b/webrtc/modules/video_coding/h264_sps_pps_tracker.cc @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h" + +#include + +#include "webrtc/base/checks.h" +#include "webrtc/base/logging.h" +#include "webrtc/common_video/h264/h264_common.h" +#include "webrtc/modules/video_coding/frame_object.h" +#include "webrtc/modules/video_coding/packet_buffer.h" + +namespace webrtc { +namespace video_coding { + +namespace { +const uint8_t start_code_h264[] = {0, 0, 0, 1}; +} // namespace + +bool H264SpsPpsTracker::CopyAndFixBitstream(VCMPacket* packet) { + RTC_DCHECK(packet->codec == kVideoCodecH264); + + const uint8_t* data = packet->dataPtr; + const size_t data_size = packet->sizeBytes; + const RTPVideoHeader& video_header = packet->video_header; + const RTPVideoHeaderH264& codec_header = video_header.codecHeader.H264; + + // Packets that only contains SPS/PPS are not decodable by themselves, and + // to avoid frames being created containing only these two nalus we don't + // insert them into the PacketBuffer. Instead we save the SPS/PPS and + // prepend the bitstream of first packet of an IDR referring to the + // corresponding SPS/PPS id. + bool insert_packet = codec_header.nalus_length == 0 ? true : false; + + int pps_id = -1; + size_t required_size = 0; + for (size_t i = 0; i < codec_header.nalus_length; ++i) { + const NaluInfo& nalu = codec_header.nalus[i]; + switch (nalu.type) { + case H264::NaluType::kSps: { + // Save SPS. + sps_data_[nalu.sps_id].size = nalu.size; + sps_data_[nalu.sps_id].data.reset(new uint8_t[nalu.size]); + memcpy(sps_data_[nalu.sps_id].data.get(), data + nalu.offset, + nalu.size); + break; + } + case H264::NaluType::kPps: { + // Save PPS. + pps_data_[nalu.pps_id].sps_id = nalu.sps_id; + pps_data_[nalu.pps_id].size = nalu.size; + pps_data_[nalu.pps_id].data.reset(new uint8_t[nalu.size]); + memcpy(pps_data_[nalu.pps_id].data.get(), data + nalu.offset, + nalu.size); + break; + } + case H264::NaluType::kIdr: { + // If this is the first packet of an IDR, make sure we have the required + // SPS/PPS and also calculate how much extra space we need in the buffer + // to prepend the SPS/PPS to the bitstream with start codes. + if (video_header.isFirstPacket) { + if (nalu.pps_id == -1) { + LOG(LS_WARNING) << "No PPS id in IDR nalu."; + return false; + } + + auto pps = pps_data_.find(nalu.pps_id); + if (pps == pps_data_.end()) { + LOG(LS_WARNING) << "No PPS with id << " << nalu.pps_id + << " received"; + return false; + } + + auto sps = sps_data_.find(pps->second.sps_id); + if (sps == sps_data_.end()) { + LOG(LS_WARNING) << "No SPS with id << " + << pps_data_[nalu.pps_id].sps_id << " received"; + return false; + } + + pps_id = nalu.pps_id; + required_size += pps->second.size + sizeof(start_code_h264); + required_size += sps->second.size + sizeof(start_code_h264); + } + FALLTHROUGH(); + } + default: { + // Something other than an SPS/PPS nalu in this packet, then it should + // be inserted into the PacketBuffer. + insert_packet = true; + } + } + } + + if (!insert_packet) + return false; + + // Calculate how much space we need for the rest of the bitstream. + if (codec_header.packetization_type == kH264StapA) { + const uint8_t* nalu_ptr = data + 1; + while (nalu_ptr < data + data_size) { + RTC_DCHECK(video_header.isFirstPacket); + required_size += sizeof(start_code_h264); + + // The first two bytes describe the length of a segment. + uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1]; + nalu_ptr += 2; + + required_size += segment_length; + nalu_ptr += segment_length; + } + } else { + if (video_header.isFirstPacket) + required_size += sizeof(start_code_h264); + required_size += data_size; + } + + // Then we copy to the new buffer. + uint8_t* buffer = new uint8_t[required_size]; + uint8_t* insert_at = buffer; + + // If pps_id != -1 then we have the SPS/PPS and they should be prepended + // to the bitstream with start codes inserted. + if (pps_id != -1) { + // Insert SPS. + memcpy(insert_at, start_code_h264, sizeof(start_code_h264)); + insert_at += sizeof(start_code_h264); + memcpy(insert_at, sps_data_[pps_data_[pps_id].sps_id].data.get(), + sps_data_[pps_data_[pps_id].sps_id].size); + insert_at += sps_data_[pps_data_[pps_id].sps_id].size; + + // Insert PPS. + memcpy(insert_at, start_code_h264, sizeof(start_code_h264)); + insert_at += sizeof(start_code_h264); + memcpy(insert_at, pps_data_[pps_id].data.get(), pps_data_[pps_id].size); + insert_at += pps_data_[pps_id].size; + } + + // Copy the rest of the bitstream and insert start codes. + if (codec_header.packetization_type == kH264StapA) { + const uint8_t* nalu_ptr = data + 1; + while (nalu_ptr < data + data_size) { + memcpy(insert_at, start_code_h264, sizeof(start_code_h264)); + insert_at += sizeof(start_code_h264); + + // The first two bytes describe the length of a segment. + uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1]; + nalu_ptr += 2; + + size_t copy_end = nalu_ptr - data + segment_length; + if (copy_end > data_size) { + delete[] buffer; + return false; + } + + memcpy(insert_at, nalu_ptr, segment_length); + insert_at += segment_length; + nalu_ptr += segment_length; + } + } else { + if (video_header.isFirstPacket) { + memcpy(insert_at, start_code_h264, sizeof(start_code_h264)); + insert_at += sizeof(start_code_h264); + } + memcpy(insert_at, data, data_size); + } + + packet->dataPtr = buffer; + packet->sizeBytes = required_size; + return true; +} + +} // namespace video_coding +} // namespace webrtc diff --git a/webrtc/modules/video_coding/h264_sps_pps_tracker.h b/webrtc/modules/video_coding/h264_sps_pps_tracker.h new file mode 100644 index 0000000000..6de092a365 --- /dev/null +++ b/webrtc/modules/video_coding/h264_sps_pps_tracker.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_ +#define WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_ + +#include +#include +#include + +#include "webrtc/modules/include/module_common_types.h" + +namespace webrtc { + +class VCMPacket; + +namespace video_coding { + +class H264SpsPpsTracker { + public: + bool CopyAndFixBitstream(VCMPacket* packet); + + private: + struct PpsInfo { + int sps_id = -1; + size_t size = 0; + std::unique_ptr data; + }; + + struct SpsInfo { + size_t size = 0; + std::unique_ptr data; + }; + + std::map pps_data_; + std::map sps_data_; +}; + +} // namespace video_coding +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_ diff --git a/webrtc/modules/video_coding/h264_sps_pps_tracker_unittest.cc b/webrtc/modules/video_coding/h264_sps_pps_tracker_unittest.cc new file mode 100644 index 0000000000..f467028f16 --- /dev/null +++ b/webrtc/modules/video_coding/h264_sps_pps_tracker_unittest.cc @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h" + +#include + +#include "webrtc/modules/video_coding/packet.h" +#include "webrtc/test/gtest.h" +#include "webrtc/common_video/h264/h264_common.h" + +namespace webrtc { +namespace video_coding { + +namespace { +const uint8_t start_code[] = {0, 0, 0, 1}; +} // namespace + +class TestH264SpsPpsTracker : public ::testing::Test { + public: + VCMPacket GetDefaultPacket() { + VCMPacket packet; + packet.codec = kVideoCodecH264; + packet.video_header.codecHeader.H264.nalus_length = 0; + packet.video_header.isFirstPacket = false; + packet.video_header.codecHeader.H264.packetization_type = kH264SingleNalu; + + return packet; + } + + void AddSps(VCMPacket* packet, int sps_id, std::vector* data) { + NaluInfo info; + info.type = H264::NaluType::kSps; + info.sps_id = sps_id; + info.pps_id = -1; + info.offset = data->size(); + info.size = 2; + data->push_back(H264::NaluType::kSps); + data->push_back(sps_id); // The sps data, just a single byte. + + packet->video_header.codecHeader.H264 + .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info; + } + + void AddPps(VCMPacket* packet, + int sps_id, + int pps_id, + std::vector* data) { + NaluInfo info; + info.type = H264::NaluType::kPps; + info.sps_id = sps_id; + info.pps_id = pps_id; + info.offset = data->size(); + info.size = 2; + data->push_back(H264::NaluType::kPps); + data->push_back(pps_id); // The pps data, just a single byte. + + packet->video_header.codecHeader.H264 + .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info; + } + + void AddIdr(VCMPacket* packet, int pps_id) { + NaluInfo info; + info.type = H264::NaluType::kIdr; + info.sps_id = -1; + info.pps_id = pps_id; + + packet->video_header.codecHeader.H264 + .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info; + } + + protected: + H264SpsPpsTracker tracker_; +}; + +TEST_F(TestH264SpsPpsTracker, NoNalus) { + uint8_t data[] = {1, 2, 3}; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.codecHeader.H264.packetization_type = kH264FuA; + packet.dataPtr = data; + packet.sizeBytes = sizeof(data); + + EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet)); + EXPECT_EQ(memcmp(packet.dataPtr, data, sizeof(data)), 0); + delete[] packet.dataPtr; +} + +TEST_F(TestH264SpsPpsTracker, FuAFirstPacket) { + uint8_t data[] = {1, 2, 3}; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.codecHeader.H264.packetization_type = kH264FuA; + packet.video_header.isFirstPacket = true; + packet.dataPtr = data; + packet.sizeBytes = sizeof(data); + + EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet)); + std::vector expected; + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {1, 2, 3}); + EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0); + delete[] packet.dataPtr; +} + +TEST_F(TestH264SpsPpsTracker, StapAIncorrectSegmentLength) { + uint8_t data[] = {0, 0, 2, 0}; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.codecHeader.H264.packetization_type = kH264StapA; + packet.video_header.isFirstPacket = true; + packet.dataPtr = data; + packet.sizeBytes = sizeof(data); + + EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet)); +} + +TEST_F(TestH264SpsPpsTracker, NoNalusFirstPacket) { + uint8_t data[] = {1, 2, 3}; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.isFirstPacket = true; + packet.dataPtr = data; + packet.sizeBytes = sizeof(data); + + EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet)); + std::vector expected; + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {1, 2, 3}); + EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0); + delete[] packet.dataPtr; +} + +TEST_F(TestH264SpsPpsTracker, IdrNoSpsPpsInserted) { + std::vector data = {1, 2, 3}; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.codecHeader.H264.packetization_type = kH264FuA; + + AddIdr(&packet, 0); + packet.dataPtr = data.data(); + packet.sizeBytes = data.size(); + + EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet)); + EXPECT_EQ(memcmp(packet.dataPtr, data.data(), data.size()), 0); + delete[] packet.dataPtr; +} + +TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoSpsPpsInserted) { + std::vector data = {1, 2, 3}; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.isFirstPacket = true; + + AddIdr(&packet, 0); + packet.dataPtr = data.data(); + packet.sizeBytes = data.size(); + + EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet)); +} + +TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoPpsInserted) { + std::vector data = {1, 2, 3}; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.isFirstPacket = true; + + AddSps(&packet, 0, &data); + AddIdr(&packet, 0); + packet.dataPtr = data.data(); + packet.sizeBytes = data.size(); + + EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet)); +} + +TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoSpsInserted) { + std::vector data = {1, 2, 3}; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.isFirstPacket = true; + + AddPps(&packet, 0, 0, &data); + AddIdr(&packet, 0); + packet.dataPtr = data.data(); + packet.sizeBytes = data.size(); + + EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet)); +} + +TEST_F(TestH264SpsPpsTracker, SpsPpsPacketThenIdrFirstPacket) { + std::vector data; + VCMPacket sps_pps_packet = GetDefaultPacket(); + + // Insert SPS/PPS + AddSps(&sps_pps_packet, 0, &data); + AddPps(&sps_pps_packet, 0, 1, &data); + sps_pps_packet.dataPtr = data.data(); + sps_pps_packet.sizeBytes = data.size(); + EXPECT_FALSE(tracker_.CopyAndFixBitstream(&sps_pps_packet)); + data.clear(); + + // Insert first packet of the IDR + VCMPacket idr_packet = GetDefaultPacket(); + idr_packet.video_header.isFirstPacket = true; + AddIdr(&idr_packet, 1); + data.insert(data.end(), {1, 2, 3}); + idr_packet.dataPtr = data.data(); + idr_packet.sizeBytes = data.size(); + EXPECT_TRUE(tracker_.CopyAndFixBitstream(&idr_packet)); + + std::vector expected; + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {H264::NaluType::kSps, 0}); + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {H264::NaluType::kPps, 1}); + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {1, 2, 3}); + EXPECT_EQ(memcmp(idr_packet.dataPtr, expected.data(), expected.size()), 0); + delete[] idr_packet.dataPtr; +} + +TEST_F(TestH264SpsPpsTracker, SpsPpsIdrInStapA) { + std::vector data; + VCMPacket packet = GetDefaultPacket(); + packet.video_header.codecHeader.H264.packetization_type = kH264StapA; + packet.video_header.isFirstPacket = true; // Always true for StapA + + data.insert(data.end(), {0}); // First byte is ignored + data.insert(data.end(), {0, 2}); // Length of segment + AddSps(&packet, 13, &data); + data.insert(data.end(), {0, 2}); // Length of segment + AddPps(&packet, 13, 27, &data); + data.insert(data.end(), {0, 5}); // Length of segment + AddIdr(&packet, 27); + data.insert(data.end(), {1, 2, 3, 2, 1}); + + packet.dataPtr = data.data(); + packet.sizeBytes = data.size(); + EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet)); + + std::vector expected; + // The SPS/PPS is repeated because this packet both contains the SPS/PPS + // and it is the first packet of an IDR, which will cause the SPS/PPS to be + // prepended to the bitstream. + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {H264::NaluType::kSps, 13}); + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {H264::NaluType::kPps, 27}); + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {H264::NaluType::kSps, 13}); + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {H264::NaluType::kPps, 27}); + expected.insert(expected.end(), start_code, start_code + sizeof(start_code)); + expected.insert(expected.end(), {1, 2, 3, 2, 1}); + + EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0); + delete[] packet.dataPtr; +} + +} // namespace video_coding +} // namespace webrtc