H264SpsPpsTracker class which keep tracks of SPS/PPS.

The H264SpsPpsTracker class:
 - Keeps track of all received SPS/PPS.
 - Decides whether a packet should be inserted into the PacketBuffer or not.
   - Don't insert if this packet only contains SPS and/or PPS.
   - Don't insert if this is the first packet of and IDR and we have not
     received the required SPS/PPS.
 - Insert start codes, and in the case of the first packet of an IDR prepend
   the bitstream with the given SPS/PPS for this IDR.

BUG=webrtc:5514

Review-Url: https://codereview.webrtc.org/2466993003
Cr-Commit-Position: refs/heads/master@{#14906}

webrtc/modules/BUILD.gn

@@ -491,6 +491,7 @@ if (rtc_include_tests) {
       "video_coding/codecs/vp8/simulcast_unittest.h",
       "video_coding/decoding_state_unittest.cc",
       "video_coding/frame_buffer2_unittest.cc",
+      "video_coding/h264_sps_pps_tracker_unittest.cc",
       "video_coding/histogram_unittest.cc",
       "video_coding/include/mock/mock_vcm_callbacks.h",
       "video_coding/jitter_buffer_unittest.cc",

webrtc/modules/include/module_common_types.h

@@ -264,6 +264,10 @@ struct NaluInfo {
   uint8_t type;
   int sps_id;
   int pps_id;
+
+  // Offset and size are only valid for non-FuA packets.
+  size_t offset;
+  size_t size;
 };
 
 const size_t kMaxNalusPerPacket = 10;

webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc

@@ -411,6 +411,8 @@ bool RtpDepacketizerH264::ProcessStapAOrSingleNalu(
 
     NaluInfo nalu;
     nalu.type = payload_data[start_offset] & kTypeMask;
+    nalu.offset = start_offset;
+    nalu.size = end_offset - start_offset;
     nalu.sps_id = -1;
     nalu.pps_id = -1;
     start_offset += H264::kNaluTypeSize;

webrtc/modules/video_coding/BUILD.gn

@@ -29,6 +29,8 @@ rtc_static_library("video_coding") {
     "generic_decoder.h",
     "generic_encoder.cc",
     "generic_encoder.h",
+    "h264_sps_pps_tracker.cc",
+    "h264_sps_pps_tracker.h",
     "histogram.cc",
     "histogram.h",
     "include/video_coding.h",

webrtc/modules/video_coding/h264_sps_pps_tracker.cc

@@ -0,0 +1,182 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h"
+
+#include <string>
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/common_video/h264/h264_common.h"
+#include "webrtc/modules/video_coding/frame_object.h"
+#include "webrtc/modules/video_coding/packet_buffer.h"
+
+namespace webrtc {
+namespace video_coding {
+
+namespace {
+const uint8_t start_code_h264[] = {0, 0, 0, 1};
+}  // namespace
+
+bool H264SpsPpsTracker::CopyAndFixBitstream(VCMPacket* packet) {
+  RTC_DCHECK(packet->codec == kVideoCodecH264);
+
+  const uint8_t* data = packet->dataPtr;
+  const size_t data_size = packet->sizeBytes;
+  const RTPVideoHeader& video_header = packet->video_header;
+  const RTPVideoHeaderH264& codec_header = video_header.codecHeader.H264;
+
+  // Packets that only contains SPS/PPS are not decodable by themselves, and
+  // to avoid frames being created containing only these two nalus we don't
+  // insert them into the PacketBuffer. Instead we save the SPS/PPS and
+  // prepend the bitstream of first packet of an IDR referring to the
+  // corresponding SPS/PPS id.
+  bool insert_packet = codec_header.nalus_length == 0 ? true : false;
+
+  int pps_id = -1;
+  size_t required_size = 0;
+  for (size_t i = 0; i < codec_header.nalus_length; ++i) {
+    const NaluInfo& nalu = codec_header.nalus[i];
+    switch (nalu.type) {
+      case H264::NaluType::kSps: {
+        // Save SPS.
+        sps_data_[nalu.sps_id].size = nalu.size;
+        sps_data_[nalu.sps_id].data.reset(new uint8_t[nalu.size]);
+        memcpy(sps_data_[nalu.sps_id].data.get(), data + nalu.offset,
+               nalu.size);
+        break;
+      }
+      case H264::NaluType::kPps: {
+        // Save PPS.
+        pps_data_[nalu.pps_id].sps_id = nalu.sps_id;
+        pps_data_[nalu.pps_id].size = nalu.size;
+        pps_data_[nalu.pps_id].data.reset(new uint8_t[nalu.size]);
+        memcpy(pps_data_[nalu.pps_id].data.get(), data + nalu.offset,
+               nalu.size);
+        break;
+      }
+      case H264::NaluType::kIdr: {
+        // If this is the first packet of an IDR, make sure we have the required
+        // SPS/PPS and also calculate how much extra space we need in the buffer
+        // to prepend the SPS/PPS to the bitstream with start codes.
+        if (video_header.isFirstPacket) {
+          if (nalu.pps_id == -1) {
+            LOG(LS_WARNING) << "No PPS id in IDR nalu.";
+            return false;
+          }
+
+          auto pps = pps_data_.find(nalu.pps_id);
+          if (pps == pps_data_.end()) {
+            LOG(LS_WARNING) << "No PPS with id << " << nalu.pps_id
+                            << " received";
+            return false;
+          }
+
+          auto sps = sps_data_.find(pps->second.sps_id);
+          if (sps == sps_data_.end()) {
+            LOG(LS_WARNING) << "No SPS with id << "
+                            << pps_data_[nalu.pps_id].sps_id << " received";
+            return false;
+          }
+
+          pps_id = nalu.pps_id;
+          required_size += pps->second.size + sizeof(start_code_h264);
+          required_size += sps->second.size + sizeof(start_code_h264);
+        }
+        FALLTHROUGH();
+      }
+      default: {
+        // Something other than an SPS/PPS nalu in this packet, then it should
+        // be inserted into the PacketBuffer.
+        insert_packet = true;
+      }
+    }
+  }
+
+  if (!insert_packet)
+    return false;
+
+  // Calculate how much space we need for the rest of the bitstream.
+  if (codec_header.packetization_type == kH264StapA) {
+    const uint8_t* nalu_ptr = data + 1;
+    while (nalu_ptr < data + data_size) {
+      RTC_DCHECK(video_header.isFirstPacket);
+      required_size += sizeof(start_code_h264);
+
+      // The first two bytes describe the length of a segment.
+      uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
+      nalu_ptr += 2;
+
+      required_size += segment_length;
+      nalu_ptr += segment_length;
+    }
+  } else {
+    if (video_header.isFirstPacket)
+      required_size += sizeof(start_code_h264);
+    required_size += data_size;
+  }
+
+  // Then we copy to the new buffer.
+  uint8_t* buffer = new uint8_t[required_size];
+  uint8_t* insert_at = buffer;
+
+  // If pps_id != -1 then we have the SPS/PPS and they should be prepended
+  // to the bitstream with start codes inserted.
+  if (pps_id != -1) {
+    // Insert SPS.
+    memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+    insert_at += sizeof(start_code_h264);
+    memcpy(insert_at, sps_data_[pps_data_[pps_id].sps_id].data.get(),
+           sps_data_[pps_data_[pps_id].sps_id].size);
+    insert_at += sps_data_[pps_data_[pps_id].sps_id].size;
+
+    // Insert PPS.
+    memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+    insert_at += sizeof(start_code_h264);
+    memcpy(insert_at, pps_data_[pps_id].data.get(), pps_data_[pps_id].size);
+    insert_at += pps_data_[pps_id].size;
+  }
+
+  // Copy the rest of the bitstream and insert start codes.
+  if (codec_header.packetization_type == kH264StapA) {
+    const uint8_t* nalu_ptr = data + 1;
+    while (nalu_ptr < data + data_size) {
+      memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+      insert_at += sizeof(start_code_h264);
+
+      // The first two bytes describe the length of a segment.
+      uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
+      nalu_ptr += 2;
+
+      size_t copy_end = nalu_ptr - data + segment_length;
+      if (copy_end > data_size) {
+        delete[] buffer;
+        return false;
+      }
+
+      memcpy(insert_at, nalu_ptr, segment_length);
+      insert_at += segment_length;
+      nalu_ptr += segment_length;
+    }
+  } else {
+    if (video_header.isFirstPacket) {
+      memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+      insert_at += sizeof(start_code_h264);
+    }
+    memcpy(insert_at, data, data_size);
+  }
+
+  packet->dataPtr = buffer;
+  packet->sizeBytes = required_size;
+  return true;
+}
+
+}  // namespace video_coding
+}  // namespace webrtc

webrtc/modules/video_coding/h264_sps_pps_tracker.h

@@ -0,0 +1,49 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_
+#define WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_
+
+#include <cstdint>
+#include <map>
+#include <memory>
+
+#include "webrtc/modules/include/module_common_types.h"
+
+namespace webrtc {
+
+class VCMPacket;
+
+namespace video_coding {
+
+class H264SpsPpsTracker {
+ public:
+  bool CopyAndFixBitstream(VCMPacket* packet);
+
+ private:
+  struct PpsInfo {
+    int sps_id = -1;
+    size_t size = 0;
+    std::unique_ptr<uint8_t[]> data;
+  };
+
+  struct SpsInfo {
+    size_t size = 0;
+    std::unique_ptr<uint8_t[]> data;
+  };
+
+  std::map<uint32_t, PpsInfo> pps_data_;
+  std::map<uint32_t, SpsInfo> sps_data_;
+};
+
+}  // namespace video_coding
+}  // namespace webrtc
+
+#endif  // WEBRTC_MODULES_VIDEO_CODING_H264_SPS_PPS_TRACKER_H_

webrtc/modules/video_coding/h264_sps_pps_tracker_unittest.cc

@@ -0,0 +1,260 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h"
+
+#include <vector>
+
+#include "webrtc/modules/video_coding/packet.h"
+#include "webrtc/test/gtest.h"
+#include "webrtc/common_video/h264/h264_common.h"
+
+namespace webrtc {
+namespace video_coding {
+
+namespace {
+const uint8_t start_code[] = {0, 0, 0, 1};
+}  // namespace
+
+class TestH264SpsPpsTracker : public ::testing::Test {
+ public:
+  VCMPacket GetDefaultPacket() {
+    VCMPacket packet;
+    packet.codec = kVideoCodecH264;
+    packet.video_header.codecHeader.H264.nalus_length = 0;
+    packet.video_header.isFirstPacket = false;
+    packet.video_header.codecHeader.H264.packetization_type = kH264SingleNalu;
+
+    return packet;
+  }
+
+  void AddSps(VCMPacket* packet, int sps_id, std::vector<uint8_t>* data) {
+    NaluInfo info;
+    info.type = H264::NaluType::kSps;
+    info.sps_id = sps_id;
+    info.pps_id = -1;
+    info.offset = data->size();
+    info.size = 2;
+    data->push_back(H264::NaluType::kSps);
+    data->push_back(sps_id);  // The sps data, just a single byte.
+
+    packet->video_header.codecHeader.H264
+        .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info;
+  }
+
+  void AddPps(VCMPacket* packet,
+              int sps_id,
+              int pps_id,
+              std::vector<uint8_t>* data) {
+    NaluInfo info;
+    info.type = H264::NaluType::kPps;
+    info.sps_id = sps_id;
+    info.pps_id = pps_id;
+    info.offset = data->size();
+    info.size = 2;
+    data->push_back(H264::NaluType::kPps);
+    data->push_back(pps_id);  // The pps data, just a single byte.
+
+    packet->video_header.codecHeader.H264
+        .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info;
+  }
+
+  void AddIdr(VCMPacket* packet, int pps_id) {
+    NaluInfo info;
+    info.type = H264::NaluType::kIdr;
+    info.sps_id = -1;
+    info.pps_id = pps_id;
+
+    packet->video_header.codecHeader.H264
+        .nalus[packet->video_header.codecHeader.H264.nalus_length++] = info;
+  }
+
+ protected:
+  H264SpsPpsTracker tracker_;
+};
+
+TEST_F(TestH264SpsPpsTracker, NoNalus) {
+  uint8_t data[] = {1, 2, 3};
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.codecHeader.H264.packetization_type = kH264FuA;
+  packet.dataPtr = data;
+  packet.sizeBytes = sizeof(data);
+
+  EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+  EXPECT_EQ(memcmp(packet.dataPtr, data, sizeof(data)), 0);
+  delete[] packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, FuAFirstPacket) {
+  uint8_t data[] = {1, 2, 3};
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.codecHeader.H264.packetization_type = kH264FuA;
+  packet.video_header.isFirstPacket = true;
+  packet.dataPtr = data;
+  packet.sizeBytes = sizeof(data);
+
+  EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+  std::vector<uint8_t> expected;
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {1, 2, 3});
+  EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0);
+  delete[] packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, StapAIncorrectSegmentLength) {
+  uint8_t data[] = {0, 0, 2, 0};
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.codecHeader.H264.packetization_type = kH264StapA;
+  packet.video_header.isFirstPacket = true;
+  packet.dataPtr = data;
+  packet.sizeBytes = sizeof(data);
+
+  EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet));
+}
+
+TEST_F(TestH264SpsPpsTracker, NoNalusFirstPacket) {
+  uint8_t data[] = {1, 2, 3};
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.isFirstPacket = true;
+  packet.dataPtr = data;
+  packet.sizeBytes = sizeof(data);
+
+  EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+  std::vector<uint8_t> expected;
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {1, 2, 3});
+  EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0);
+  delete[] packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, IdrNoSpsPpsInserted) {
+  std::vector<uint8_t> data = {1, 2, 3};
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.codecHeader.H264.packetization_type = kH264FuA;
+
+  AddIdr(&packet, 0);
+  packet.dataPtr = data.data();
+  packet.sizeBytes = data.size();
+
+  EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+  EXPECT_EQ(memcmp(packet.dataPtr, data.data(), data.size()), 0);
+  delete[] packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoSpsPpsInserted) {
+  std::vector<uint8_t> data = {1, 2, 3};
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.isFirstPacket = true;
+
+  AddIdr(&packet, 0);
+  packet.dataPtr = data.data();
+  packet.sizeBytes = data.size();
+
+  EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet));
+}
+
+TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoPpsInserted) {
+  std::vector<uint8_t> data = {1, 2, 3};
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.isFirstPacket = true;
+
+  AddSps(&packet, 0, &data);
+  AddIdr(&packet, 0);
+  packet.dataPtr = data.data();
+  packet.sizeBytes = data.size();
+
+  EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet));
+}
+
+TEST_F(TestH264SpsPpsTracker, IdrFirstPacketNoSpsInserted) {
+  std::vector<uint8_t> data = {1, 2, 3};
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.isFirstPacket = true;
+
+  AddPps(&packet, 0, 0, &data);
+  AddIdr(&packet, 0);
+  packet.dataPtr = data.data();
+  packet.sizeBytes = data.size();
+
+  EXPECT_FALSE(tracker_.CopyAndFixBitstream(&packet));
+}
+
+TEST_F(TestH264SpsPpsTracker, SpsPpsPacketThenIdrFirstPacket) {
+  std::vector<uint8_t> data;
+  VCMPacket sps_pps_packet = GetDefaultPacket();
+
+  // Insert SPS/PPS
+  AddSps(&sps_pps_packet, 0, &data);
+  AddPps(&sps_pps_packet, 0, 1, &data);
+  sps_pps_packet.dataPtr = data.data();
+  sps_pps_packet.sizeBytes = data.size();
+  EXPECT_FALSE(tracker_.CopyAndFixBitstream(&sps_pps_packet));
+  data.clear();
+
+  // Insert first packet of the IDR
+  VCMPacket idr_packet = GetDefaultPacket();
+  idr_packet.video_header.isFirstPacket = true;
+  AddIdr(&idr_packet, 1);
+  data.insert(data.end(), {1, 2, 3});
+  idr_packet.dataPtr = data.data();
+  idr_packet.sizeBytes = data.size();
+  EXPECT_TRUE(tracker_.CopyAndFixBitstream(&idr_packet));
+
+  std::vector<uint8_t> expected;
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {H264::NaluType::kSps, 0});
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {H264::NaluType::kPps, 1});
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {1, 2, 3});
+  EXPECT_EQ(memcmp(idr_packet.dataPtr, expected.data(), expected.size()), 0);
+  delete[] idr_packet.dataPtr;
+}
+
+TEST_F(TestH264SpsPpsTracker, SpsPpsIdrInStapA) {
+  std::vector<uint8_t> data;
+  VCMPacket packet = GetDefaultPacket();
+  packet.video_header.codecHeader.H264.packetization_type = kH264StapA;
+  packet.video_header.isFirstPacket = true;  // Always true for StapA
+
+  data.insert(data.end(), {0});     // First byte is ignored
+  data.insert(data.end(), {0, 2});  // Length of segment
+  AddSps(&packet, 13, &data);
+  data.insert(data.end(), {0, 2});  // Length of segment
+  AddPps(&packet, 13, 27, &data);
+  data.insert(data.end(), {0, 5});  // Length of segment
+  AddIdr(&packet, 27);
+  data.insert(data.end(), {1, 2, 3, 2, 1});
+
+  packet.dataPtr = data.data();
+  packet.sizeBytes = data.size();
+  EXPECT_TRUE(tracker_.CopyAndFixBitstream(&packet));
+
+  std::vector<uint8_t> expected;
+  // The SPS/PPS is repeated because this packet both contains the SPS/PPS
+  // and it is the first packet of an IDR, which will cause the SPS/PPS to be
+  // prepended to the bitstream.
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {H264::NaluType::kSps, 13});
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {H264::NaluType::kPps, 27});
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {H264::NaluType::kSps, 13});
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {H264::NaluType::kPps, 27});
+  expected.insert(expected.end(), start_code, start_code + sizeof(start_code));
+  expected.insert(expected.end(), {1, 2, 3, 2, 1});
+
+  EXPECT_EQ(memcmp(packet.dataPtr, expected.data(), expected.size()), 0);
+  delete[] packet.dataPtr;
+}
+
+}  // namespace video_coding
+}  // namespace webrtc