Fix H26xPacketBuffer Behavior Changes for Padding Packets

This commit fixes the issue of H26xPacketBuffer not supporting the
 RTP padding packet.

Bug: webrtc:383841353
Change-Id: Ibd87cd9c18577d990fa56a2fdfed1552d33b58a2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/371840
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43586}

modules/video_coding/h26x_packet_buffer.cc

@@ -103,6 +103,24 @@ H26xPacketBuffer::H26xPacketBuffer(bool h264_idr_only_keyframes_allowed)
   last_continuous_in_sequence_.fill(std::numeric_limits<int64_t>::min());
 }
 
+H26xPacketBuffer::InsertResult H26xPacketBuffer::InsertPadding(
+    uint16_t unwrapped_seq_num) {
+  int64_t* last_continuous_unwrapped_seq_num =
+      GetContinuousSequence(last_continuous_in_sequence_, unwrapped_seq_num);
+  if (last_continuous_unwrapped_seq_num == nullptr) {
+    last_continuous_in_sequence_[last_continuous_in_sequence_index_] =
+        unwrapped_seq_num;
+    last_continuous_unwrapped_seq_num =
+        &last_continuous_in_sequence_[last_continuous_in_sequence_index_];
+    last_continuous_in_sequence_index_ =
+        (last_continuous_in_sequence_index_ + 1) %
+        last_continuous_in_sequence_.size();
+  } else {
+    *last_continuous_unwrapped_seq_num = unwrapped_seq_num;
+  }
+  return {};
+}
+
 H26xPacketBuffer::InsertResult H26xPacketBuffer::InsertPacket(
     std::unique_ptr<Packet> packet) {
   RTC_DCHECK(packet->video_header.codec == kVideoCodecH264 ||

modules/video_coding/h26x_packet_buffer.h

@@ -37,6 +37,7 @@ class H26xPacketBuffer {
 
   ABSL_MUST_USE_RESULT InsertResult
   InsertPacket(std::unique_ptr<Packet> packet);
+  ABSL_MUST_USE_RESULT InsertResult InsertPadding(uint16_t unwrapped_seq_num);
 
   // Out of band supplied codec parameters for H.264.
   void SetSpropParameterSets(const std::string& sprop_parameter_sets);

video/rtp_video_stream_receiver2.cc

@@ -363,6 +363,7 @@ void RtpVideoStreamReceiver2::AddReceiveCodec(
       payload_type, raw_payload ? std::make_unique<VideoRtpDepacketizerRaw>()
                                 : CreateVideoRtpDepacketizer(video_codec));
   pt_codec_params_.emplace(payload_type, codec_params);
+  pt_codec_.emplace(payload_type, video_codec);
 }
 
 void RtpVideoStreamReceiver2::RemoveReceiveCodecs() {
@@ -372,6 +373,7 @@ void RtpVideoStreamReceiver2::RemoveReceiveCodecs() {
   payload_type_map_.clear();
   packet_buffer_.ResetSpsPpsIdrIsH264Keyframe();
   h26x_packet_buffer_.reset();
+  pt_codec_.clear();
 }
 
 std::optional<Syncable::Info> RtpVideoStreamReceiver2::GetSyncInfo() const {
@@ -692,7 +694,8 @@ bool RtpVideoStreamReceiver2::OnReceivedPayloadData(
   packet->times_nacked = times_nacked;
 
   if (codec_payload.size() == 0) {
-    NotifyReceiverOfEmptyPacket(packet->seq_num());
+    NotifyReceiverOfEmptyPacket(packet->seq_num(),
+                                IsH26xPayloadType(packet->payload_type));
     rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
     return false;
   }
@@ -1108,6 +1111,15 @@ RtpVideoStreamReceiver2::GetSenderReportStats() const {
   return rtp_rtcp_->GetSenderReportStats();
 }
 
+bool RtpVideoStreamReceiver2::IsH26xPayloadType(uint8_t payload_type) const {
+  RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
+  auto it = pt_codec_.find(payload_type);
+  if (it == pt_codec_.end()) {
+    return false;
+  }
+  return it->second == kVideoCodecH264 || it->second == kVideoCodecH265;
+}
+
 void RtpVideoStreamReceiver2::ManageFrame(
     std::unique_ptr<RtpFrameObject> frame) {
   RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
@@ -1121,7 +1133,8 @@ void RtpVideoStreamReceiver2::ReceivePacket(const RtpPacketReceived& packet) {
     // Padding or keep-alive packet.
     // TODO(nisse): Could drop empty packets earlier, but need to figure out how
     // they should be counted in stats.
-    NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
+    NotifyReceiverOfEmptyPacket(packet.SequenceNumber(),
+                                IsH26xPayloadType(packet.PayloadType()));
     return;
   }
   if (packet.PayloadType() == red_payload_type_) {
@@ -1191,7 +1204,8 @@ void RtpVideoStreamReceiver2::ParseAndHandleEncapsulatingHeader(
   if (packet.payload()[0] == ulpfec_receiver_->ulpfec_payload_type()) {
     // Notify video_receiver about received FEC packets to avoid NACKing these
     // packets.
-    NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
+    NotifyReceiverOfEmptyPacket(packet.SequenceNumber(),
+                                IsH26xPayloadType(packet.PayloadType()));
   }
   if (ulpfec_receiver_->AddReceivedRedPacket(packet)) {
     ulpfec_receiver_->ProcessReceivedFec();
@@ -1201,13 +1215,18 @@ void RtpVideoStreamReceiver2::ParseAndHandleEncapsulatingHeader(
 // In the case of a video stream without picture ids and no rtx the
 // RtpFrameReferenceFinder will need to know about padding to
 // correctly calculate frame references.
-void RtpVideoStreamReceiver2::NotifyReceiverOfEmptyPacket(uint16_t seq_num) {
+void RtpVideoStreamReceiver2::NotifyReceiverOfEmptyPacket(uint16_t seq_num,
+                                                          bool is_h26x) {
   RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
   RTC_DCHECK_RUN_ON(&worker_task_checker_);
 
   OnCompleteFrames(reference_finder_->PaddingReceived(seq_num));
 
-  OnInsertedPacket(packet_buffer_.InsertPadding(seq_num));
+  if (is_h26x && h26x_packet_buffer_) {
+    OnInsertedPacket(h26x_packet_buffer_->InsertPadding(seq_num));
+  } else {
+    OnInsertedPacket(packet_buffer_.InsertPadding(seq_num));
+  }
   if (nack_module_) {
     nack_module_->OnReceivedPacket(seq_num, /*is_recovered=*/false);
   }

video/rtp_video_stream_receiver2.h

@@ -300,7 +300,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender,
   // This function assumes that it's being called from only one thread.
   void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet)
       RTC_RUN_ON(packet_sequence_checker_);
-  void NotifyReceiverOfEmptyPacket(uint16_t seq_num)
+  void NotifyReceiverOfEmptyPacket(uint16_t seq_num, bool is_h26x)
       RTC_RUN_ON(packet_sequence_checker_);
   bool IsRedEnabled() const;
   void InsertSpsPpsIntoTracker(uint8_t payload_type)
@@ -320,6 +320,9 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender,
                           FrameInstrumentationData>& frame_instrumentation_data,
       int spatial_idx);
 
+  bool IsH26xPayloadType(uint8_t payload_type) const
+      RTC_RUN_ON(packet_sequence_checker_);
+
   const Environment env_;
   TaskQueueBase* const worker_queue_;
 
@@ -410,6 +413,11 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender,
   // Maps a payload type to a map of out-of-band supplied codec parameters.
   std::map<uint8_t, webrtc::CodecParameterMap> pt_codec_params_
       RTC_GUARDED_BY(packet_sequence_checker_);
+
+  // Maps payload type to the VideoCodecType.
+  std::map<uint8_t, webrtc::VideoCodecType> pt_codec_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
   int16_t last_payload_type_ RTC_GUARDED_BY(packet_sequence_checker_) = -1;
 
   bool has_received_frame_ RTC_GUARDED_BY(packet_sequence_checker_);

video/rtp_video_stream_receiver2_unittest.cc

@@ -956,7 +956,19 @@ TEST_P(RtpVideoStreamReceiver2TestH264, ForceSpsPpsIdrIsKeyframe) {
                                                     idr_video_header, 0);
 }
 
-TEST_F(RtpVideoStreamReceiver2Test, PaddingInMediaStream) {
+class RtpVideoStreamReceiver2TestPadding
+    : public RtpVideoStreamReceiver2Test,
+      public ::testing::WithParamInterface<std::string> {
+ protected:
+  RtpVideoStreamReceiver2TestPadding()
+      : RtpVideoStreamReceiver2Test(GetParam()) {}
+};
+
+INSTANTIATE_TEST_SUITE_P(PaddingInMediaStreamAndH26xPacketBuffer,
+                         RtpVideoStreamReceiver2TestPadding,
+                         Values("", "WebRTC-Video-H26xPacketBuffer/Enabled/"));
+
+TEST_P(RtpVideoStreamReceiver2TestPadding, PaddingInMediaStream) {
   RtpPacketReceived rtp_packet;
   RTPVideoHeader video_header = GetDefaultH264VideoHeader();
   rtc::CopyOnWriteBuffer data({'1', '2', '3'});
@@ -993,6 +1005,64 @@ TEST_F(RtpVideoStreamReceiver2Test, PaddingInMediaStream) {
                                                     video_header, 0);
 }
 
+TEST_P(RtpVideoStreamReceiver2TestPadding, EmptyPaddingInMediaStream) {
+  constexpr int kH264PayloadType = 98;
+  RtpPacketReceived rtp_packet_idr, rtp_packet_padding, rtp_packet_slice;
+  // Example Stap-A packet with SPS, PPS, and IDR.
+  std::vector<uint8_t> raw_rtp_with_sps_pps_idr{
+      0x80, 0xe2, 0x13, 0xba, 0x87, 0xa0, 0x0a, 0x8a, 0x00, 0x00, 0x6f,
+      0x00, 0x78, 0x00, 0x19, 0x67, 0x42, 0x40, 0x29, 0x95, 0xb8, 0x78,
+      0x2f, 0xf9, 0x70, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00,
+      0x03, 0x00, 0x78, 0x8d, 0xa1, 0xc3, 0x2e, 0x00, 0x04, 0x68, 0xce,
+      0x3c, 0x80, 0x00, 0x07, 0x05, 0x88, 0x80, 0x03, 0x53, 0xff, 0xff};
+  // Example Empty padding packet next Idr.
+  std::vector<uint8_t> raw_rtp_empty_padding{
+      0x80, 0x62, 0x13, 0xbb, 0x87, 0xa0, 0x21, 0x0a, 0x00, 0x00, 0x6f, 0x00};
+  // Example Single NALU packet with slice.
+  std::vector<uint8_t> raw_rtp_slice(
+      {0x80, 0xE2, 0x13, 0xbc, 0x87, 0xa0, 0x21, 0x0a, 0x00, 0x00, 0x6f,
+       0x00, 0x01, 0x9a, 0x02, 0x3f, 0xc1, 0x48, 0x9a, 0xeb, 0xea, 0xff});
+
+  // Example EncodedFrame with SPS, PPS, and IDR.
+  std::vector<uint8_t> expect_frame_with_sps_pps_idr{
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0x40, 0x29, 0x95, 0xb8, 0x78, 0x2f,
+      0xf9, 0x70, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00,
+      0x78, 0x8d, 0xa1, 0xc3, 0x2e, 0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x3c,
+      0x80, 0x00, 0x00, 0x00, 0x01, 0x05, 0x88, 0x80, 0x03, 0x53, 0xff, 0xff};
+  // Example EncodedFrame with slice.
+  std::vector<uint8_t> expect_frame_with_slice{0x00, 0x00, 0x00, 0x01, 0x01,
+                                               0x9a, 0x02, 0x3f, 0xc1, 0x48,
+                                               0x9a, 0xeb, 0xea, 0xff};
+  rtp_packet_idr.Parse(raw_rtp_with_sps_pps_idr.data(),
+                       raw_rtp_with_sps_pps_idr.size());
+  rtp_packet_padding.Parse(raw_rtp_empty_padding.data(),
+                           raw_rtp_empty_padding.size());
+  rtp_packet_slice.Parse(raw_rtp_slice.data(), raw_rtp_slice.size());
+
+  // Prepare the receiver for H264.
+  webrtc::CodecParameterMap codec_params;
+  rtp_video_stream_receiver_->AddReceiveCodec(kH264PayloadType, kVideoCodecH264,
+                                              codec_params, false);
+  rtp_video_stream_receiver_->StartReceive();
+
+  // Expect IDR frame.
+  mock_on_complete_frame_callback_.AppendExpectedBitstream(
+      expect_frame_with_sps_pps_idr.data(),
+      expect_frame_with_sps_pps_idr.size());
+  EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+
+  rtp_video_stream_receiver_->OnRtpPacket(rtp_packet_idr);
+
+  rtp_video_stream_receiver_->OnRtpPacket(rtp_packet_padding);
+
+  // Expect single NALU frame.
+  mock_on_complete_frame_callback_.ClearExpectedBitstream();
+  mock_on_complete_frame_callback_.AppendExpectedBitstream(
+      expect_frame_with_slice.data(), expect_frame_with_slice.size());
+  EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
+  rtp_video_stream_receiver_->OnRtpPacket(rtp_packet_slice);
+}
+
 TEST_F(RtpVideoStreamReceiver2Test, RequestKeyframeIfFirstFrameIsDelta) {
   RtpPacketReceived rtp_packet;
   rtp_packet.SetPayloadType(kPayloadType);