diff --git a/modules/video_coding/h26x_packet_buffer.cc b/modules/video_coding/h26x_packet_buffer.cc index 9020bb7c1e..2afe1426b1 100644 --- a/modules/video_coding/h26x_packet_buffer.cc +++ b/modules/video_coding/h26x_packet_buffer.cc @@ -103,6 +103,24 @@ H26xPacketBuffer::H26xPacketBuffer(bool h264_idr_only_keyframes_allowed) last_continuous_in_sequence_.fill(std::numeric_limits::min()); } +H26xPacketBuffer::InsertResult H26xPacketBuffer::InsertPadding( + uint16_t unwrapped_seq_num) { + int64_t* last_continuous_unwrapped_seq_num = + GetContinuousSequence(last_continuous_in_sequence_, unwrapped_seq_num); + if (last_continuous_unwrapped_seq_num == nullptr) { + last_continuous_in_sequence_[last_continuous_in_sequence_index_] = + unwrapped_seq_num; + last_continuous_unwrapped_seq_num = + &last_continuous_in_sequence_[last_continuous_in_sequence_index_]; + last_continuous_in_sequence_index_ = + (last_continuous_in_sequence_index_ + 1) % + last_continuous_in_sequence_.size(); + } else { + *last_continuous_unwrapped_seq_num = unwrapped_seq_num; + } + return {}; +} + H26xPacketBuffer::InsertResult H26xPacketBuffer::InsertPacket( std::unique_ptr packet) { RTC_DCHECK(packet->video_header.codec == kVideoCodecH264 || diff --git a/modules/video_coding/h26x_packet_buffer.h b/modules/video_coding/h26x_packet_buffer.h index 62b97d3d42..c3db72bdd6 100644 --- a/modules/video_coding/h26x_packet_buffer.h +++ b/modules/video_coding/h26x_packet_buffer.h @@ -37,6 +37,7 @@ class H26xPacketBuffer { ABSL_MUST_USE_RESULT InsertResult InsertPacket(std::unique_ptr packet); + ABSL_MUST_USE_RESULT InsertResult InsertPadding(uint16_t unwrapped_seq_num); // Out of band supplied codec parameters for H.264. void SetSpropParameterSets(const std::string& sprop_parameter_sets); diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index 3982cf11e3..2405d08401 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -363,6 +363,7 @@ void RtpVideoStreamReceiver2::AddReceiveCodec( payload_type, raw_payload ? std::make_unique() : CreateVideoRtpDepacketizer(video_codec)); pt_codec_params_.emplace(payload_type, codec_params); + pt_codec_.emplace(payload_type, video_codec); } void RtpVideoStreamReceiver2::RemoveReceiveCodecs() { @@ -372,6 +373,7 @@ void RtpVideoStreamReceiver2::RemoveReceiveCodecs() { payload_type_map_.clear(); packet_buffer_.ResetSpsPpsIdrIsH264Keyframe(); h26x_packet_buffer_.reset(); + pt_codec_.clear(); } std::optional RtpVideoStreamReceiver2::GetSyncInfo() const { @@ -692,7 +694,8 @@ bool RtpVideoStreamReceiver2::OnReceivedPayloadData( packet->times_nacked = times_nacked; if (codec_payload.size() == 0) { - NotifyReceiverOfEmptyPacket(packet->seq_num()); + NotifyReceiverOfEmptyPacket(packet->seq_num(), + IsH26xPayloadType(packet->payload_type)); rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); return false; } @@ -1108,6 +1111,15 @@ RtpVideoStreamReceiver2::GetSenderReportStats() const { return rtp_rtcp_->GetSenderReportStats(); } +bool RtpVideoStreamReceiver2::IsH26xPayloadType(uint8_t payload_type) const { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + auto it = pt_codec_.find(payload_type); + if (it == pt_codec_.end()) { + return false; + } + return it->second == kVideoCodecH264 || it->second == kVideoCodecH265; +} + void RtpVideoStreamReceiver2::ManageFrame( std::unique_ptr frame) { RTC_DCHECK_RUN_ON(&packet_sequence_checker_); @@ -1121,7 +1133,8 @@ void RtpVideoStreamReceiver2::ReceivePacket(const RtpPacketReceived& packet) { // Padding or keep-alive packet. // TODO(nisse): Could drop empty packets earlier, but need to figure out how // they should be counted in stats. - NotifyReceiverOfEmptyPacket(packet.SequenceNumber()); + NotifyReceiverOfEmptyPacket(packet.SequenceNumber(), + IsH26xPayloadType(packet.PayloadType())); return; } if (packet.PayloadType() == red_payload_type_) { @@ -1191,7 +1204,8 @@ void RtpVideoStreamReceiver2::ParseAndHandleEncapsulatingHeader( if (packet.payload()[0] == ulpfec_receiver_->ulpfec_payload_type()) { // Notify video_receiver about received FEC packets to avoid NACKing these // packets. - NotifyReceiverOfEmptyPacket(packet.SequenceNumber()); + NotifyReceiverOfEmptyPacket(packet.SequenceNumber(), + IsH26xPayloadType(packet.PayloadType())); } if (ulpfec_receiver_->AddReceivedRedPacket(packet)) { ulpfec_receiver_->ProcessReceivedFec(); @@ -1201,13 +1215,18 @@ void RtpVideoStreamReceiver2::ParseAndHandleEncapsulatingHeader( // In the case of a video stream without picture ids and no rtx the // RtpFrameReferenceFinder will need to know about padding to // correctly calculate frame references. -void RtpVideoStreamReceiver2::NotifyReceiverOfEmptyPacket(uint16_t seq_num) { +void RtpVideoStreamReceiver2::NotifyReceiverOfEmptyPacket(uint16_t seq_num, + bool is_h26x) { RTC_DCHECK_RUN_ON(&packet_sequence_checker_); RTC_DCHECK_RUN_ON(&worker_task_checker_); OnCompleteFrames(reference_finder_->PaddingReceived(seq_num)); - OnInsertedPacket(packet_buffer_.InsertPadding(seq_num)); + if (is_h26x && h26x_packet_buffer_) { + OnInsertedPacket(h26x_packet_buffer_->InsertPadding(seq_num)); + } else { + OnInsertedPacket(packet_buffer_.InsertPadding(seq_num)); + } if (nack_module_) { nack_module_->OnReceivedPacket(seq_num, /*is_recovered=*/false); } diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h index fc171a87d2..34b00dfa1a 100644 --- a/video/rtp_video_stream_receiver2.h +++ b/video/rtp_video_stream_receiver2.h @@ -300,7 +300,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, // This function assumes that it's being called from only one thread. void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet) RTC_RUN_ON(packet_sequence_checker_); - void NotifyReceiverOfEmptyPacket(uint16_t seq_num) + void NotifyReceiverOfEmptyPacket(uint16_t seq_num, bool is_h26x) RTC_RUN_ON(packet_sequence_checker_); bool IsRedEnabled() const; void InsertSpsPpsIntoTracker(uint8_t payload_type) @@ -320,6 +320,9 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, FrameInstrumentationData>& frame_instrumentation_data, int spatial_idx); + bool IsH26xPayloadType(uint8_t payload_type) const + RTC_RUN_ON(packet_sequence_checker_); + const Environment env_; TaskQueueBase* const worker_queue_; @@ -410,6 +413,11 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, // Maps a payload type to a map of out-of-band supplied codec parameters. std::map pt_codec_params_ RTC_GUARDED_BY(packet_sequence_checker_); + + // Maps payload type to the VideoCodecType. + std::map pt_codec_ + RTC_GUARDED_BY(packet_sequence_checker_); + int16_t last_payload_type_ RTC_GUARDED_BY(packet_sequence_checker_) = -1; bool has_received_frame_ RTC_GUARDED_BY(packet_sequence_checker_); diff --git a/video/rtp_video_stream_receiver2_unittest.cc b/video/rtp_video_stream_receiver2_unittest.cc index fed4adb710..07c23a4f33 100644 --- a/video/rtp_video_stream_receiver2_unittest.cc +++ b/video/rtp_video_stream_receiver2_unittest.cc @@ -956,7 +956,19 @@ TEST_P(RtpVideoStreamReceiver2TestH264, ForceSpsPpsIdrIsKeyframe) { idr_video_header, 0); } -TEST_F(RtpVideoStreamReceiver2Test, PaddingInMediaStream) { +class RtpVideoStreamReceiver2TestPadding + : public RtpVideoStreamReceiver2Test, + public ::testing::WithParamInterface { + protected: + RtpVideoStreamReceiver2TestPadding() + : RtpVideoStreamReceiver2Test(GetParam()) {} +}; + +INSTANTIATE_TEST_SUITE_P(PaddingInMediaStreamAndH26xPacketBuffer, + RtpVideoStreamReceiver2TestPadding, + Values("", "WebRTC-Video-H26xPacketBuffer/Enabled/")); + +TEST_P(RtpVideoStreamReceiver2TestPadding, PaddingInMediaStream) { RtpPacketReceived rtp_packet; RTPVideoHeader video_header = GetDefaultH264VideoHeader(); rtc::CopyOnWriteBuffer data({'1', '2', '3'}); @@ -993,6 +1005,64 @@ TEST_F(RtpVideoStreamReceiver2Test, PaddingInMediaStream) { video_header, 0); } +TEST_P(RtpVideoStreamReceiver2TestPadding, EmptyPaddingInMediaStream) { + constexpr int kH264PayloadType = 98; + RtpPacketReceived rtp_packet_idr, rtp_packet_padding, rtp_packet_slice; + // Example Stap-A packet with SPS, PPS, and IDR. + std::vector raw_rtp_with_sps_pps_idr{ + 0x80, 0xe2, 0x13, 0xba, 0x87, 0xa0, 0x0a, 0x8a, 0x00, 0x00, 0x6f, + 0x00, 0x78, 0x00, 0x19, 0x67, 0x42, 0x40, 0x29, 0x95, 0xb8, 0x78, + 0x2f, 0xf9, 0x70, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, + 0x03, 0x00, 0x78, 0x8d, 0xa1, 0xc3, 0x2e, 0x00, 0x04, 0x68, 0xce, + 0x3c, 0x80, 0x00, 0x07, 0x05, 0x88, 0x80, 0x03, 0x53, 0xff, 0xff}; + // Example Empty padding packet next Idr. + std::vector raw_rtp_empty_padding{ + 0x80, 0x62, 0x13, 0xbb, 0x87, 0xa0, 0x21, 0x0a, 0x00, 0x00, 0x6f, 0x00}; + // Example Single NALU packet with slice. + std::vector raw_rtp_slice( + {0x80, 0xE2, 0x13, 0xbc, 0x87, 0xa0, 0x21, 0x0a, 0x00, 0x00, 0x6f, + 0x00, 0x01, 0x9a, 0x02, 0x3f, 0xc1, 0x48, 0x9a, 0xeb, 0xea, 0xff}); + + // Example EncodedFrame with SPS, PPS, and IDR. + std::vector expect_frame_with_sps_pps_idr{ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0x40, 0x29, 0x95, 0xb8, 0x78, 0x2f, + 0xf9, 0x70, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x78, 0x8d, 0xa1, 0xc3, 0x2e, 0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x3c, + 0x80, 0x00, 0x00, 0x00, 0x01, 0x05, 0x88, 0x80, 0x03, 0x53, 0xff, 0xff}; + // Example EncodedFrame with slice. + std::vector expect_frame_with_slice{0x00, 0x00, 0x00, 0x01, 0x01, + 0x9a, 0x02, 0x3f, 0xc1, 0x48, + 0x9a, 0xeb, 0xea, 0xff}; + rtp_packet_idr.Parse(raw_rtp_with_sps_pps_idr.data(), + raw_rtp_with_sps_pps_idr.size()); + rtp_packet_padding.Parse(raw_rtp_empty_padding.data(), + raw_rtp_empty_padding.size()); + rtp_packet_slice.Parse(raw_rtp_slice.data(), raw_rtp_slice.size()); + + // Prepare the receiver for H264. + webrtc::CodecParameterMap codec_params; + rtp_video_stream_receiver_->AddReceiveCodec(kH264PayloadType, kVideoCodecH264, + codec_params, false); + rtp_video_stream_receiver_->StartReceive(); + + // Expect IDR frame. + mock_on_complete_frame_callback_.AppendExpectedBitstream( + expect_frame_with_sps_pps_idr.data(), + expect_frame_with_sps_pps_idr.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet_idr); + + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet_padding); + + // Expect single NALU frame. + mock_on_complete_frame_callback_.ClearExpectedBitstream(); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + expect_frame_with_slice.data(), expect_frame_with_slice.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet_slice); +} + TEST_F(RtpVideoStreamReceiver2Test, RequestKeyframeIfFirstFrameIsDelta) { RtpPacketReceived rtp_packet; rtp_packet.SetPayloadType(kPayloadType);