diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 9e5ec4a72d..4ca7266579 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -161,6 +161,8 @@ rtc_library("rtp_rtcp") { "source/forward_error_correction_internal.h", "source/packet_loss_stats.cc", "source/packet_loss_stats.h", + "source/packet_sequencer.cc", + "source/packet_sequencer.h", "source/receive_statistics_impl.cc", "source/receive_statistics_impl.h", "source/remote_ntp_time_estimator.cc", diff --git a/modules/rtp_rtcp/source/packet_sequencer.cc b/modules/rtp_rtcp/source/packet_sequencer.cc new file mode 100644 index 0000000000..03ea9b8154 --- /dev/null +++ b/modules/rtp_rtcp/source/packet_sequencer.cc @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/packet_sequencer.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { +// RED header is first byte of payload, if present. +constexpr size_t kRedForFecHeaderLength = 1; + +// Timestamps use a 90kHz clock. +constexpr uint32_t kTimestampTicksPerMs = 90; +} // namespace + +PacketSequencer::PacketSequencer(uint32_t media_ssrc, + uint32_t rtx_ssrc, + bool require_marker_before_media_padding, + Clock* clock) + : media_ssrc_(media_ssrc), + rtx_ssrc_(rtx_ssrc), + require_marker_before_media_padding_(require_marker_before_media_padding), + clock_(clock), + media_sequence_number_(0), + rtx_sequence_number_(0), + last_payload_type_(-1), + last_rtp_timestamp_(0), + last_capture_time_ms_(0), + last_timestamp_time_ms_(0), + last_packet_marker_bit_(false) {} + +bool PacketSequencer::Sequence(RtpPacketToSend& packet) { + if (packet.packet_type() == RtpPacketMediaType::kPadding && + !PopulatePaddingFields(packet)) { + // This padding packet can't be sent with current state, return without + // updating the sequence number. + return false; + } + + if (packet.Ssrc() == media_ssrc_) { + packet.SetSequenceNumber(media_sequence_number_++); + if (packet.packet_type() != RtpPacketMediaType::kPadding) { + UpdateLastPacketState(packet); + } + return true; + } + + RTC_DCHECK_EQ(packet.Ssrc(), rtx_ssrc_); + packet.SetSequenceNumber(rtx_sequence_number_++); + return true; +} + +void PacketSequencer::SetRtpState(const RtpState& state) { + media_sequence_number_ = state.sequence_number; + last_rtp_timestamp_ = state.timestamp; + last_capture_time_ms_ = state.capture_time_ms; + last_timestamp_time_ms_ = state.last_timestamp_time_ms; +} + +void PacketSequencer::PupulateRtpState(RtpState& state) const { + state.sequence_number = media_sequence_number_; + state.timestamp = last_rtp_timestamp_; + state.capture_time_ms = last_capture_time_ms_; + state.last_timestamp_time_ms = last_timestamp_time_ms_; +} + +void PacketSequencer::UpdateLastPacketState(const RtpPacketToSend& packet) { + // Remember marker bit to determine if padding can be inserted with + // sequence number following |packet|. + last_packet_marker_bit_ = packet.Marker(); + // Remember media payload type to use in the padding packet if rtx is + // disabled. + if (packet.is_red()) { + RTC_DCHECK_GE(packet.payload_size(), kRedForFecHeaderLength); + last_payload_type_ = packet.PayloadBuffer()[0]; + } else { + last_payload_type_ = packet.PayloadType(); + } + // Save timestamps to generate timestamp field and extensions for the padding. + last_rtp_timestamp_ = packet.Timestamp(); + last_timestamp_time_ms_ = clock_->TimeInMilliseconds(); + last_capture_time_ms_ = packet.capture_time_ms(); +} + +bool PacketSequencer::PopulatePaddingFields(RtpPacketToSend& packet) { + if (packet.Ssrc() == media_ssrc_) { + if (last_payload_type_ == -1) { + return false; + } + + // Without RTX we can't send padding in the middle of frames. + // For audio marker bits doesn't mark the end of a frame and frames + // are usually a single packet, so for now we don't apply this rule + // for audio. + if (require_marker_before_media_padding_ && !last_packet_marker_bit_) { + return false; + } + + packet.SetTimestamp(last_rtp_timestamp_); + packet.set_capture_time_ms(last_capture_time_ms_); + packet.SetPayloadType(last_payload_type_); + return true; + } + + RTC_DCHECK_EQ(packet.Ssrc(), rtx_ssrc_); + if (packet.payload_size() > 0) { + // This is payload padding packet, don't update timestamp fields. + return true; + } + + packet.SetTimestamp(last_rtp_timestamp_); + packet.set_capture_time_ms(last_capture_time_ms_); + + // Only change the timestamp of padding packets sent over RTX. + // Padding only packets over RTP has to be sent as part of a media + // frame (and therefore the same timestamp). + int64_t now_ms = clock_->TimeInMilliseconds(); + if (last_timestamp_time_ms_ > 0) { + packet.SetTimestamp(packet.Timestamp() + + (now_ms - last_timestamp_time_ms_) * + kTimestampTicksPerMs); + if (packet.capture_time_ms() > 0) { + packet.set_capture_time_ms(packet.capture_time_ms() + + (now_ms - last_timestamp_time_ms_)); + } + } + + return true; +} + +} // namespace webrtc diff --git a/modules/rtp_rtcp/source/packet_sequencer.h b/modules/rtp_rtcp/source/packet_sequencer.h new file mode 100644 index 0000000000..67255164f3 --- /dev/null +++ b/modules/rtp_rtcp/source/packet_sequencer.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_PACKET_SEQUENCER_H_ +#define MODULES_RTP_RTCP_SOURCE_PACKET_SEQUENCER_H_ + +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +// Helper class used to assign RTP sequence numbers and populate some fields for +// padding packets based on the last sequenced packets. +// This class is not thread safe, the caller must provide that. +class PacketSequencer { + public: + // If |require_marker_before_media_padding_| is true, padding packets on the + // media ssrc is not allowed unless the last sequenced media packet had the + // marker bit set (i.e. don't insert padding packets between the first and + // last packets of a video frame). + PacketSequencer(uint32_t media_ssrc, + uint32_t rtx_ssrc, + bool require_marker_before_media_padding, + Clock* clock); + + // Assigns sequence number, and in the case of non-RTX padding also timestamps + // and payload type. + // Returns false if sequencing failed, which it can do for instance if the + // packet to squence is padding on the media ssrc, but the media is mid frame + // (the last marker bit is false). + bool Sequence(RtpPacketToSend& packet); + + void set_media_sequence_number(uint16_t sequence_number) { + media_sequence_number_ = sequence_number; + } + void set_rtx_sequence_number(uint16_t sequence_number) { + rtx_sequence_number_ = sequence_number; + } + + void SetRtpState(const RtpState& state); + void PupulateRtpState(RtpState& state) const; + + uint16_t media_sequence_number() const { return media_sequence_number_; } + uint16_t rtx_sequence_number() const { return rtx_sequence_number_; } + + private: + void UpdateLastPacketState(const RtpPacketToSend& packet); + bool PopulatePaddingFields(RtpPacketToSend& packet); + + const uint32_t media_ssrc_; + const uint32_t rtx_ssrc_; + const bool require_marker_before_media_padding_; + Clock* const clock_; + + uint16_t media_sequence_number_; + uint16_t rtx_sequence_number_; + + int8_t last_payload_type_; + uint32_t last_rtp_timestamp_; + int64_t last_capture_time_ms_; + int64_t last_timestamp_time_ms_; + bool last_packet_marker_bit_; +}; + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_SOURCE_PACKET_SEQUENCER_H_ diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc index 45d5a6a65d..c8ea999e95 100644 --- a/modules/rtp_rtcp/source/rtp_sender.cc +++ b/modules/rtp_rtcp/source/rtp_sender.cc @@ -42,14 +42,10 @@ constexpr size_t kMaxPaddingLength = 224; constexpr size_t kMinAudioPaddingLength = 50; constexpr size_t kRtpHeaderLength = 12; constexpr uint16_t kMaxInitRtpSeqNumber = 32767; // 2^15 -1. -constexpr uint32_t kTimestampTicksPerMs = 90; // Min size needed to get payload padding from packet history. constexpr int kMinPayloadPaddingBytes = 50; -// RED header if first byte of payload. -constexpr size_t kRedForFecHeaderLength = 1; - template constexpr RtpExtensionSize CreateExtensionSize() { return {Extension::kId, Extension::kValueSizeBytes}; @@ -173,17 +169,15 @@ RTPSender::RTPSender(const RtpRtcpInterface::Configuration& config, paced_sender_(packet_sender), sending_media_(true), // Default to sending media. max_packet_size_(IP_PACKET_SIZE - 28), // Default is IP-v4/UDP. - last_payload_type_(-1), rtp_header_extension_map_(config.extmap_allow_mixed), // RTP variables - sequence_number_forced_(false), + sequencer_(config.local_media_ssrc, + config.rtx_send_ssrc.value_or(config.local_media_ssrc), + /*require_marker_before_media_padding_=*/!config.audio, + config.clock), always_send_mid_and_rid_(config.always_send_mid_and_rid), ssrc_has_acked_(false), rtx_ssrc_has_acked_(false), - last_rtp_timestamp_(0), - capture_time_ms_(0), - last_timestamp_time_ms_(0), - last_packet_marker_bit_(false), csrcs_(), rtx_(kRtxOff), supports_bwe_extension_(false), @@ -192,8 +186,8 @@ RTPSender::RTPSender(const RtpRtcpInterface::Configuration& config, // This random initialization is not intended to be cryptographic strong. timestamp_offset_ = random_.Rand(); // Random start, 16 bits. Can't be 0. - sequence_number_rtx_ = random_.Rand(1, kMaxInitRtpSeqNumber); - sequence_number_ = random_.Rand(1, kMaxInitRtpSeqNumber); + sequencer_.set_rtx_sequence_number(random_.Rand(1, kMaxInitRtpSeqNumber)); + sequencer_.set_media_sequence_number(random_.Rand(1, kMaxInitRtpSeqNumber)); RTC_DCHECK(paced_sender_); RTC_DCHECK(packet_history_); @@ -449,23 +443,11 @@ std::vector> RTPSender::GeneratePadding( std::make_unique(&rtp_header_extension_map_); padding_packet->set_packet_type(RtpPacketMediaType::kPadding); padding_packet->SetMarker(false); - padding_packet->SetTimestamp(last_rtp_timestamp_); - padding_packet->set_capture_time_ms(capture_time_ms_); if (rtx_ == kRtxOff) { - if (last_payload_type_ == -1) { - break; - } - // Without RTX we can't send padding in the middle of frames. - // For audio marker bits doesn't mark the end of a frame and frames - // are usually a single packet, so for now we don't apply this rule - // for audio. - if (!audio_configured_ && !last_packet_marker_bit_) { - break; - } - padding_packet->SetSsrc(ssrc_); - padding_packet->SetPayloadType(last_payload_type_); - padding_packet->SetSequenceNumber(sequence_number_++); + if (!sequencer_.Sequence(*padding_packet)) { + break; + } } else { // Without abs-send-time or transport sequence number a media packet // must be sent before padding so that the timestamps used for @@ -476,24 +458,13 @@ std::vector> RTPSender::GeneratePadding( TransportSequenceNumber::kId))) { break; } - // Only change the timestamp of padding packets sent over RTX. - // Padding only packets over RTP has to be sent as part of a media - // frame (and therefore the same timestamp). - int64_t now_ms = clock_->TimeInMilliseconds(); - if (last_timestamp_time_ms_ > 0) { - padding_packet->SetTimestamp(padding_packet->Timestamp() + - (now_ms - last_timestamp_time_ms_) * - kTimestampTicksPerMs); - if (padding_packet->capture_time_ms() > 0) { - padding_packet->set_capture_time_ms( - padding_packet->capture_time_ms() + - (now_ms - last_timestamp_time_ms_)); - } - } + RTC_DCHECK(rtx_ssrc_); padding_packet->SetSsrc(*rtx_ssrc_); - padding_packet->SetSequenceNumber(sequence_number_rtx_++); padding_packet->SetPayloadType(rtx_payload_type_map_.begin()->second); + if (!sequencer_.Sequence(*padding_packet)) { + break; + } } if (rtp_header_extension_map_.IsRegistered(TransportSequenceNumber::kId)) { @@ -558,13 +529,6 @@ size_t RTPSender::ExpectedPerPacketOverhead() const { return max_media_packet_header_; } -uint16_t RTPSender::AllocateSequenceNumber(uint16_t packets_to_send) { - MutexLock lock(&send_mutex_); - uint16_t first_allocated_sequence_number = sequence_number_; - sequence_number_ += packets_to_send; - return first_allocated_sequence_number; -} - std::unique_ptr RTPSender::AllocatePacket() const { MutexLock lock(&send_mutex_); // TODO(danilchap): Find better motivator and value for extra capacity. @@ -611,11 +575,7 @@ bool RTPSender::AssignSequenceNumber(RtpPacketToSend* packet) { MutexLock lock(&send_mutex_); if (!sending_media_) return false; - RTC_DCHECK(packet->Ssrc() == ssrc_); - packet->SetSequenceNumber(sequence_number_++); - - UpdateLastPacketState(*packet); - return true; + return sequencer_.Sequence(*packet); } bool RTPSender::AssignSequenceNumbersAndStoreLastPacketState( @@ -625,31 +585,11 @@ bool RTPSender::AssignSequenceNumbersAndStoreLastPacketState( if (!sending_media_) return false; for (auto& packet : packets) { - RTC_DCHECK_EQ(packet->Ssrc(), ssrc_); - packet->SetSequenceNumber(sequence_number_++); + sequencer_.Sequence(*packet); } - UpdateLastPacketState(**packets.rbegin()); return true; } -void RTPSender::UpdateLastPacketState(const RtpPacketToSend& packet) { - // Remember marker bit to determine if padding can be inserted with - // sequence number following |packet|. - last_packet_marker_bit_ = packet.Marker(); - // Remember media payload type to use in the padding packet if rtx is - // disabled. - if (packet.is_red()) { - RTC_DCHECK_GE(packet.payload_size(), kRedForFecHeaderLength); - last_payload_type_ = packet.PayloadBuffer()[0]; - } else { - last_payload_type_ = packet.PayloadType(); - } - // Save timestamps to generate timestamp field and extensions for the padding. - last_rtp_timestamp_ = packet.Timestamp(); - last_timestamp_time_ms_ = clock_->TimeInMilliseconds(); - capture_time_ms_ = packet.capture_time_ms(); -} - void RTPSender::SetSendingMediaStatus(bool enabled) { MutexLock lock(&send_mutex_); sending_media_ = enabled; @@ -701,11 +641,10 @@ void RTPSender::SetSequenceNumber(uint16_t seq) { bool updated_sequence_number = false; { MutexLock lock(&send_mutex_); - sequence_number_forced_ = true; - if (sequence_number_ != seq) { + if (sequencer_.media_sequence_number() != seq) { updated_sequence_number = true; } - sequence_number_ = seq; + sequencer_.set_media_sequence_number(seq); } if (updated_sequence_number) { @@ -717,7 +656,7 @@ void RTPSender::SetSequenceNumber(uint16_t seq) { uint16_t RTPSender::SequenceNumber() const { MutexLock lock(&send_mutex_); - return sequence_number_; + return sequencer_.media_sequence_number(); } static void CopyHeaderAndExtensionsToRtxPacket(const RtpPacketToSend& packet, @@ -790,12 +729,12 @@ std::unique_ptr RTPSender::BuildRtxPacket( rtx_packet->SetPayloadType(kv->second); - // Replace sequence number. - rtx_packet->SetSequenceNumber(sequence_number_rtx_++); - // Replace SSRC. rtx_packet->SetSsrc(*rtx_ssrc_); + // Replace sequence number. + sequencer_.Sequence(*rtx_packet); + CopyHeaderAndExtensionsToRtxPacket(packet, rtx_packet.get()); // RTX packets are sent on an SSRC different from the main media, so the @@ -841,12 +780,9 @@ std::unique_ptr RTPSender::BuildRtxPacket( void RTPSender::SetRtpState(const RtpState& rtp_state) { MutexLock lock(&send_mutex_); - sequence_number_ = rtp_state.sequence_number; - sequence_number_forced_ = true; + timestamp_offset_ = rtp_state.start_timestamp; - last_rtp_timestamp_ = rtp_state.timestamp; - capture_time_ms_ = rtp_state.capture_time_ms; - last_timestamp_time_ms_ = rtp_state.last_timestamp_time_ms; + sequencer_.SetRtpState(rtp_state); ssrc_has_acked_ = rtp_state.ssrc_has_acked; UpdateHeaderSizes(); } @@ -855,18 +791,15 @@ RtpState RTPSender::GetRtpState() const { MutexLock lock(&send_mutex_); RtpState state; - state.sequence_number = sequence_number_; state.start_timestamp = timestamp_offset_; - state.timestamp = last_rtp_timestamp_; - state.capture_time_ms = capture_time_ms_; - state.last_timestamp_time_ms = last_timestamp_time_ms_; state.ssrc_has_acked = ssrc_has_acked_; + sequencer_.PupulateRtpState(state); return state; } void RTPSender::SetRtxRtpState(const RtpState& rtp_state) { MutexLock lock(&send_mutex_); - sequence_number_rtx_ = rtp_state.sequence_number; + sequencer_.set_rtx_sequence_number(rtp_state.sequence_number); rtx_ssrc_has_acked_ = rtp_state.ssrc_has_acked; } @@ -874,18 +807,13 @@ RtpState RTPSender::GetRtxRtpState() const { MutexLock lock(&send_mutex_); RtpState state; - state.sequence_number = sequence_number_rtx_; + state.sequence_number = sequencer_.rtx_sequence_number(); state.start_timestamp = timestamp_offset_; state.ssrc_has_acked = rtx_ssrc_has_acked_; return state; } -int64_t RTPSender::LastTimestampTimeMs() const { - MutexLock lock(&send_mutex_); - return last_timestamp_time_ms_; -} - void RTPSender::UpdateHeaderSizes() { const size_t rtp_header_length = kRtpHeaderLength + sizeof(uint32_t) * csrcs_.size(); diff --git a/modules/rtp_rtcp/source/rtp_sender.h b/modules/rtp_rtcp/source/rtp_sender.h index 8c579429cc..fbf135049c 100644 --- a/modules/rtp_rtcp/source/rtp_sender.h +++ b/modules/rtp_rtcp/source/rtp_sender.h @@ -26,6 +26,7 @@ #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" #include "modules/rtp_rtcp/include/rtp_packet_sender.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/packet_sequencer.h" #include "modules/rtp_rtcp/source/rtp_packet_history.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" @@ -146,8 +147,6 @@ class RTPSender { RTC_LOCKS_EXCLUDED(send_mutex_); // Expected header overhead per media packet. size_t ExpectedPerPacketOverhead() const RTC_LOCKS_EXCLUDED(send_mutex_); - uint16_t AllocateSequenceNumber(uint16_t packets_to_send) - RTC_LOCKS_EXCLUDED(send_mutex_); // Including RTP headers. size_t MaxRtpPacketSize() const RTC_LOCKS_EXCLUDED(send_mutex_); @@ -173,8 +172,6 @@ class RTPSender { RTC_LOCKS_EXCLUDED(send_mutex_); RtpState GetRtxRtpState() const RTC_LOCKS_EXCLUDED(send_mutex_); - int64_t LastTimestampTimeMs() const RTC_LOCKS_EXCLUDED(send_mutex_); - private: std::unique_ptr BuildRtxPacket( const RtpPacketToSend& packet); @@ -206,17 +203,13 @@ class RTPSender { bool sending_media_ RTC_GUARDED_BY(send_mutex_); size_t max_packet_size_; - int8_t last_payload_type_ RTC_GUARDED_BY(send_mutex_); - RtpHeaderExtensionMap rtp_header_extension_map_ RTC_GUARDED_BY(send_mutex_); size_t max_media_packet_header_ RTC_GUARDED_BY(send_mutex_); size_t max_padding_fec_packet_header_ RTC_GUARDED_BY(send_mutex_); // RTP variables uint32_t timestamp_offset_ RTC_GUARDED_BY(send_mutex_); - bool sequence_number_forced_ RTC_GUARDED_BY(send_mutex_); - uint16_t sequence_number_ RTC_GUARDED_BY(send_mutex_); - uint16_t sequence_number_rtx_ RTC_GUARDED_BY(send_mutex_); + PacketSequencer sequencer_ RTC_GUARDED_BY(send_mutex_); // RID value to send in the RID or RepairedRID header extension. std::string rid_ RTC_GUARDED_BY(send_mutex_); // MID value to send in the MID header extension. @@ -227,10 +220,6 @@ class RTPSender { // when to stop sending the MID and RID header extensions. bool ssrc_has_acked_ RTC_GUARDED_BY(send_mutex_); bool rtx_ssrc_has_acked_ RTC_GUARDED_BY(send_mutex_); - uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(send_mutex_); - int64_t capture_time_ms_ RTC_GUARDED_BY(send_mutex_); - int64_t last_timestamp_time_ms_ RTC_GUARDED_BY(send_mutex_); - bool last_packet_marker_bit_ RTC_GUARDED_BY(send_mutex_); std::vector csrcs_ RTC_GUARDED_BY(send_mutex_); int rtx_ RTC_GUARDED_BY(send_mutex_); // Mapping rtx_payload_type_map_[associated] = rtx.