diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 3ae0794631..956ac0c771 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -177,6 +177,8 @@ std::vector CreateRtpStreamSenders( configuration.rtx_send_ssrc = rtp_config.rtx.ssrcs[i]; } + configuration.need_rtp_packet_infos = rtp_config.lntf.enabled; + auto rtp_rtcp = RtpRtcp::Create(configuration); rtp_rtcp->SetSendingStatus(false); rtp_rtcp->SetSendingMediaStatus(false); @@ -192,7 +194,6 @@ std::vector CreateRtpStreamSenders( video_config.frame_encryptor = frame_encryptor; video_config.require_frame_encryption = crypto_options.sframe.require_frame_encryption; - video_config.need_rtp_packet_infos = rtp_config.lntf.enabled; video_config.enable_retransmit_all_layers = false; video_config.field_trials = &field_trial_config; const bool should_disable_red_and_ulpfec = @@ -770,7 +771,7 @@ std::vector RtpVideoSender::GetSentRtpPacketInfos( rtc::ArrayView sequence_numbers) const { for (const auto& rtp_stream : rtp_streams_) { if (ssrc == rtp_stream.rtp_rtcp->SSRC()) { - return rtp_stream.sender_video->GetSentRtpPacketInfos(sequence_numbers); + return rtp_stream.rtp_rtcp->GetSentRtpPacketInfos(sequence_numbers); } } return std::vector(); diff --git a/modules/rtp_rtcp/include/rtp_rtcp.h b/modules/rtp_rtcp/include/rtp_rtcp.h index fbb3bb3241..b256f381d8 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp.h +++ b/modules/rtp_rtcp/include/rtp_rtcp.h @@ -28,6 +28,7 @@ #include "modules/rtp_rtcp/include/rtp_packet_sender.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/deprecation.h" @@ -125,6 +126,8 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { uint32_t local_media_ssrc = 0; absl::optional rtx_send_ssrc; + bool need_rtp_packet_infos = false; + private: RTC_DISALLOW_COPY_AND_ASSIGN(Configuration); }; @@ -284,6 +287,9 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { virtual std::vector> GeneratePadding( size_t target_size_bytes) = 0; + virtual std::vector GetSentRtpPacketInfos( + rtc::ArrayView sequence_numbers) const = 0; + // ************************************************************************** // RTCP // ************************************************************************** diff --git a/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h b/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h index 83bc7ccec7..8864df01db 100644 --- a/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h +++ b/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h @@ -94,6 +94,9 @@ class MockRtpRtcp : public RtpRtcp { MOCK_METHOD1( GeneratePadding, std::vector>(size_t target_size_bytes)); + MOCK_CONST_METHOD1(GetSentRtpPacketInfos, + std::vector( + rtc::ArrayView sequence_numbers)); MOCK_METHOD2(RegisterRtcpObservers, void(RtcpIntraFrameObserver* intra_frame_callback, RtcpBandwidthObserver* bandwidth_callback)); diff --git a/modules/rtp_rtcp/source/rtp_packet_to_send.h b/modules/rtp_rtcp/source/rtp_packet_to_send.h index d30bbbf3ad..57493e3802 100644 --- a/modules/rtp_rtcp/source/rtp_packet_to_send.h +++ b/modules/rtp_rtcp/source/rtp_packet_to_send.h @@ -98,12 +98,19 @@ class RtpPacketToSend : public RtpPacket { VideoTimingExtension::kNetwork2TimestampDeltaOffset); } + void set_first_packet_of_frame(bool is_first_packet) { + is_first_packet_of_frame_ = is_first_packet; + } + + bool is_first_packet_of_frame() const { return is_first_packet_of_frame_; } + private: int64_t capture_time_ms_ = 0; absl::optional packet_type_; bool allow_retransmission_ = false; absl::optional retransmitted_sequence_number_; std::vector application_data_; + bool is_first_packet_of_frame_ = false; }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index dfbac29d03..ff301433ae 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -224,6 +224,7 @@ uint32_t ModuleRtpRtcpImpl::StartTimestamp() const { void ModuleRtpRtcpImpl::SetStartTimestamp(const uint32_t timestamp) { rtcp_sender_.SetTimestampOffset(timestamp); rtp_sender_->packet_generator.SetTimestampOffset(timestamp); + rtp_sender_->packet_sender.SetTimestampOffset(timestamp); } uint16_t ModuleRtpRtcpImpl::SequenceNumber() const { @@ -393,6 +394,13 @@ ModuleRtpRtcpImpl::GeneratePadding(size_t target_size_bytes) { target_size_bytes, rtp_sender_->packet_sender.MediaHasBeenSent()); } +std::vector +ModuleRtpRtcpImpl::GetSentRtpPacketInfos( + rtc::ArrayView sequence_numbers) const { + RTC_DCHECK(rtp_sender_); + return rtp_sender_->packet_sender.GetSentRtpPacketInfos(sequence_numbers); +} + size_t ModuleRtpRtcpImpl::MaxRtpPacketSize() const { RTC_DCHECK(rtp_sender_); return rtp_sender_->packet_generator.MaxRtpPacketSize(); diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.h b/modules/rtp_rtcp/source/rtp_rtcp_impl.h index c03683f48e..80488a8e1f 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl.h +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.h @@ -143,6 +143,9 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { std::vector> GeneratePadding( size_t target_size_bytes) override; + std::vector GetSentRtpPacketInfos( + rtc::ArrayView sequence_numbers) const override; + // RTCP part. // Get RTCP status. diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc index 5e4cce99a7..e2595664f6 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc @@ -152,6 +152,7 @@ class RtpRtcpModule : public RtcpPacketTypeCounterObserver { config.rtt_stats = &rtt_stats_; config.rtcp_report_interval_ms = rtcp_report_interval_ms_; config.local_media_ssrc = is_sender_ ? kSenderSsrc : kReceiverSsrc; + config.need_rtp_packet_infos = true; impl_.reset(new ModuleRtpRtcpImpl(config)); impl_->SetRemoteSSRC(is_sender_ ? kReceiverSsrc : kSenderSsrc); @@ -569,4 +570,61 @@ TEST_F(RtpRtcpImplTest, ConfigurableRtcpReportInterval) { EXPECT_EQ(sender_.transport_.NumRtcpSent(), 2u); } +TEST_F(RtpRtcpImplTest, StoresPacketInfoForSentPackets) { + const uint32_t kStartTimestamp = 1u; + SetUp(); + sender_.impl_->SetStartTimestamp(kStartTimestamp); + + PacedPacketInfo pacing_info; + RtpPacketToSend packet(nullptr); + packet.set_packet_type(RtpPacketToSend::Type::kVideo); + packet.SetSsrc(kSenderSsrc); + + // Single-packet frame. + packet.SetTimestamp(1); + packet.SetSequenceNumber(1); + packet.set_first_packet_of_frame(true); + packet.SetMarker(true); + sender_.impl_->TrySendPacket(&packet, pacing_info); + + std::vector seqno_info = + sender_.impl_->GetSentRtpPacketInfos(std::vector{1}); + + EXPECT_THAT(seqno_info, ElementsAre(RtpSequenceNumberMap::Info( + /*timestamp=*/1 - kStartTimestamp, + /*is_first=*/1, + /*is_last=*/1))); + + // Three-packet frame. + packet.SetTimestamp(2); + packet.SetSequenceNumber(2); + packet.set_first_packet_of_frame(true); + packet.SetMarker(false); + sender_.impl_->TrySendPacket(&packet, pacing_info); + + packet.SetSequenceNumber(3); + packet.set_first_packet_of_frame(false); + sender_.impl_->TrySendPacket(&packet, pacing_info); + + packet.SetSequenceNumber(4); + packet.SetMarker(true); + sender_.impl_->TrySendPacket(&packet, pacing_info); + + seqno_info = + sender_.impl_->GetSentRtpPacketInfos(std::vector{2, 3, 4}); + + EXPECT_THAT(seqno_info, ElementsAre(RtpSequenceNumberMap::Info( + /*timestamp=*/2 - kStartTimestamp, + /*is_first=*/1, + /*is_last=*/0), + RtpSequenceNumberMap::Info( + /*timestamp=*/2 - kStartTimestamp, + /*is_first=*/0, + /*is_last=*/0), + RtpSequenceNumberMap::Info( + /*timestamp=*/2 - kStartTimestamp, + /*is_first=*/0, + /*is_last=*/1))); +} + } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender_egress.cc b/modules/rtp_rtcp/source/rtp_sender_egress.cc index dc888f9881..fafd3ca4c4 100644 --- a/modules/rtp_rtcp/source/rtp_sender_egress.cc +++ b/modules/rtp_rtcp/source/rtp_sender_egress.cc @@ -25,6 +25,7 @@ namespace { constexpr uint32_t kTimestampTicksPerMs = 90; constexpr int kSendSideDelayWindowMs = 1000; constexpr int kBitrateStatisticsWindowMs = 1000; +constexpr size_t kRtpSequenceNumberMapMaxEntries = 1 << 13; bool IsEnabled(absl::string_view name, const WebRtcKeyValueConfig* field_trials) { @@ -67,6 +68,7 @@ RtpSenderEgress::RtpSenderEgress(const RtpRtcp::Configuration& config, transport_(config.outgoing_transport), event_log_(config.event_log), is_audio_(config.audio), + need_rtp_packet_infos_(config.need_rtp_packet_infos), transport_feedback_observer_(config.transport_feedback_callback), send_side_delay_observer_(config.send_side_delay_observer), send_packet_observer_(config.send_packet_observer), @@ -75,14 +77,18 @@ RtpSenderEgress::RtpSenderEgress(const RtpRtcp::Configuration& config, bitrate_callback_(config.send_bitrate_observer), media_has_been_sent_(false), force_part_of_allocation_(false), + timestamp_offset_(0), max_delay_it_(send_delays_.end()), sum_delays_ms_(0), total_packet_send_delay_ms_(0), rtp_overhead_bytes_per_packet_(0), total_bitrate_sent_(kBitrateStatisticsWindowMs, RateStatistics::kBpsScale), - nack_bitrate_sent_(kBitrateStatisticsWindowMs, - RateStatistics::kBpsScale) {} + nack_bitrate_sent_(kBitrateStatisticsWindowMs, RateStatistics::kBpsScale), + rtp_sequence_number_map_(need_rtp_packet_infos_ + ? std::make_unique( + kRtpSequenceNumberMapMaxEntries) + : nullptr) {} void RtpSenderEgress::SendPacket(RtpPacketToSend* packet, const PacedPacketInfo& pacing_info) { @@ -113,6 +119,20 @@ void RtpSenderEgress::SendPacket(RtpPacketToSend* packet, { rtc::CritScope lock(&lock_); options.included_in_allocation = force_part_of_allocation_; + + if (need_rtp_packet_infos_ && + packet->packet_type() == RtpPacketToSend::Type::kVideo) { + RTC_DCHECK(rtp_sequence_number_map_); + // Last packet of a frame, add it to sequence number info map. + const uint32_t timestamp = packet->Timestamp() - timestamp_offset_; + bool is_first_packet_of_frame = packet->is_first_packet_of_frame(); + bool is_last_packet_of_frame = packet->Marker(); + + rtp_sequence_number_map_->InsertPacket( + packet->SequenceNumber(), + RtpSequenceNumberMap::Info(timestamp, is_first_packet_of_frame, + is_last_packet_of_frame)); + } } // Bug webrtc:7859. While FEC is invoked from rtp_sender_video, and not after @@ -226,6 +246,35 @@ void RtpSenderEgress::SetMediaHasBeenSent(bool media_sent) { media_has_been_sent_ = media_sent; } +void RtpSenderEgress::SetTimestampOffset(uint32_t timestamp) { + rtc::CritScope lock(&lock_); + timestamp_offset_ = timestamp; +} + +std::vector RtpSenderEgress::GetSentRtpPacketInfos( + rtc::ArrayView sequence_numbers) const { + RTC_DCHECK(!sequence_numbers.empty()); + if (!need_rtp_packet_infos_) { + return std::vector(); + } + + std::vector results; + results.reserve(sequence_numbers.size()); + + rtc::CritScope cs(&lock_); + for (uint16_t sequence_number : sequence_numbers) { + const auto& info = rtp_sequence_number_map_->Get(sequence_number); + if (!info) { + // The empty vector will be returned. We can delay the clearing + // of the vector until after we exit the critical section. + return std::vector(); + } + results.push_back(*info); + } + + return results; +} + bool RtpSenderEgress::HasCorrectSsrc(const RtpPacketToSend& packet) const { switch (*packet.packet_type()) { case RtpPacketMediaType::kAudio: diff --git a/modules/rtp_rtcp/source/rtp_sender_egress.h b/modules/rtp_rtcp/source/rtp_sender_egress.h index e72a0cff67..3d4999f964 100644 --- a/modules/rtp_rtcp/source/rtp_sender_egress.h +++ b/modules/rtp_rtcp/source/rtp_sender_egress.h @@ -23,6 +23,7 @@ #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_packet_history.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" #include "rtc_base/critical_section.h" #include "rtc_base/rate_statistics.h" #include "rtc_base/thread_annotations.h" @@ -64,6 +65,15 @@ class RtpSenderEgress { void ForceIncludeSendPacketsInAllocation(bool part_of_allocation); bool MediaHasBeenSent() const; void SetMediaHasBeenSent(bool media_sent); + void SetTimestampOffset(uint32_t timestamp); + + // For each sequence number in |sequence_number|, recall the last RTP packet + // which bore it - its timestamp and whether it was the first and/or last + // packet in that frame. If all of the given sequence numbers could be + // recalled, return a vector with all of them (in corresponding order). + // If any could not be recalled, return an empty vector. + std::vector GetSentRtpPacketInfos( + rtc::ArrayView sequence_numbers) const; private: // Maps capture time in milliseconds to send-side delay in milliseconds. @@ -100,6 +110,7 @@ class RtpSenderEgress { Transport* const transport_; RtcEventLog* const event_log_; const bool is_audio_; + const bool need_rtp_packet_infos_; TransportFeedbackObserver* const transport_feedback_observer_; SendSideDelayObserver* const send_side_delay_observer_; @@ -111,6 +122,7 @@ class RtpSenderEgress { rtc::CriticalSection lock_; bool media_has_been_sent_ RTC_GUARDED_BY(lock_); bool force_part_of_allocation_ RTC_GUARDED_BY(lock_); + uint32_t timestamp_offset_ RTC_GUARDED_BY(lock_); SendDelayMap send_delays_ RTC_GUARDED_BY(lock_); SendDelayMap::const_iterator max_delay_it_ RTC_GUARDED_BY(lock_); @@ -122,6 +134,13 @@ class RtpSenderEgress { StreamDataCounters rtx_rtp_stats_ RTC_GUARDED_BY(lock_); RateStatistics total_bitrate_sent_ RTC_GUARDED_BY(lock_); RateStatistics nack_bitrate_sent_ RTC_GUARDED_BY(lock_); + + // Maps sent packets' sequence numbers to a tuple consisting of: + // 1. The timestamp, without the randomizing offset mandated by the RFC. + // 2. Whether the packet was the first in its frame. + // 3. Whether the packet was the last in its frame. + const std::unique_ptr rtp_sequence_number_map_ + RTC_GUARDED_BY(lock_); }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 99fb822cc5..532e06950c 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -41,7 +41,6 @@ namespace webrtc { namespace { constexpr size_t kRedForFecHeaderLength = 1; -constexpr size_t kRtpSequenceNumberMapMaxEntries = 1 << 13; constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4; // This is experimental field trial to exclude transport sequence number from @@ -253,7 +252,6 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock, PlayoutDelayOracle* playout_delay_oracle, FrameEncryptorInterface* frame_encryptor, bool require_frame_encryption, - bool need_rtp_packet_infos, bool enable_retransmit_all_layers, const WebRtcKeyValueConfig& field_trials) : RTPSenderVideo([&] { @@ -263,7 +261,6 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock, config.flexfec_sender = flexfec_sender; config.frame_encryptor = frame_encryptor; config.require_frame_encryption = require_frame_encryption; - config.need_rtp_packet_infos = need_rtp_packet_infos; config.enable_retransmit_all_layers = enable_retransmit_all_layers; config.field_trials = &field_trials; return config; @@ -280,10 +277,6 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) transmit_color_space_next_frame_(false), current_playout_delay_{-1, -1}, playout_delay_pending_(false), - rtp_sequence_number_map_(config.need_rtp_packet_infos - ? std::make_unique( - kRtpSequenceNumberMapMaxEntries) - : nullptr), red_payload_type_(config.red_payload_type), ulpfec_payload_type_(config.ulpfec_payload_type), flexfec_sender_(config.flexfec_sender), @@ -729,7 +722,6 @@ bool RTPSenderVideo::SendVideo( if (num_packets == 0) return false; - uint16_t first_sequence_number; bool first_frame = first_frame_sent_(); std::vector> rtp_packets; for (size_t i = 0; i < num_packets; ++i) { @@ -753,16 +745,14 @@ bool RTPSenderVideo::SendVideo( expected_payload_capacity = limits.max_payload_len; } + packet->set_first_packet_of_frame(i == 0); + if (!packetizer->NextPacket(packet.get())) return false; RTC_DCHECK_LE(packet->payload_size(), expected_payload_capacity); if (!rtp_sender_->AssignSequenceNumber(packet.get())) return false; - if (rtp_sequence_number_map_ && i == 0) { - first_sequence_number = packet->SequenceNumber(); - } - // No FEC protection for upper temporal layers, if used. bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx; @@ -802,13 +792,6 @@ bool RTPSenderVideo::SendVideo( } } - if (rtp_sequence_number_map_) { - const uint32_t timestamp = rtp_timestamp - rtp_sender_->TimestampOffset(); - rtc::CritScope cs(&crit_); - rtp_sequence_number_map_->InsertFrame(first_sequence_number, num_packets, - timestamp); - } - LogAndSendToNetwork(std::move(rtp_packets), unpacketized_payload_size); TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp", @@ -832,37 +815,6 @@ uint32_t RTPSenderVideo::PacketizationOverheadBps() const { .value_or(0); } -std::vector RTPSenderVideo::GetSentRtpPacketInfos( - rtc::ArrayView sequence_numbers) const { - RTC_DCHECK(!sequence_numbers.empty()); - - std::vector results; - if (!rtp_sequence_number_map_) { - return results; - } - results.reserve(sequence_numbers.size()); - - { - rtc::CritScope cs(&crit_); - for (uint16_t sequence_number : sequence_numbers) { - const absl::optional info = - rtp_sequence_number_map_->Get(sequence_number); - if (!info) { - // The empty vector will be returned. We can delay the clearing - // of the vector until after we exit the critical section. - break; - } - results.push_back(*info); - } - } - - if (results.size() != sequence_numbers.size()) { - results.clear(); // Some sequence number was not found. - } - - return results; -} - bool RTPSenderVideo::AllowRetransmission( uint8_t temporal_id, int32_t retransmission_settings, diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index 5f01803055..2df62adf21 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -28,7 +28,6 @@ #include "modules/rtp_rtcp/source/playout_delay_oracle.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "modules/rtp_rtcp/source/rtp_sender.h" -#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" #include "modules/rtp_rtcp/source/ulpfec_generator.h" #include "rtc_base/critical_section.h" @@ -74,7 +73,6 @@ class RTPSenderVideo { PlayoutDelayOracle* playout_delay_oracle = nullptr; FrameEncryptorInterface* frame_encryptor = nullptr; bool require_frame_encryption = false; - bool need_rtp_packet_infos = false; bool enable_retransmit_all_layers = false; absl::optional red_payload_type; absl::optional ulpfec_payload_type; @@ -90,7 +88,6 @@ class RTPSenderVideo { PlayoutDelayOracle* playout_delay_oracle, FrameEncryptorInterface* frame_encryptor, bool require_frame_encryption, - bool need_rtp_packet_infos, bool enable_retransmit_all_layers, const WebRtcKeyValueConfig& field_trials); virtual ~RTPSenderVideo(); @@ -129,14 +126,6 @@ class RTPSenderVideo { // or extension/ uint32_t PacketizationOverheadBps() const; - // For each sequence number in |sequence_number|, recall the last RTP packet - // which bore it - its timestamp and whether it was the first and/or last - // packet in that frame. If all of the given sequence numbers could be - // recalled, return a vector with all of them (in corresponding order). - // If any could not be recalled, return an empty vector. - std::vector GetSentRtpPacketInfos( - rtc::ArrayView sequence_numbers) const; - protected: static uint8_t GetTemporalId(const RTPVideoHeader& header); bool AllowRetransmission(uint8_t temporal_id, @@ -208,13 +197,6 @@ class RTPSenderVideo { // Should never be held when calling out of this class. rtc::CriticalSection crit_; - // Maps sent packets' sequence numbers to a tuple consisting of: - // 1. The timestamp, without the randomizing offset mandated by the RFC. - // 2. Whether the packet was the first in its frame. - // 3. Whether the packet was the last in its frame. - const std::unique_ptr rtp_sequence_number_map_ - RTC_PT_GUARDED_BY(crit_); - // RED/ULPFEC. const absl::optional red_payload_type_; const absl::optional ulpfec_payload_type_;