diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 74015a8668..cd9283e2a1 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -740,6 +740,17 @@ uint32_t RtpVideoSender::GetProtectionBitrateBps() const { return protection_bitrate_bps_; } +absl::optional RtpVideoSender::GetSentRtpPacketInfo( + uint32_t ssrc, + uint16_t seq_num) const { + for (const auto& rtp_stream : rtp_streams_) { + if (ssrc == rtp_stream.rtp_rtcp->SSRC()) { + return rtp_stream.sender_video->GetSentRtpPacketInfo(seq_num); + } + } + return absl::nullopt; +} + int RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params, const FecProtectionParams* key_params, uint32_t* sent_video_rate_bps, diff --git a/call/rtp_video_sender.h b/call/rtp_video_sender.h index d50cb7c2d7..8518c91dd2 100644 --- a/call/rtp_video_sender.h +++ b/call/rtp_video_sender.h @@ -16,6 +16,7 @@ #include #include +#include "absl/types/optional.h" #include "api/call/transport.h" #include "api/fec_controller.h" #include "api/video_codecs/video_encoder.h" @@ -26,6 +27,7 @@ #include "logging/rtc_event_log/rtc_event_log.h" #include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/rtp_rtcp/source/rtp_sender_video.h" +#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" #include "modules/utility/include/process_thread.h" #include "rtc_base/constructor_magic.h" @@ -138,6 +140,10 @@ class RtpVideoSender : public RtpVideoSenderInterface, size_t height, size_t num_temporal_layers) override; + absl::optional GetSentRtpPacketInfo( + uint32_t ssrc, + uint16_t seq_num) const override; + // From PacketFeedbackObserver. void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override; void OnPacketFeedbackVector( diff --git a/call/rtp_video_sender_interface.h b/call/rtp_video_sender_interface.h index ecaca9b256..51cf56b609 100644 --- a/call/rtp_video_sender_interface.h +++ b/call/rtp_video_sender_interface.h @@ -14,8 +14,10 @@ #include #include +#include "absl/types/optional.h" #include "call/rtp_config.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" #include "modules/utility/include/process_thread.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -55,6 +57,9 @@ class RtpVideoSenderInterface : public EncodedImageCallback { virtual void SetEncodingData(size_t width, size_t height, size_t num_temporal_layers) = 0; + virtual absl::optional GetSentRtpPacketInfo( + uint32_t ssrc, + uint16_t seq_num) const = 0; }; } // namespace webrtc #endif // CALL_RTP_VIDEO_SENDER_INTERFACE_H_ diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 10bc2184d4..a90afbf559 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -38,6 +39,7 @@ namespace webrtc { namespace { constexpr size_t kRedForFecHeaderLength = 1; +constexpr size_t kRtpSequenceNumberMapMaxEntries = 1 << 13; constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4; void BuildRedPayload(const RtpPacketToSend& media_packet, @@ -196,6 +198,14 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock, last_rotation_(kVideoRotation_0), transmit_color_space_next_frame_(false), playout_delay_oracle_(playout_delay_oracle), + // TODO(eladalon): Choose whether to instantiate rtp_sequence_number_map_ + // according to the negotiation of the RTCP message. + rtp_sequence_number_map_( + field_trials.Lookup("WebRTC-RtcpLossNotification").find("Enabled") != + std::string::npos + ? absl::make_unique( + kRtpSequenceNumberMapMaxEntries) + : nullptr), red_payload_type_(-1), ulpfec_payload_type_(-1), flexfec_sender_(flexfec_sender), @@ -622,7 +632,7 @@ bool RTPSenderVideo::SendVideo(VideoFrameType frame_type, const uint8_t temporal_id = GetTemporalId(*video_header); StorageType storage = GetStorageType(temporal_id, retransmission_settings, expected_retransmission_time_ms); - size_t num_packets = packetizer->NumPackets(); + const size_t num_packets = packetizer->NumPackets(); size_t unpacketized_payload_size; if (fragmentation && fragmentation->fragmentationVectorSize > 0) { @@ -638,6 +648,7 @@ bool RTPSenderVideo::SendVideo(VideoFrameType frame_type, if (num_packets == 0) return false; + uint16_t first_sequence_number; bool first_frame = first_frame_sent_(); for (size_t i = 0; i < num_packets; ++i) { std::unique_ptr packet; @@ -667,6 +678,10 @@ bool RTPSenderVideo::SendVideo(VideoFrameType frame_type, return false; packetized_payload_size += packet->payload_size(); + if (rtp_sequence_number_map_ && i == 0) { + first_sequence_number = packet->SequenceNumber(); + } + if (i == 0) { playout_delay_oracle_->OnSentPacket(packet->SequenceNumber(), playout_delay); @@ -709,6 +724,13 @@ bool RTPSenderVideo::SendVideo(VideoFrameType frame_type, } } + if (rtp_sequence_number_map_) { + const uint32_t timestamp = rtp_timestamp - rtp_sender_->TimestampOffset(); + rtc::CritScope cs(&crit_); + rtp_sequence_number_map_->InsertFrame(first_sequence_number, num_packets, + timestamp); + } + rtc::CritScope cs(&stats_crit_); RTC_DCHECK_GE(packetized_payload_size, unpacketized_payload_size); packetization_overhead_bitrate_.Update( @@ -736,6 +758,15 @@ uint32_t RTPSenderVideo::PacketizationOverheadBps() const { .value_or(0); } +absl::optional RTPSenderVideo::GetSentRtpPacketInfo( + uint16_t sequence_number) const { + if (!rtp_sequence_number_map_) { + return absl::nullopt; + } + rtc::CritScope cs(&crit_); + return rtp_sequence_number_map_->Get(sequence_number); +} + StorageType RTPSenderVideo::GetStorageType( uint8_t temporal_id, int32_t retransmission_settings, diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index 034e7909cf..7144afdded 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -21,6 +21,7 @@ #include "modules/rtp_rtcp/source/playout_delay_oracle.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "modules/rtp_rtcp/source/rtp_sender.h" +#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" #include "modules/rtp_rtcp/source/ulpfec_generator.h" #include "rtc_base/critical_section.h" #include "rtc_base/one_time_event.h" @@ -93,6 +94,14 @@ class RTPSenderVideo { // or extension/ uint32_t PacketizationOverheadBps() const; + // Recall the last RTP packet whose sequence number was |sequence_number|. + // Return the timestamp of the video frame that packet belonged too, as well + // as whether the packet was the first and/or last packet in the frame. + // absl::nullopt returned if no such packet can be recalled (e.g. it happened + // too long ago). + absl::optional GetSentRtpPacketInfo( + uint16_t sequence_number) const; + protected: static uint8_t GetTemporalId(const RTPVideoHeader& header); StorageType GetStorageType(uint8_t temporal_id, @@ -166,6 +175,13 @@ class RTPSenderVideo { // delay extension on header. PlayoutDelayOracle* const playout_delay_oracle_; + // Maps sent packets' sequence numbers to a tuple consisting of: + // 1. The timestamp, without the randomizing offset mandated by the RFC. + // 2. Whether the packet was the first in its frame. + // 3. Whether the packet was the last in its frame. + const std::unique_ptr rtp_sequence_number_map_ + RTC_PT_GUARDED_BY(crit_); + // RED/ULPFEC. int red_payload_type_ RTC_GUARDED_BY(crit_); int ulpfec_payload_type_ RTC_GUARDED_BY(crit_); diff --git a/video/video_send_stream_impl_unittest.cc b/video/video_send_stream_impl_unittest.cc index cc454760a2..fc0f785bf6 100644 --- a/video/video_send_stream_impl_unittest.cc +++ b/video/video_send_stream_impl_unittest.cc @@ -11,10 +11,12 @@ #include #include "absl/memory/memory.h" +#include "absl/types/optional.h" #include "call/rtp_video_sender.h" #include "call/test/mock_bitrate_allocator.h" #include "call/test/mock_rtp_transport_controller_send.h" #include "logging/rtc_event_log/rtc_event_log.h" +#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" #include "modules/utility/include/process_thread.h" #include "modules/video_coding/fec_controller_default.h" #include "rtc_base/experiments/alr_experiment.h" @@ -67,6 +69,10 @@ class MockRtpVideoSender : public RtpVideoSenderInterface { MOCK_CONST_METHOD0(GetPayloadBitrateBps, uint32_t()); MOCK_CONST_METHOD0(GetProtectionBitrateBps, uint32_t()); MOCK_METHOD3(SetEncodingData, void(size_t, size_t, size_t)); + MOCK_CONST_METHOD2( + GetSentRtpPacketInfo, + absl::optional(uint32_t ssrc, + uint16_t seq_num)); }; BitrateAllocationUpdate CreateAllocation(int bitrate_bps) {