Give VideoSendStreamImpl access to RTP timestamps

When a LossNotification RTCP message is received, the sequence numbers
it refers to must be converted to timestamps before passing the message
down to the encoder. This CL gives VideoSendStreamImpl access to that
information via VideoSendStreamImpl::rtp_video_sender_.

TBR=sprang@webrtc.org

Bug: webrtc:10501
Change-Id: If207f0b6d2fb344da35b525cc104e8ba5cc614ec
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131323
Commit-Queue: Elad Alon <eladalon@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27489}

call/rtp_video_sender.cc

@@ -740,6 +740,17 @@ uint32_t RtpVideoSender::GetProtectionBitrateBps() const {
   return protection_bitrate_bps_;
 }
 
+absl::optional<RtpSequenceNumberMap::Info> RtpVideoSender::GetSentRtpPacketInfo(
+    uint32_t ssrc,
+    uint16_t seq_num) const {
+  for (const auto& rtp_stream : rtp_streams_) {
+    if (ssrc == rtp_stream.rtp_rtcp->SSRC()) {
+      return rtp_stream.sender_video->GetSentRtpPacketInfo(seq_num);
+    }
+  }
+  return absl::nullopt;
+}
+
 int RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params,
                                       const FecProtectionParams* key_params,
                                       uint32_t* sent_video_rate_bps,

call/rtp_video_sender.h

@@ -16,6 +16,7 @@
 #include <unordered_set>
 #include <vector>
 
+#include "absl/types/optional.h"
 #include "api/call/transport.h"
 #include "api/fec_controller.h"
 #include "api/video_codecs/video_encoder.h"
@@ -26,6 +27,7 @@
 #include "logging/rtc_event_log/rtc_event_log.h"
 #include "modules/rtp_rtcp/include/flexfec_sender.h"
 #include "modules/rtp_rtcp/source/rtp_sender_video.h"
+#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
 #include "modules/rtp_rtcp/source/rtp_video_header.h"
 #include "modules/utility/include/process_thread.h"
 #include "rtc_base/constructor_magic.h"
@@ -138,6 +140,10 @@ class RtpVideoSender : public RtpVideoSenderInterface,
                        size_t height,
                        size_t num_temporal_layers) override;
 
+  absl::optional<RtpSequenceNumberMap::Info> GetSentRtpPacketInfo(
+      uint32_t ssrc,
+      uint16_t seq_num) const override;
+
   // From PacketFeedbackObserver.
   void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override;
   void OnPacketFeedbackVector(

call/rtp_video_sender_interface.h

@@ -14,8 +14,10 @@
 #include <map>
 #include <vector>
 
+#include "absl/types/optional.h"
 #include "call/rtp_config.h"
 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
+#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
 #include "modules/utility/include/process_thread.h"
 #include "modules/video_coding/include/video_codec_interface.h"
 
@@ -55,6 +57,9 @@ class RtpVideoSenderInterface : public EncodedImageCallback {
   virtual void SetEncodingData(size_t width,
                                size_t height,
                                size_t num_temporal_layers) = 0;
+  virtual absl::optional<RtpSequenceNumberMap::Info> GetSentRtpPacketInfo(
+      uint32_t ssrc,
+      uint16_t seq_num) const = 0;
 };
 }  // namespace webrtc
 #endif  // CALL_RTP_VIDEO_SENDER_INTERFACE_H_

modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -15,6 +15,7 @@
 
 #include <limits>
 #include <memory>
+#include <string>
 #include <utility>
 #include <vector>
 
@@ -38,6 +39,7 @@ namespace webrtc {
 
 namespace {
 constexpr size_t kRedForFecHeaderLength = 1;
+constexpr size_t kRtpSequenceNumberMapMaxEntries = 1 << 13;
 constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
 
 void BuildRedPayload(const RtpPacketToSend& media_packet,
@@ -196,6 +198,14 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock,
       last_rotation_(kVideoRotation_0),
       transmit_color_space_next_frame_(false),
       playout_delay_oracle_(playout_delay_oracle),
+      // TODO(eladalon): Choose whether to instantiate rtp_sequence_number_map_
+      // according to the negotiation of the RTCP message.
+      rtp_sequence_number_map_(
+          field_trials.Lookup("WebRTC-RtcpLossNotification").find("Enabled") !=
+                  std::string::npos
+              ? absl::make_unique<RtpSequenceNumberMap>(
+                    kRtpSequenceNumberMapMaxEntries)
+              : nullptr),
       red_payload_type_(-1),
       ulpfec_payload_type_(-1),
       flexfec_sender_(flexfec_sender),
@@ -622,7 +632,7 @@ bool RTPSenderVideo::SendVideo(VideoFrameType frame_type,
   const uint8_t temporal_id = GetTemporalId(*video_header);
   StorageType storage = GetStorageType(temporal_id, retransmission_settings,
                                        expected_retransmission_time_ms);
-  size_t num_packets = packetizer->NumPackets();
+  const size_t num_packets = packetizer->NumPackets();
 
   size_t unpacketized_payload_size;
   if (fragmentation && fragmentation->fragmentationVectorSize > 0) {
@@ -638,6 +648,7 @@ bool RTPSenderVideo::SendVideo(VideoFrameType frame_type,
   if (num_packets == 0)
     return false;
 
+  uint16_t first_sequence_number;
   bool first_frame = first_frame_sent_();
   for (size_t i = 0; i < num_packets; ++i) {
     std::unique_ptr<RtpPacketToSend> packet;
@@ -667,6 +678,10 @@ bool RTPSenderVideo::SendVideo(VideoFrameType frame_type,
       return false;
     packetized_payload_size += packet->payload_size();
 
+    if (rtp_sequence_number_map_ && i == 0) {
+      first_sequence_number = packet->SequenceNumber();
+    }
+
     if (i == 0) {
       playout_delay_oracle_->OnSentPacket(packet->SequenceNumber(),
                                           playout_delay);
@@ -709,6 +724,13 @@ bool RTPSenderVideo::SendVideo(VideoFrameType frame_type,
     }
   }
 
+  if (rtp_sequence_number_map_) {
+    const uint32_t timestamp = rtp_timestamp - rtp_sender_->TimestampOffset();
+    rtc::CritScope cs(&crit_);
+    rtp_sequence_number_map_->InsertFrame(first_sequence_number, num_packets,
+                                          timestamp);
+  }
+
   rtc::CritScope cs(&stats_crit_);
   RTC_DCHECK_GE(packetized_payload_size, unpacketized_payload_size);
   packetization_overhead_bitrate_.Update(
@@ -736,6 +758,15 @@ uint32_t RTPSenderVideo::PacketizationOverheadBps() const {
       .value_or(0);
 }
 
+absl::optional<RtpSequenceNumberMap::Info> RTPSenderVideo::GetSentRtpPacketInfo(
+    uint16_t sequence_number) const {
+  if (!rtp_sequence_number_map_) {
+    return absl::nullopt;
+  }
+  rtc::CritScope cs(&crit_);
+  return rtp_sequence_number_map_->Get(sequence_number);
+}
+
 StorageType RTPSenderVideo::GetStorageType(
     uint8_t temporal_id,
     int32_t retransmission_settings,

modules/rtp_rtcp/source/rtp_sender_video.h

@@ -21,6 +21,7 @@
 #include "modules/rtp_rtcp/source/playout_delay_oracle.h"
 #include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
 #include "modules/rtp_rtcp/source/rtp_sender.h"
+#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
 #include "modules/rtp_rtcp/source/ulpfec_generator.h"
 #include "rtc_base/critical_section.h"
 #include "rtc_base/one_time_event.h"
@@ -93,6 +94,14 @@ class RTPSenderVideo {
   // or extension/
   uint32_t PacketizationOverheadBps() const;
 
+  // Recall the last RTP packet whose sequence number was |sequence_number|.
+  // Return the timestamp of the video frame that packet belonged too, as well
+  // as whether the packet was the first and/or last packet in the frame.
+  // absl::nullopt returned if no such packet can be recalled (e.g. it happened
+  // too long ago).
+  absl::optional<RtpSequenceNumberMap::Info> GetSentRtpPacketInfo(
+      uint16_t sequence_number) const;
+
  protected:
   static uint8_t GetTemporalId(const RTPVideoHeader& header);
   StorageType GetStorageType(uint8_t temporal_id,
@@ -166,6 +175,13 @@ class RTPSenderVideo {
   // delay extension on header.
   PlayoutDelayOracle* const playout_delay_oracle_;
 
+  // Maps sent packets' sequence numbers to a tuple consisting of:
+  // 1. The timestamp, without the randomizing offset mandated by the RFC.
+  // 2. Whether the packet was the first in its frame.
+  // 3. Whether the packet was the last in its frame.
+  const std::unique_ptr<RtpSequenceNumberMap> rtp_sequence_number_map_
+      RTC_PT_GUARDED_BY(crit_);
+
   // RED/ULPFEC.
   int red_payload_type_ RTC_GUARDED_BY(crit_);
   int ulpfec_payload_type_ RTC_GUARDED_BY(crit_);

video/video_send_stream_impl_unittest.cc

@@ -11,10 +11,12 @@
 #include <string>
 
 #include "absl/memory/memory.h"
+#include "absl/types/optional.h"
 #include "call/rtp_video_sender.h"
 #include "call/test/mock_bitrate_allocator.h"
 #include "call/test/mock_rtp_transport_controller_send.h"
 #include "logging/rtc_event_log/rtc_event_log.h"
+#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
 #include "modules/utility/include/process_thread.h"
 #include "modules/video_coding/fec_controller_default.h"
 #include "rtc_base/experiments/alr_experiment.h"
@@ -67,6 +69,10 @@ class MockRtpVideoSender : public RtpVideoSenderInterface {
   MOCK_CONST_METHOD0(GetPayloadBitrateBps, uint32_t());
   MOCK_CONST_METHOD0(GetProtectionBitrateBps, uint32_t());
   MOCK_METHOD3(SetEncodingData, void(size_t, size_t, size_t));
+  MOCK_CONST_METHOD2(
+      GetSentRtpPacketInfo,
+      absl::optional<RtpSequenceNumberMap::Info>(uint32_t ssrc,
+                                                 uint16_t seq_num));
 };
 
 BitrateAllocationUpdate CreateAllocation(int bitrate_bps) {