diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc
index 07e8e625ed..796575b095 100644
--- a/call/rtp_video_sender.cc
+++ b/call/rtp_video_sender.cc
@@ -49,10 +49,6 @@ RtpStreamSender::~RtpStreamSender() = default;
 
 namespace {
 static const int kMinSendSidePacketHistorySize = 600;
-// Assume an average video stream has around 3 packets per frame (1 mbps / 30
-// fps / 1400B) A sequence number set with size 5500 will be able to store
-// packet sequence number for at least last 60 seconds.
-static const int kSendSideSeqNumSetMaxSize = 5500;
 // We don't do MTU discovery, so assume that we have the standard ethernet MTU.
 static const size_t kPathMTU = 1500;
 
@@ -780,30 +776,22 @@ int RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params,
   return 0;
 }
 
-void RtpVideoSender::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) {
-  const auto ssrcs = rtp_config_.ssrcs;
-  if (std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()) {
-    feedback_packet_seq_num_set_.insert(seq_num);
-    if (feedback_packet_seq_num_set_.size() > kSendSideSeqNumSetMaxSize) {
-      RTC_LOG(LS_WARNING) << "Feedback packet sequence number set exceed it's "
-                             "max size', will get reset.";
-      feedback_packet_seq_num_set_.clear();
-    }
-  }
-}
-
 void RtpVideoSender::OnPacketFeedbackVector(
     const std::vector<PacketFeedback>& packet_feedback_vector) {
   if (fec_controller_->UseLossVectorMask()) {
     rtc::CritScope cs(&crit_);
     for (const PacketFeedback& packet : packet_feedback_vector) {
-      const bool lost = packet.arrival_time_ms == PacketFeedback::kNotReceived;
-      // Lost feedbacks are not considered to be lost packets.
-      auto it = feedback_packet_seq_num_set_.find(packet.sequence_number);
-      if (it != feedback_packet_seq_num_set_.end()) {
-        loss_mask_vector_.push_back(lost);
-        feedback_packet_seq_num_set_.erase(it);
+      if (packet.send_time_ms == PacketFeedback::kNoSendTime || !packet.ssrc ||
+          std::find(rtp_config_.ssrcs.begin(), rtp_config_.ssrcs.end(),
+                    *packet.ssrc) == rtp_config_.ssrcs.end()) {
+        // If packet send time is missing, the feedback for this packet has
+        // probably already been processed, so ignore it.
+        // If packet does not belong to a registered media ssrc, we are also
+        // not interested in it.
+        continue;
       }
+      loss_mask_vector_.push_back(packet.arrival_time_ms ==
+                                  PacketFeedback::kNotReceived);
     }
   }
 
diff --git a/call/rtp_video_sender.h b/call/rtp_video_sender.h
index 162875b204..db8ce85b5b 100644
--- a/call/rtp_video_sender.h
+++ b/call/rtp_video_sender.h
@@ -146,7 +146,7 @@ class RtpVideoSender : public RtpVideoSenderInterface,
       rtc::ArrayView<const uint16_t> sequence_numbers) const override;
 
   // From PacketFeedbackObserver.
-  void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override;
+  void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override {}
   void OnPacketFeedbackVector(
       const std::vector<PacketFeedback>& packet_feedback_vector) override;
 
@@ -191,7 +191,6 @@ class RtpVideoSender : public RtpVideoSenderInterface,
   uint32_t protection_bitrate_bps_;
   uint32_t encoder_target_rate_bps_;
 
-  std::unordered_set<uint16_t> feedback_packet_seq_num_set_;
   std::vector<bool> loss_mask_vector_ RTC_GUARDED_BY(crit_);
 
   std::vector<FrameCounts> frame_counts_ RTC_GUARDED_BY(crit_);
diff --git a/call/rtp_video_sender_unittest.cc b/call/rtp_video_sender_unittest.cc
index d0fe7b311f..bd23bc6766 100644
--- a/call/rtp_video_sender_unittest.cc
+++ b/call/rtp_video_sender_unittest.cc
@@ -433,6 +433,8 @@ TEST(RtpVideoSenderTest, DoesNotRetrasmitAckedPackets) {
   EXPECT_EQ(
       EncodedImageCallback::Result::OK,
       test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
+  const int64_t send_time_ms = test.clock().TimeInMilliseconds();
+
   test.clock().AdvanceTimeMilliseconds(33);
 
   ASSERT_TRUE(event.Wait(kTimeoutMs));
@@ -473,11 +475,13 @@ TEST(RtpVideoSenderTest, DoesNotRetrasmitAckedPackets) {
                                           transport_sequence_numbers[0]);
   received_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[0];
   received_packet_feedback.ssrc = kSsrc1;
+  received_packet_feedback.send_time_ms = send_time_ms;
 
   PacketFeedback lost_packet_feedback(PacketFeedback::kNotReceived,
                                       transport_sequence_numbers[1]);
   lost_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[1];
   lost_packet_feedback.ssrc = kSsrc1;
+  lost_packet_feedback.send_time_ms = send_time_ms;
   std::vector<PacketFeedback> feedback_vector = {received_packet_feedback,
                                                  lost_packet_feedback};