diff --git a/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 7d54ea4b24..8add0a680a 100644 --- a/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -113,11 +113,16 @@ enum KeyFrameRequestMethod { kKeyFrameReqPliRtcp, kKeyFrameReqFirRtcp }; enum RtpRtcpPacketType { kPacketRtp = 0, kPacketKeepAlive = 1 }; +// kConditionallyRetransmitHigherLayers allows retransmission of video frames +// in higher layers if either the last frame in that layer was too far back in +// time, or if we estimate that a new frame will be available in a lower layer +// in a shorter time than it would take to request and receive a retransmission. enum RetransmissionMode : uint8_t { kRetransmitOff = 0x0, kRetransmitFECPackets = 0x1, kRetransmitBaseLayer = 0x2, kRetransmitHigherLayers = 0x4, + kConditionallyRetransmitHigherLayers = 0x8, kRetransmitAllPackets = 0xFF }; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format.h b/webrtc/modules/rtp_rtcp/source/rtp_format.h index 378f38a29b..7b47921c2a 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_format.h @@ -41,10 +41,6 @@ class RtpPacketizer { // Returns true on success, false otherwise. virtual bool NextPacket(RtpPacketToSend* packet) = 0; - virtual ProtectionType GetProtectionType() = 0; - - virtual StorageType GetStorageType(uint32_t retransmission_settings) = 0; - virtual std::string ToString() = 0; }; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc index 092ac1cc49..5aa7e27d23 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -389,15 +389,6 @@ void RtpPacketizerH264::NextFragmentPacket(RtpPacketToSend* rtp_packet) { packets_.pop(); } -ProtectionType RtpPacketizerH264::GetProtectionType() { - return kProtectedPacket; -} - -StorageType RtpPacketizerH264::GetStorageType( - uint32_t retransmission_settings) { - return kAllowRetransmission; -} - std::string RtpPacketizerH264::ToString() { return "RtpPacketizerH264"; } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h index 3742ec5d36..efa9b97fb8 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h @@ -41,10 +41,6 @@ class RtpPacketizerH264 : public RtpPacketizer { // Returns true on success, false otherwise. bool NextPacket(RtpPacketToSend* rtp_packet) override; - ProtectionType GetProtectionType() override; - - StorageType GetStorageType(uint32_t retransmission_settings) override; - std::string ToString() override; private: diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.cc index 696dc2bd35..ffb5f77494 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.cc @@ -108,15 +108,6 @@ bool RtpPacketizerGeneric::NextPacket(RtpPacketToSend* packet) { return true; } -ProtectionType RtpPacketizerGeneric::GetProtectionType() { - return kProtectedPacket; -} - -StorageType RtpPacketizerGeneric::GetStorageType( - uint32_t retransmission_settings) { - return kAllowRetransmission; -} - std::string RtpPacketizerGeneric::ToString() { return "RtpPacketizerGeneric"; } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h b/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h index 16d92e7a0c..9fa3cce464 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h @@ -43,10 +43,6 @@ class RtpPacketizerGeneric : public RtpPacketizer { // Returns true on success, false otherwise. bool NextPacket(RtpPacketToSend* packet) override; - ProtectionType GetProtectionType() override; - - StorageType GetStorageType(uint32_t retransmission_settings) override; - std::string ToString() override; private: diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc index b31de933dd..e56a977f28 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc @@ -211,25 +211,6 @@ bool RtpPacketizerVp8::NextPacket(RtpPacketToSend* packet) { return true; } -ProtectionType RtpPacketizerVp8::GetProtectionType() { - bool protect = - hdr_info_.temporalIdx == 0 || hdr_info_.temporalIdx == kNoTemporalIdx; - return protect ? kProtectedPacket : kUnprotectedPacket; -} - -StorageType RtpPacketizerVp8::GetStorageType(uint32_t retransmission_settings) { - if (hdr_info_.temporalIdx == 0 && - !(retransmission_settings & kRetransmitBaseLayer)) { - return kDontRetransmit; - } - if (hdr_info_.temporalIdx != kNoTemporalIdx && - hdr_info_.temporalIdx > 0 && - !(retransmission_settings & kRetransmitHigherLayers)) { - return kDontRetransmit; - } - return kAllowRetransmission; -} - std::string RtpPacketizerVp8::ToString() { return "RtpPacketizerVp8"; } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h b/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h index edb3d4d18f..f8b7c2fe47 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h @@ -72,10 +72,6 @@ class RtpPacketizerVp8 : public RtpPacketizer { // Returns true on success, false otherwise. bool NextPacket(RtpPacketToSend* packet) override; - ProtectionType GetProtectionType() override; - - StorageType GetStorageType(uint32_t retransmission_settings) override; - std::string ToString() override; private: diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc index 6f02010172..b08ec8b03c 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc @@ -472,24 +472,6 @@ RtpPacketizerVp9::RtpPacketizerVp9(const RTPVideoHeaderVP9& hdr, RtpPacketizerVp9::~RtpPacketizerVp9() { } -ProtectionType RtpPacketizerVp9::GetProtectionType() { - bool protect = - hdr_.temporal_idx == 0 || hdr_.temporal_idx == kNoTemporalIdx; - return protect ? kProtectedPacket : kUnprotectedPacket; -} - -StorageType RtpPacketizerVp9::GetStorageType(uint32_t retransmission_settings) { - StorageType storage = kAllowRetransmission; - if (hdr_.temporal_idx == 0 && - !(retransmission_settings & kRetransmitBaseLayer)) { - storage = kDontRetransmit; - } else if (hdr_.temporal_idx != kNoTemporalIdx && hdr_.temporal_idx > 0 && - !(retransmission_settings & kRetransmitHigherLayers)) { - storage = kDontRetransmit; - } - return storage; -} - std::string RtpPacketizerVp9::ToString() { return "RtpPacketizerVp9"; } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h index 7b0b06c884..cf911a3ec6 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h @@ -39,10 +39,6 @@ class RtpPacketizerVp9 : public RtpPacketizer { virtual ~RtpPacketizerVp9(); - ProtectionType GetProtectionType() override; - - StorageType GetStorageType(uint32_t retransmission_settings) override; - std::string ToString() override; // The payload data must be one encoded VP9 layer frame. diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc index 40d25d0761..49ded0737a 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc @@ -580,37 +580,6 @@ TEST_F(RtpPacketizerVp9Test, TestRespectsLastPacketReductionLen) { EXPECT_TRUE(packet.Marker()); } -TEST_F(RtpPacketizerVp9Test, TestBaseLayerProtectionAndStorageType) { - const size_t kFrameSize = 10; - const size_t kPacketSize = 12; - - // I:0, P:0, L:1, F:1, B:1, E:1, V:0 (2hdr + 10 payload) - // L: T:0, U:0, S:0, D:0 - expected_.flexible_mode = true; - expected_.temporal_idx = 0; - Init(kFrameSize, kPacketSize); - EXPECT_EQ(kProtectedPacket, packetizer_->GetProtectionType()); - EXPECT_EQ(kAllowRetransmission, - packetizer_->GetStorageType(kRetransmitBaseLayer)); - EXPECT_EQ(kDontRetransmit, packetizer_->GetStorageType(kRetransmitOff)); -} - -TEST_F(RtpPacketizerVp9Test, TestHigherLayerProtectionAndStorageType) { - const size_t kFrameSize = 10; - const size_t kPacketSize = 12; - - // I:0, P:0, L:1, F:1, B:1, E:1, V:0 (2hdr + 10 payload) - // L: T:1, U:0, S:0, D:0 - expected_.flexible_mode = true; - expected_.temporal_idx = 1; - Init(kFrameSize, kPacketSize); - EXPECT_EQ(kUnprotectedPacket, packetizer_->GetProtectionType()); - EXPECT_EQ(kDontRetransmit, packetizer_->GetStorageType(kRetransmitBaseLayer)); - EXPECT_EQ(kAllowRetransmission, - packetizer_->GetStorageType(kRetransmitHigherLayers)); -} - - class RtpDepacketizerVp9Test : public ::testing::Test { protected: RtpDepacketizerVp9Test() diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 31d1b0e43c..6167b44246 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -31,6 +31,7 @@ namespace { const int64_t kRtpRtcpMaxIdleTimeProcessMs = 5; const int64_t kRtpRtcpRttProcessTimeMs = 1000; const int64_t kRtpRtcpBitrateProcessTimeMs = 10; +const int64_t kDefaultExpectedRetransmissionTimeMs = 125; } // namespace RTPExtensionType StringToRtpExtensionType(const std::string& extension) { @@ -430,9 +431,20 @@ bool ModuleRtpRtcpImpl::SendOutgoingData( if (rtcp_sender_.TimeToSendRTCPReport(kVideoFrameKey == frame_type)) { rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport); } + int64_t expected_retransmission_time_ms = rtt_ms(); + if (expected_retransmission_time_ms == 0) { + // No rtt available (|kRtpRtcpRttProcessTimeMs| not yet passed?), so try to + // poll avg_rtt_ms directly from rtcp receiver. + if (rtcp_receiver_.RTT(rtcp_receiver_.RemoteSSRC(), nullptr, + &expected_retransmission_time_ms, nullptr, + nullptr) == -1) { + expected_retransmission_time_ms = kDefaultExpectedRetransmissionTimeMs; + } + } return rtp_sender_->SendOutgoingData( frame_type, payload_type, time_stamp, capture_time_ms, payload_data, - payload_size, fragmentation, rtp_video_header, transport_frame_id_out); + payload_size, fragmentation, rtp_video_header, transport_frame_id_out, + expected_retransmission_time_ms); } bool ModuleRtpRtcpImpl::TimeToSendPacket(uint32_t ssrc, diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc index 89e7735c7a..8608759594 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc @@ -232,7 +232,7 @@ int32_t RTPSender::RegisterPayload( if (payload_type_map_.end() != it) { // We already use this payload type. RtpUtility::Payload* payload = it->second; - assert(payload); + RTC_DCHECK(payload); // Check if it's the same as we already have. if (RtpUtility::StringCompare( @@ -355,7 +355,7 @@ int32_t RTPSender::CheckPayloadType(int8_t payload_type, } SetSendPayloadType(payload_type); RtpUtility::Payload* payload = it->second; - assert(payload); + RTC_DCHECK(payload); if (!payload->audio && !audio_configured_) { video_->SetVideoCodecType(payload->typeSpecific.Video.videoCodecType); *video_type = payload->typeSpecific.Video.videoCodecType; @@ -371,7 +371,8 @@ bool RTPSender::SendOutgoingData(FrameType frame_type, size_t payload_size, const RTPFragmentationHeader* fragmentation, const RTPVideoHeader* rtp_header, - uint32_t* transport_frame_id_out) { + uint32_t* transport_frame_id_out, + int64_t expected_retransmission_time_ms) { uint32_t ssrc; uint16_t sequence_number; uint32_t rtp_timestamp; @@ -395,20 +396,29 @@ bool RTPSender::SendOutgoingData(FrameType frame_type, return false; } + switch (frame_type) { + case kAudioFrameSpeech: + case kAudioFrameCN: + RTC_CHECK(audio_configured_); + break; + case kVideoFrameKey: + case kVideoFrameDelta: + RTC_CHECK(!audio_configured_); + break; + case kEmptyFrame: + break; + } + bool result; if (audio_configured_) { TRACE_EVENT_ASYNC_STEP1("webrtc", "Audio", rtp_timestamp, "Send", "type", FrameTypeToString(frame_type)); - assert(frame_type == kAudioFrameSpeech || frame_type == kAudioFrameCN || - frame_type == kEmptyFrame); result = audio_->SendAudio(frame_type, payload_type, rtp_timestamp, payload_data, payload_size, fragmentation); } else { TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type", FrameTypeToString(frame_type)); - assert(frame_type != kAudioFrameSpeech && frame_type != kAudioFrameCN); - if (frame_type == kEmptyFrame) return true; @@ -419,7 +429,8 @@ bool RTPSender::SendOutgoingData(FrameType frame_type, result = video_->SendVideo(video_type, frame_type, payload_type, rtp_timestamp, capture_time_ms, payload_data, - payload_size, fragmentation, rtp_header); + payload_size, fragmentation, rtp_header, + expected_retransmission_time_ms); } rtc::CritScope cs(&statistics_crit_); @@ -1105,7 +1116,7 @@ rtc::Optional RTPSender::FlexfecSsrc() const { } void RTPSender::SetCsrcs(const std::vector& csrcs) { - assert(csrcs.size() <= kRtpCsrcSize); + RTC_DCHECK_LE(csrcs.size(), kRtpCsrcSize); rtc::CritScope lock(&send_critsect_); csrcs_ = csrcs; } @@ -1136,7 +1147,7 @@ int32_t RTPSender::SetAudioLevel(uint8_t level_d_bov) { } RtpVideoCodecTypes RTPSender::VideoCodecType() const { - assert(!audio_configured_ && "Sender is an audio stream!"); + RTC_DCHECK(!audio_configured_) << "Sender is an audio stream!"; return video_->VideoCodecType(); } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.h b/webrtc/modules/rtp_rtcp/source/rtp_sender.h index 069357f031..9c4fdc1b3d 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.h @@ -108,7 +108,8 @@ class RTPSender { size_t payload_size, const RTPFragmentationHeader* fragmentation, const RTPVideoHeader* rtp_header, - uint32_t* transport_frame_id_out); + uint32_t* transport_frame_id_out, + int64_t expected_retransmission_time_ms); // RTP header extension int32_t RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id); diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc index 14bbff781c..5a4775959e 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -25,7 +25,9 @@ #include "webrtc/modules/rtp_rtcp/source/rtp_sender.h" #include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h" #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" +#include "webrtc/rtc_base/arraysize.h" #include "webrtc/rtc_base/buffer.h" +#include "webrtc/rtc_base/ptr_util.h" #include "webrtc/rtc_base/rate_limiter.h" #include "webrtc/test/field_trial.h" #include "webrtc/test/gmock.h" @@ -55,6 +57,7 @@ const size_t kMaxPaddingSize = 224u; const int kVideoRotationExtensionId = 5; const size_t kGenericHeaderLength = 1; const uint8_t kPayloadData[] = {47, 11, 32, 93, 89}; +const int64_t kDefaultExpectedRetransmissionTimeMs = 125; using ::testing::_; using ::testing::ElementsAreArray; @@ -238,7 +241,8 @@ class RtpSenderTest : public ::testing::TestWithParam { EXPECT_TRUE(rtp_sender_->SendOutgoingData( kVideoFrameKey, kPayloadType, kTimestamp, kCaptureTimeMs, kPayloadData, - sizeof(kPayloadData), nullptr, nullptr, nullptr)); + sizeof(kPayloadData), nullptr, nullptr, nullptr, + kDefaultExpectedRetransmissionTimeMs)); } }; @@ -249,15 +253,32 @@ class RtpSenderTestWithoutPacer : public RtpSenderTest { void SetUp() override { SetUpRtpSender(false); } }; +class TestRtpSenderVideo : public RTPSenderVideo { + public: + TestRtpSenderVideo(Clock* clock, + RTPSender* rtp_sender, + FlexfecSender* flexfec_sender) + : RTPSenderVideo(clock, rtp_sender, flexfec_sender) {} + ~TestRtpSenderVideo() override {} + + StorageType GetStorageType(const RTPVideoHeader& header, + int32_t retransmission_settings, + int64_t expected_retransmission_time_ms) { + return RTPSenderVideo::GetStorageType(GetTemporalId(header), + retransmission_settings, + expected_retransmission_time_ms); + } +}; + class RtpSenderVideoTest : public RtpSenderTest { protected: void SetUp() override { // TODO(pbos): Set up to use pacer. SetUpRtpSender(false); rtp_sender_video_.reset( - new RTPSenderVideo(&fake_clock_, rtp_sender_.get(), nullptr)); + new TestRtpSenderVideo(&fake_clock_, rtp_sender_.get(), nullptr)); } - std::unique_ptr rtp_sender_video_; + std::unique_ptr rtp_sender_video_; }; TEST_P(RtpSenderTestWithoutPacer, AllocatePacketSetCsrc) { @@ -861,9 +882,9 @@ TEST_P(RtpSenderTestWithoutPacer, SendGenericVideo) { uint8_t payload[] = {47, 11, 32, 93, 89}; // Send keyframe - ASSERT_TRUE(rtp_sender_->SendOutgoingData(kVideoFrameKey, payload_type, 1234, - 4321, payload, sizeof(payload), - nullptr, nullptr, nullptr)); + ASSERT_TRUE(rtp_sender_->SendOutgoingData( + kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload), + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); auto sent_payload = transport_.last_sent_packet().payload(); uint8_t generic_header = sent_payload[0]; @@ -878,7 +899,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendGenericVideo) { ASSERT_TRUE(rtp_sender_->SendOutgoingData( kVideoFrameDelta, payload_type, 1234, 4321, payload, sizeof(payload), - nullptr, nullptr, nullptr)); + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); sent_payload = transport_.last_sent_packet().payload(); generic_header = sent_payload[0]; @@ -998,7 +1019,8 @@ TEST_P(RtpSenderTest, NoFlexfecForTimingFrames) { video_header.video_timing.flags = TimingFrameFlags::kTriggeredByTimer; EXPECT_TRUE(rtp_sender_->SendOutgoingData( kVideoFrameKey, kPayloadType, kTimestamp, kCaptureTimeMs, kPayloadData, - sizeof(kPayloadData), nullptr, &video_header, nullptr)); + sizeof(kPayloadData), nullptr, &video_header, nullptr, + kDefaultExpectedRetransmissionTimeMs)); EXPECT_CALL(mock_rtc_event_log_, LogRtpHeader(PacketDirection::kOutgoingPacket, _, _, _)) @@ -1023,7 +1045,8 @@ TEST_P(RtpSenderTest, NoFlexfecForTimingFrames) { video_header.video_timing.flags = TimingFrameFlags::kInvalid; EXPECT_TRUE(rtp_sender_->SendOutgoingData( kVideoFrameKey, kPayloadType, kTimestamp + 1, kCaptureTimeMs + 1, - kPayloadData, sizeof(kPayloadData), nullptr, &video_header, nullptr)); + kPayloadData, sizeof(kPayloadData), nullptr, &video_header, nullptr, + kDefaultExpectedRetransmissionTimeMs)); EXPECT_CALL(mock_rtc_event_log_, LogRtpHeader(PacketDirection::kOutgoingPacket, _, _, _)) @@ -1168,9 +1191,9 @@ TEST_P(RtpSenderTest, FrameCountCallbacks) { EXPECT_CALL(mock_paced_sender_, InsertPacket(_, _, _, _, _, _)) .Times(::testing::AtLeast(2)); - ASSERT_TRUE(rtp_sender_->SendOutgoingData(kVideoFrameKey, payload_type, 1234, - 4321, payload, sizeof(payload), - nullptr, nullptr, nullptr)); + ASSERT_TRUE(rtp_sender_->SendOutgoingData( + kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload), + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); EXPECT_EQ(1U, callback.num_calls_); EXPECT_EQ(ssrc, callback.ssrc_); @@ -1179,7 +1202,7 @@ TEST_P(RtpSenderTest, FrameCountCallbacks) { ASSERT_TRUE(rtp_sender_->SendOutgoingData( kVideoFrameDelta, payload_type, 1234, 4321, payload, sizeof(payload), - nullptr, nullptr, nullptr)); + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); EXPECT_EQ(2U, callback.num_calls_); EXPECT_EQ(ssrc, callback.ssrc_); @@ -1245,7 +1268,7 @@ TEST_P(RtpSenderTest, BitrateCallbacks) { for (uint32_t i = 0; i < kNumPackets; ++i) { ASSERT_TRUE(rtp_sender_->SendOutgoingData( kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload), - nullptr, nullptr, nullptr)); + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); fake_clock_.AdvanceTimeMilliseconds(kPacketInterval); } @@ -1327,8 +1350,8 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacks) { // Send a frame. ASSERT_TRUE(rtp_sender_->SendOutgoingData( - kVideoFrameKey, payload_type, 1234, 4321, payload, - sizeof(payload), nullptr, nullptr, nullptr)); + kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload), + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); StreamDataCounters expected; expected.transmitted.payload_bytes = 6; expected.transmitted.header_bytes = 12; @@ -1369,8 +1392,8 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacks) { fec_params.max_fec_frames = 1; rtp_sender_->SetFecParameters(fec_params, fec_params); ASSERT_TRUE(rtp_sender_->SendOutgoingData( - kVideoFrameDelta, payload_type, 1234, 4321, payload, - sizeof(payload), nullptr, nullptr, nullptr)); + kVideoFrameDelta, payload_type, 1234, 4321, payload, sizeof(payload), + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); expected.transmitted.payload_bytes = 40; expected.transmitted.header_bytes = 60; expected.transmitted.packets = 5; @@ -1388,8 +1411,8 @@ TEST_P(RtpSenderAudioTest, SendAudio) { uint8_t payload[] = {47, 11, 32, 93, 89}; ASSERT_TRUE(rtp_sender_->SendOutgoingData( - kAudioFrameCN, payload_type, 1234, 4321, payload, - sizeof(payload), nullptr, nullptr, nullptr)); + kAudioFrameCN, payload_type, 1234, 4321, payload, sizeof(payload), + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); auto sent_payload = transport_.last_sent_packet().payload(); EXPECT_THAT(sent_payload, ElementsAreArray(payload)); @@ -1407,8 +1430,8 @@ TEST_P(RtpSenderAudioTest, SendAudioWithAudioLevelExtension) { uint8_t payload[] = {47, 11, 32, 93, 89}; ASSERT_TRUE(rtp_sender_->SendOutgoingData( - kAudioFrameCN, payload_type, 1234, 4321, payload, - sizeof(payload), nullptr, nullptr, nullptr)); + kAudioFrameCN, payload_type, 1234, 4321, payload, sizeof(payload), + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); auto sent_payload = transport_.last_sent_packet().payload(); EXPECT_THAT(sent_payload, ElementsAreArray(payload)); @@ -1445,22 +1468,22 @@ TEST_P(RtpSenderAudioTest, CheckMarkerBitForTelephoneEvents) { // During start, it takes the starting timestamp as last sent timestamp. // The duration is calculated as the difference of current and last sent // timestamp. So for first call it will skip since the duration is zero. - ASSERT_TRUE(rtp_sender_->SendOutgoingData(kEmptyFrame, kPayloadType, - capture_time_ms, 0, nullptr, 0, - nullptr, nullptr, nullptr)); + ASSERT_TRUE(rtp_sender_->SendOutgoingData( + kEmptyFrame, kPayloadType, capture_time_ms, 0, nullptr, 0, nullptr, + nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); // DTMF Sample Length is (Frequency/1000) * Duration. // So in this case, it is (8000/1000) * 500 = 4000. // Sending it as two packets. ASSERT_TRUE(rtp_sender_->SendOutgoingData( - kEmptyFrame, kPayloadType, capture_time_ms + 2000, 0, - nullptr, 0, nullptr, nullptr, nullptr)); + kEmptyFrame, kPayloadType, capture_time_ms + 2000, 0, nullptr, 0, nullptr, + nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); // Marker Bit should be set to 1 for first packet. EXPECT_TRUE(transport_.last_sent_packet().Marker()); ASSERT_TRUE(rtp_sender_->SendOutgoingData( - kEmptyFrame, kPayloadType, capture_time_ms + 4000, 0, - nullptr, 0, nullptr, nullptr, nullptr)); + kEmptyFrame, kPayloadType, capture_time_ms + 4000, 0, nullptr, 0, nullptr, + nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); // Marker Bit should be set to 0 for rest of the packets. EXPECT_FALSE(transport_.last_sent_packet().Marker()); } @@ -1478,8 +1501,8 @@ TEST_P(RtpSenderTestWithoutPacer, BytesReportedCorrectly) { uint8_t payload[] = {47, 11, 32, 93, 89}; ASSERT_TRUE(rtp_sender_->SendOutgoingData( - kVideoFrameKey, kPayloadType, 1234, 4321, payload, - sizeof(payload), nullptr, nullptr, nullptr)); + kVideoFrameKey, kPayloadType, 1234, 4321, payload, sizeof(payload), + nullptr, nullptr, nullptr, kDefaultExpectedRetransmissionTimeMs)); // Will send 2 full-size padding packets. rtp_sender_->TimeToSendPadding(1, PacedPacketInfo()); @@ -1553,7 +1576,7 @@ TEST_P(RtpSenderVideoTest, KeyFrameHasCVO) { hdr.rotation = kVideoRotation_0; rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, kPayload, kTimestamp, 0, kFrame, sizeof(kFrame), nullptr, - &hdr); + &hdr, kDefaultExpectedRetransmissionTimeMs); VideoRotation rotation; EXPECT_TRUE( @@ -1579,7 +1602,8 @@ TEST_P(RtpSenderVideoTest, TimingFrameHasPacketizationTimstampSet) { fake_clock_.AdvanceTimeMilliseconds(kPacketizationTimeMs); rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, kPayload, kTimestamp, kCaptureTimestamp, kFrame, - sizeof(kFrame), nullptr, &hdr); + sizeof(kFrame), nullptr, &hdr, + kDefaultExpectedRetransmissionTimeMs); VideoSendTiming timing; EXPECT_TRUE(transport_.last_sent_packet().GetExtension( &timing)); @@ -1595,14 +1619,14 @@ TEST_P(RtpSenderVideoTest, DeltaFrameHasCVOWhenChanged) { RTPVideoHeader hdr = {0}; hdr.rotation = kVideoRotation_90; - EXPECT_TRUE(rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, - kPayload, kTimestamp, 0, kFrame, - sizeof(kFrame), nullptr, &hdr)); + EXPECT_TRUE(rtp_sender_video_->SendVideo( + kRtpVideoGeneric, kVideoFrameKey, kPayload, kTimestamp, 0, kFrame, + sizeof(kFrame), nullptr, &hdr, kDefaultExpectedRetransmissionTimeMs)); hdr.rotation = kVideoRotation_0; - EXPECT_TRUE(rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameDelta, - kPayload, kTimestamp + 1, 0, kFrame, - sizeof(kFrame), nullptr, &hdr)); + EXPECT_TRUE(rtp_sender_video_->SendVideo( + kRtpVideoGeneric, kVideoFrameDelta, kPayload, kTimestamp + 1, 0, kFrame, + sizeof(kFrame), nullptr, &hdr, kDefaultExpectedRetransmissionTimeMs)); VideoRotation rotation; EXPECT_TRUE( @@ -1617,13 +1641,13 @@ TEST_P(RtpSenderVideoTest, DeltaFrameHasCVOWhenNonZero) { RTPVideoHeader hdr = {0}; hdr.rotation = kVideoRotation_90; - EXPECT_TRUE(rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, - kPayload, kTimestamp, 0, kFrame, - sizeof(kFrame), nullptr, &hdr)); + EXPECT_TRUE(rtp_sender_video_->SendVideo( + kRtpVideoGeneric, kVideoFrameKey, kPayload, kTimestamp, 0, kFrame, + sizeof(kFrame), nullptr, &hdr, kDefaultExpectedRetransmissionTimeMs)); - EXPECT_TRUE(rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameDelta, - kPayload, kTimestamp + 1, 0, kFrame, - sizeof(kFrame), nullptr, &hdr)); + EXPECT_TRUE(rtp_sender_video_->SendVideo( + kRtpVideoGeneric, kVideoFrameDelta, kPayload, kTimestamp + 1, 0, kFrame, + sizeof(kFrame), nullptr, &hdr, kDefaultExpectedRetransmissionTimeMs)); VideoRotation rotation; EXPECT_TRUE( @@ -1652,6 +1676,224 @@ TEST_P(RtpSenderVideoTest, SendVideoWithCameraAndFlipCVO) { ConvertCVOByteToVideoRotation(flip_bit | camera_bit | 3)); } +TEST_P(RtpSenderVideoTest, RetransmissionTypesGeneric) { + RTPVideoHeader header; + header.codec = kRtpVideoGeneric; + + EXPECT_EQ(kDontRetransmit, + rtp_sender_video_->GetStorageType( + header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitBaseLayer, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, + rtp_sender_video_->GetStorageType( + header, kConditionallyRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitAllPackets, + kDefaultExpectedRetransmissionTimeMs)); +} + +TEST_P(RtpSenderVideoTest, RetransmissionTypesH264) { + RTPVideoHeader header; + header.codec = kRtpVideoH264; + header.codecHeader.H264.packetization_mode = + H264PacketizationMode::NonInterleaved; + + EXPECT_EQ(kDontRetransmit, + rtp_sender_video_->GetStorageType( + header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitBaseLayer, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, + rtp_sender_video_->GetStorageType( + header, kConditionallyRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitAllPackets, + kDefaultExpectedRetransmissionTimeMs)); +} + +TEST_P(RtpSenderVideoTest, RetransmissionTypesVP8BaseLayer) { + RTPVideoHeader header; + header.codec = kRtpVideoVp8; + header.codecHeader.VP8.temporalIdx = 0; + + EXPECT_EQ(kDontRetransmit, + rtp_sender_video_->GetStorageType( + header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitBaseLayer, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType( + header, kRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, + rtp_sender_video_->GetStorageType( + header, kRetransmitHigherLayers | kRetransmitBaseLayer, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType( + header, kConditionallyRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ( + kAllowRetransmission, + rtp_sender_video_->GetStorageType( + header, kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitAllPackets, + kDefaultExpectedRetransmissionTimeMs)); +} + +TEST_P(RtpSenderVideoTest, RetransmissionTypesVP8HigherLayers) { + RTPVideoHeader header; + header.codec = kRtpVideoVp8; + + for (int tid = 1; tid <= kMaxTemporalStreams; ++tid) { + header.codecHeader.VP8.temporalIdx = tid; + + EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType( + header, kRetransmitOff, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType( + header, kRetransmitBaseLayer, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, + rtp_sender_video_->GetStorageType( + header, kRetransmitHigherLayers | kRetransmitBaseLayer, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitAllPackets, + kDefaultExpectedRetransmissionTimeMs)); + } +} + +TEST_P(RtpSenderVideoTest, RetransmissionTypesVP9) { + RTPVideoHeader header; + header.codec = kRtpVideoVp9; + + for (int tid = 1; tid <= kMaxTemporalStreams; ++tid) { + header.codecHeader.VP9.temporal_idx = tid; + + EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType( + header, kRetransmitOff, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kDontRetransmit, rtp_sender_video_->GetStorageType( + header, kRetransmitBaseLayer, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitHigherLayers, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, + rtp_sender_video_->GetStorageType( + header, kRetransmitHigherLayers | kRetransmitBaseLayer, + kDefaultExpectedRetransmissionTimeMs)); + EXPECT_EQ(kAllowRetransmission, rtp_sender_video_->GetStorageType( + header, kRetransmitAllPackets, + kDefaultExpectedRetransmissionTimeMs)); + } +} + +TEST_P(RtpSenderVideoTest, ConditionalRetransmit) { + const int64_t kFrameIntervalMs = 33; + const int64_t kRttMs = (kFrameIntervalMs * 3) / 2; + const uint8_t kSettings = + kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers; + + // Insert VP8 frames for all temporal layers, but stop before the final index. + RTPVideoHeader header; + header.codec = kRtpVideoVp8; + + // Fill averaging window to prevent rounding errors. + constexpr int kNumRepetitions = + (RTPSenderVideo::kTLRateWindowSizeMs + (kFrameIntervalMs / 2)) / + kFrameIntervalMs; + constexpr int kPattern[] = {0, 2, 1, 2}; + for (size_t i = 0; i < arraysize(kPattern) * kNumRepetitions; ++i) { + header.codecHeader.VP8.temporalIdx = kPattern[i % arraysize(kPattern)]; + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs); + fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + } + + // Since we're at the start of the pattern, the next expected frame in TL0 is + // right now. We will wait at most one expected retransmission time before + // acknowledging that it did not arrive, which means this frame and the next + // will not be retransmitted. + header.codecHeader.VP8.temporalIdx = 1; + EXPECT_EQ(StorageType::kDontRetransmit, + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs)); + fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + EXPECT_EQ(StorageType::kDontRetransmit, + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs)); + fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + + // The TL0 frame did not arrive. So allow retransmission. + EXPECT_EQ(StorageType::kAllowRetransmission, + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs)); + fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + + // Insert a frame for TL2. We just had frame in TL1, so the next one there is + // in three frames away. TL0 is still too far in the past. So, allow + // retransmission. + header.codecHeader.VP8.temporalIdx = 2; + EXPECT_EQ(StorageType::kAllowRetransmission, + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs)); + fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + + // Another TL2, next in TL1 is two frames away. Allow again. + EXPECT_EQ(StorageType::kAllowRetransmission, + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs)); + fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + + // Yet another TL2, next in TL1 is now only one frame away, so don't store + // for retransmission. + EXPECT_EQ(StorageType::kDontRetransmit, + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs)); +} + +TEST_P(RtpSenderVideoTest, ConditionalRetransmitLimit) { + const int64_t kFrameIntervalMs = 200; + const int64_t kRttMs = (kFrameIntervalMs * 3) / 2; + const int32_t kSettings = + kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers; + + // Insert VP8 frames for all temporal layers, but stop before the final index. + RTPVideoHeader header; + header.codec = kRtpVideoVp8; + + // Fill averaging window to prevent rounding errors. + constexpr int kNumRepetitions = + (RTPSenderVideo::kTLRateWindowSizeMs + (kFrameIntervalMs / 2)) / + kFrameIntervalMs; + constexpr int kPattern[] = {0, 2, 2, 2}; + for (size_t i = 0; i < arraysize(kPattern) * kNumRepetitions; ++i) { + header.codecHeader.VP8.temporalIdx = kPattern[i % arraysize(kPattern)]; + + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs); + fake_clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); + } + + // Since we're at the start of the pattern, the next expected frame will be + // right now in TL0. Put it in TL1 instead. Regular rules would dictate that + // we don't store for retransmission because we expect a frame in a lower + // layer, but that last frame in TL1 was a long time ago in absolute terms, + // so allow retransmission anyway. + header.codecHeader.VP8.temporalIdx = 1; + EXPECT_EQ(StorageType::kAllowRetransmission, + rtp_sender_video_->GetStorageType(header, kSettings, kRttMs)); +} + TEST_P(RtpSenderTest, OnOverheadChanged) { MockOverheadObserver mock_overhead_observer; rtp_sender_.reset( diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc index 38450742be..4e46b78baa 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -13,9 +13,10 @@ #include #include +#include #include -#include #include +#include #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "webrtc/modules/rtp_rtcp/source/byte_io.h" @@ -33,6 +34,7 @@ namespace webrtc { namespace { constexpr size_t kRedForFecHeaderLength = 1; +constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4; void BuildRedPayload(const RtpPacketToSend& media_packet, RtpPacketToSend* red_packet) { @@ -53,7 +55,8 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock, : rtp_sender_(rtp_sender), clock_(clock), video_type_(kRtpVideoGeneric), - retransmission_settings_(kRetransmitBaseLayer), + retransmission_settings_(kRetransmitBaseLayer | + kConditionallyRetransmitHigherLayers), last_rotation_(kVideoRotation_0), red_payload_type_(-1), ulpfec_payload_type_(-1), @@ -292,7 +295,8 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, const uint8_t* payload_data, size_t payload_size, const RTPFragmentationHeader* fragmentation, - const RTPVideoHeader* video_header) { + const RTPVideoHeader* video_header, + int64_t expected_retransmission_time_ms) { if (payload_size == 0) return false; @@ -365,8 +369,11 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, std::unique_ptr packetizer(RtpPacketizer::Create( video_type, max_data_payload_length, last_packet_reduction_len, video_header ? &(video_header->codecHeader) : nullptr, frame_type)); - // Media packet storage. - StorageType storage = packetizer->GetStorageType(retransmission_settings); + + const uint8_t temporal_id = + video_header ? GetTemporalId(*video_header) : kNoTemporalIdx; + StorageType storage = GetStorageType(temporal_id, retransmission_settings, + expected_retransmission_time_ms); // TODO(changbin): we currently don't support to configure the codec to // output multiple partitions for VP8. Should remove below check after the @@ -392,7 +399,9 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, if (!rtp_sender_->AssignSequenceNumber(packet.get())) return false; - bool protect_packet = (packetizer->GetProtectionType() == kProtectedPacket); + // No FEC protection for upper temporal layers, if used. + bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx; + // Put packetization finish timestamp into extension. if (packet->HasExtension()) { packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds()); @@ -453,4 +462,90 @@ void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) { retransmission_settings_ = settings; } +StorageType RTPSenderVideo::GetStorageType( + uint8_t temporal_id, + int32_t retransmission_settings, + int64_t expected_retransmission_time_ms) { + if (retransmission_settings == kRetransmitOff) + return StorageType::kDontRetransmit; + if (retransmission_settings == kRetransmitAllPackets) + return StorageType::kAllowRetransmission; + + rtc::CritScope cs(&stats_crit_); + // Media packet storage. + if ((retransmission_settings & kConditionallyRetransmitHigherLayers) && + UpdateConditionalRetransmit(temporal_id, + expected_retransmission_time_ms)) { + retransmission_settings |= kRetransmitHigherLayers; + } + + if (temporal_id == kNoTemporalIdx) + return kAllowRetransmission; + + if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0) + return kAllowRetransmission; + + if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0) + return kAllowRetransmission; + + return kDontRetransmit; +} + +uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) { + switch (header.codec) { + case kRtpVideoVp8: + return header.codecHeader.VP8.temporalIdx; + case kRtpVideoVp9: + return header.codecHeader.VP9.temporal_idx; + default: + return kNoTemporalIdx; + } +} + +bool RTPSenderVideo::UpdateConditionalRetransmit( + uint8_t temporal_id, + int64_t expected_retransmission_time_ms) { + int64_t now_ms = clock_->TimeInMilliseconds(); + // Update stats for any temporal layer. + TemporalLayerStats* current_layer_stats = + &frame_stats_by_temporal_layer_[temporal_id]; + current_layer_stats->frame_rate_fp1000s.Update(1, now_ms); + int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_ms; + current_layer_stats->last_frame_time_ms = now_ms; + + // Conditional retransmit only applies to upper layers. + if (temporal_id != kNoTemporalIdx && temporal_id > 0) { + if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) { + // Too long since a retransmittable frame in this layer, enable NACK + // protection. + return true; + } else { + // Estimate when the next frame of any lower layer will be sent. + const int64_t kUndefined = std::numeric_limits::max(); + int64_t expected_next_frame_time = kUndefined; + for (int i = temporal_id - 1; i >= 0; --i) { + TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i]; + rtc::Optional rate = stats->frame_rate_fp1000s.Rate(now_ms); + if (rate) { + int64_t tl_next = stats->last_frame_time_ms + 1000000 / *rate; + if (tl_next - now_ms > -expected_retransmission_time_ms && + tl_next < expected_next_frame_time) { + expected_next_frame_time = tl_next; + } + } + } + + if (expected_next_frame_time == kUndefined || + expected_next_frame_time - now_ms > expected_retransmission_time_ms) { + // The next frame in a lower layer is expected at a later time (or + // unable to tell due to lack of data) than a retransmission is + // estimated to be able to arrive, so allow this packet to be nacked. + return true; + } + } + } + + return false; +} + } // namespace webrtc diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h index 9fb4648b82..e4a24052e5 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h @@ -11,9 +11,8 @@ #ifndef WEBRTC_MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ #define WEBRTC_MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ -#include +#include #include -#include #include "webrtc/common_types.h" #include "webrtc/modules/rtp_rtcp/include/flexfec_sender.h" @@ -32,10 +31,13 @@ #include "webrtc/typedefs.h" namespace webrtc { +class RtpPacketizer; class RtpPacketToSend; class RTPSenderVideo { public: + static constexpr int64_t kTLRateWindowSizeMs = 2500; + RTPSenderVideo(Clock* clock, RTPSender* rtpSender, FlexfecSender* flexfec_sender); @@ -55,7 +57,8 @@ class RTPSenderVideo { const uint8_t* payload_data, size_t payload_size, const RTPFragmentationHeader* fragmentation, - const RTPVideoHeader* video_header); + const RTPVideoHeader* video_header, + int64_t expected_retransmission_time_ms); void SetVideoCodecType(RtpVideoCodecTypes type); @@ -76,7 +79,24 @@ class RTPSenderVideo { int SelectiveRetransmissions() const; void SetSelectiveRetransmissions(uint8_t settings); + protected: + static uint8_t GetTemporalId(const RTPVideoHeader& header); + StorageType GetStorageType(uint8_t temporal_id, + int32_t retransmission_settings, + int64_t expected_retransmission_time_ms); + private: + struct TemporalLayerStats { + TemporalLayerStats() + : frame_rate_fp1000s(kTLRateWindowSizeMs, 1000 * 1000), + last_frame_time_ms(0) {} + // Frame rate, in frames per 1000 seconds. This essentially turns the fps + // value into a fixed point value with three decimals. Improves precision at + // low frame rates. + RateStatistics frame_rate_fp1000s; + int64_t last_frame_time_ms; + }; + size_t CalculateFecPacketOverhead() const EXCLUSIVE_LOCKS_REQUIRED(crit_); void SendVideoPacket(std::unique_ptr packet, @@ -103,6 +123,10 @@ class RTPSenderVideo { bool flexfec_enabled() const { return flexfec_sender_ != nullptr; } + bool UpdateConditionalRetransmit(uint8_t temporal_id, + int64_t expected_retransmission_time_ms) + EXCLUSIVE_LOCKS_REQUIRED(stats_crit_); + RTPSender* const rtp_sender_; Clock* const clock_; @@ -131,6 +155,10 @@ class RTPSenderVideo { RateStatistics fec_bitrate_ GUARDED_BY(stats_crit_); // Bitrate used for video payload and RTP headers. RateStatistics video_bitrate_ GUARDED_BY(stats_crit_); + + std::map frame_stats_by_temporal_layer_ + GUARDED_BY(stats_crit_); + OneTimeEvent first_frame_sent_; };