From 131bc498e65a29e4224648d6d6b72abd3de87592 Mon Sep 17 00:00:00 2001 From: brandtr Date: Thu, 10 Nov 2016 05:01:11 -0800 Subject: [PATCH] Wire up FlexfecSender in RTPSenderVideo. This CL adds the ability for RTPSenderVideo to generate and send FlexFEC packets. BUG=webrtc:5654 Review-Url: https://codereview.webrtc.org/2490523002 Cr-Commit-Position: refs/heads/master@{#15016} --- .../modules/rtp_rtcp/include/flexfec_sender.h | 2 +- .../modules/rtp_rtcp/source/flexfec_sender.cc | 43 ++++--- .../rtp_rtcp/source/rtp_sender_video.cc | 120 +++++++++++++----- .../rtp_rtcp/source/rtp_sender_video.h | 25 +++- 4 files changed, 129 insertions(+), 61 deletions(-) diff --git a/webrtc/modules/rtp_rtcp/include/flexfec_sender.h b/webrtc/modules/rtp_rtcp/include/flexfec_sender.h index f50dfb6b39..058e0160f6 100644 --- a/webrtc/modules/rtp_rtcp/include/flexfec_sender.h +++ b/webrtc/modules/rtp_rtcp/include/flexfec_sender.h @@ -74,7 +74,7 @@ class FlexfecSender { // Implementation. UlpfecGenerator ulpfec_generator_ GUARDED_BY(sequence_checker_); - RtpHeaderExtensionMap rtp_header_extension_map_ GUARDED_BY(sequence_checker_); + const RtpHeaderExtensionMap rtp_header_extension_map_; }; } // namespace webrtc diff --git a/webrtc/modules/rtp_rtcp/source/flexfec_sender.cc b/webrtc/modules/rtp_rtcp/source/flexfec_sender.cc index 55e8e52669..ec8bbdcbe9 100644 --- a/webrtc/modules/rtp_rtcp/source/flexfec_sender.cc +++ b/webrtc/modules/rtp_rtcp/source/flexfec_sender.cc @@ -38,6 +38,25 @@ const int kMsToRtpTimestamp = kVideoPayloadTypeFrequency / 1000; // How often to log the generated FEC packets to the text log. constexpr int64_t kPacketLogIntervalMs = 10000; +RtpHeaderExtensionMap RegisterBweExtensions( + const std::vector& rtp_header_extensions) { + RtpHeaderExtensionMap map; + for (const auto& extension : rtp_header_extensions) { + if (extension.uri == TransportSequenceNumber::kUri) { + map.Register(extension.id); + } else if (extension.uri == AbsoluteSendTime::kUri) { + map.Register(extension.id); + } else if (extension.uri == TransmissionOffset::kUri) { + map.Register(extension.id); + } else { + LOG(LS_INFO) << "FlexfecSender only supports RTP header extensions for " + << "BWE, so the extension " << extension.ToString() + << " will not be used."; + } + } + return map; +} + } // namespace FlexfecSender::FlexfecSender( @@ -57,7 +76,7 @@ FlexfecSender::FlexfecSender( protected_media_ssrc_(protected_media_ssrc), seq_num_(random_.Rand(1, kMaxInitRtpSeqNumber)), ulpfec_generator_(ForwardErrorCorrection::CreateFlexfec()), - rtp_header_extension_map_() { + rtp_header_extension_map_(RegisterBweExtensions(rtp_header_extensions)) { // This object should not have been instantiated if FlexFEC is disabled. RTC_DCHECK_GE(payload_type, 0); RTC_DCHECK_LE(payload_type, 127); @@ -65,24 +84,6 @@ FlexfecSender::FlexfecSender( // It's OK to create this object on a different thread/task queue than // the one used during main operation. sequence_checker_.Detach(); - - // Register RTP header extensions for BWE. - for (const auto& extension : rtp_header_extensions) { - if (extension.uri == RtpExtension::kTransportSequenceNumberUri) { - rtp_header_extension_map_.Register(kRtpExtensionTransportSequenceNumber, - extension.id); - } else if (extension.uri == RtpExtension::kAbsSendTimeUri) { - rtp_header_extension_map_.Register(kRtpExtensionAbsoluteSendTime, - extension.id); - } else if (extension.uri == RtpExtension::kTimestampOffsetUri) { - rtp_header_extension_map_.Register(kRtpExtensionTransmissionTimeOffset, - extension.id); - } else { - LOG(LS_WARNING) << "RTP header extension with id: " << extension.id - << ", uri: " << extension.uri - << ", is unsupported by FlexfecSender."; - } - } } FlexfecSender::~FlexfecSender() = default; @@ -158,8 +159,10 @@ FlexfecSender::GetFecPackets() { return fec_packets_to_send; } +// The overhead is BWE RTP header extensions and FlexFEC header. size_t FlexfecSender::MaxPacketOverhead() const { - return kFlexfecMaxHeaderSize; + return rtp_header_extension_map_.GetTotalLengthInBytes() + + kFlexfecMaxHeaderSize; } } // namespace webrtc diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc index 916561ea87..f80caeaf7e 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -52,6 +52,7 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock, RTPSender* rtp_sender) last_rotation_(kVideoRotation_0), red_payload_type_(-1), ulpfec_payload_type_(-1), + flexfec_sender_(nullptr), // TODO(brandtr): Wire up in future CL. delta_fec_params_{0, 1, kFecMaskRandom}, key_fec_params_{0, 1, kFecMaskRandom}, fec_bitrate_(1000, RateStatistics::kBpsScale), @@ -111,10 +112,10 @@ void RTPSenderVideo::SendVideoPacket(std::unique_ptr packet, "seqnum", seq_num); } -void RTPSenderVideo::SendVideoPacketAsRed( +void RTPSenderVideo::SendVideoPacketAsRedMaybeWithUlpfec( std::unique_ptr media_packet, StorageType media_packet_storage, - bool protect) { + bool protect_media_packet) { uint32_t rtp_timestamp = media_packet->Timestamp(); uint16_t media_seq_num = media_packet->SequenceNumber(); @@ -128,21 +129,23 @@ void RTPSenderVideo::SendVideoPacketAsRed( // Only protect while creating RED and FEC packets, not when sending. rtc::CritScope cs(&crit_); red_packet->SetPayloadType(red_payload_type_); - if (protect) { - ulpfec_generator_.AddRtpPacketAndGenerateFec( - media_packet->data(), media_packet->payload_size(), - media_packet->headers_size()); - } - uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets(); - if (num_fec_packets > 0) { - uint16_t first_fec_sequence_number = - rtp_sender_->AllocateSequenceNumber(num_fec_packets); - fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed( - red_payload_type_, ulpfec_payload_type_, first_fec_sequence_number, - media_packet->headers_size()); - RTC_DCHECK_EQ(num_fec_packets, fec_packets.size()); - if (retransmission_settings_ & kRetransmitFECPackets) - fec_storage = kAllowRetransmission; + if (ulpfec_enabled()) { + if (protect_media_packet) { + ulpfec_generator_.AddRtpPacketAndGenerateFec( + media_packet->data(), media_packet->payload_size(), + media_packet->headers_size()); + } + uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets(); + if (num_fec_packets > 0) { + uint16_t first_fec_sequence_number = + rtp_sender_->AllocateSequenceNumber(num_fec_packets); + fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed( + red_payload_type_, ulpfec_payload_type_, first_fec_sequence_number, + media_packet->headers_size()); + RTC_DCHECK_EQ(num_fec_packets, fec_packets.size()); + if (retransmission_settings_ & kRetransmitFECPackets) + fec_storage = kAllowRetransmission; + } } } // Send |red_packet| instead of |packet| for allocated sequence number. @@ -170,10 +173,40 @@ void RTPSenderVideo::SendVideoPacketAsRed( rtc::CritScope cs(&stats_crit_); fec_bitrate_.Update(fec_packet->length(), clock_->TimeInMilliseconds()); TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"), - "Video::PacketFec", "timestamp", rtp_timestamp, + "Video::PacketUlpfec", "timestamp", rtp_timestamp, "seqnum", fec_sequence_number); } else { - LOG(LS_WARNING) << "Failed to send FEC packet " << fec_sequence_number; + LOG(LS_WARNING) << "Failed to send ULPFEC packet " << fec_sequence_number; + } + } +} + +void RTPSenderVideo::SendVideoPacketWithFlexfec( + std::unique_ptr media_packet, + StorageType media_packet_storage, + bool protect_media_packet) { + RTC_DCHECK(flexfec_sender_); + + if (protect_media_packet) + flexfec_sender_->AddRtpPacketAndGenerateFec(*media_packet); + + SendVideoPacket(std::move(media_packet), media_packet_storage); + + if (flexfec_sender_->FecAvailable()) { + std::vector> fec_packets = + flexfec_sender_->GetFecPackets(); + for (auto& fec_packet : fec_packets) { + uint32_t timestamp = fec_packet->Timestamp(); + uint16_t seq_num = fec_packet->SequenceNumber(); + if (rtp_sender_->SendToNetwork(std::move(fec_packet), kDontRetransmit, + RtpPacketSender::kLowPriority)) { + // TODO(brandtr): Wire up stats here. + TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"), + "Video::PacketFlexfec", "timestamp", timestamp, + "seqnum", seq_num); + } else { + LOG(LS_WARNING) << "Failed to send FlexFEC packet " << seq_num; + } } } } @@ -198,7 +231,7 @@ void RTPSenderVideo::SetUlpfecConfig(int red_payload_type, // ensure that RED and ULPFEC are only enabled together. RTC_DCHECK(red_enabled() || !ulpfec_enabled()); - // Reset FEC rates. + // Reset FEC parameters. delta_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom}; key_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom}; } @@ -211,29 +244,33 @@ void RTPSenderVideo::GetUlpfecConfig(int* red_payload_type, } size_t RTPSenderVideo::FecPacketOverhead() const { + if (flexfec_enabled()) + return flexfec_sender_->MaxPacketOverhead(); + rtc::CritScope cs(&crit_); size_t overhead = 0; if (red_enabled()) { - // Overhead is FEC headers plus RED for FEC header plus anything in RTP - // header beyond the 12 bytes base header (CSRC list, extensions...) + // The RED overhead is due to a small header. + overhead += kRedForFecHeaderLength; + } + if (ulpfec_enabled()) { + // For ULPFEC, the overhead is the FEC headers plus RED for FEC header + // (see above) plus anything in RTP header beyond the 12 bytes base header + // (CSRC list, extensions...) // This reason for the header extensions to be included here is that // from an FEC viewpoint, they are part of the payload to be protected. // (The base RTP header is already protected by the FEC header.) - return ulpfec_generator_.MaxPacketOverhead() + kRedForFecHeaderLength + - (rtp_sender_->RtpHeaderLength() - kRtpHeaderSize); + overhead += ulpfec_generator_.MaxPacketOverhead() + + (rtp_sender_->RtpHeaderLength() - kRtpHeaderSize); } - if (ulpfec_enabled()) - overhead += ulpfec_generator_.MaxPacketOverhead(); return overhead; } void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params, const FecProtectionParams& key_params) { rtc::CritScope cs(&crit_); - if (ulpfec_enabled()) { - delta_fec_params_ = delta_params; - key_fec_params_ = key_params; - } + delta_fec_params_ = delta_params; + key_fec_params_ = key_params; } bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, @@ -290,11 +327,18 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, bool first_frame = first_frame_sent_(); { rtc::CritScope cs(&crit_); + + // Media packet storage. + storage = packetizer->GetStorageType(retransmission_settings_); + + // FEC settings. const FecProtectionParams& fec_params = frame_type == kVideoFrameKey ? key_fec_params_ : delta_fec_params_; - ulpfec_generator_.SetFecParameters(fec_params); - storage = packetizer->GetStorageType(retransmission_settings_); + if (flexfec_enabled()) + flexfec_sender_->SetFecParameters(fec_params); red_enabled = this->red_enabled(); + if (ulpfec_enabled()) + ulpfec_generator_.SetFecParameters(fec_params); } // TODO(changbin): we currently don't support to configure the codec to @@ -321,9 +365,15 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, if (!rtp_sender_->AssignSequenceNumber(packet.get())) return false; - if (red_enabled) { - SendVideoPacketAsRed(std::move(packet), storage, - packetizer->GetProtectionType() == kProtectedPacket); + const bool protect_packet = + (packetizer->GetProtectionType() == kProtectedPacket); + if (flexfec_enabled()) { + // TODO(brandtr): Remove the FlexFEC code path when FlexfecSender + // is wired up to PacedSender instead. + SendVideoPacketWithFlexfec(std::move(packet), storage, protect_packet); + } else if (red_enabled) { + SendVideoPacketAsRedMaybeWithUlpfec(std::move(packet), storage, + protect_packet); } else { SendVideoPacket(std::move(packet), storage); } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h index 56167eb6eb..cc5b551402 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h @@ -13,6 +13,7 @@ #include #include +#include #include "webrtc/base/criticalsection.h" #include "webrtc/base/onetimeevent.h" @@ -20,8 +21,8 @@ #include "webrtc/base/sequenced_task_checker.h" #include "webrtc/base/thread_annotations.h" #include "webrtc/common_types.h" +#include "webrtc/modules/rtp_rtcp/include/flexfec_sender.h" #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "webrtc/modules/rtp_rtcp/source/forward_error_correction.h" #include "webrtc/modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "webrtc/modules/rtp_rtcp/source/rtp_sender.h" #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" @@ -76,9 +77,16 @@ class RTPSenderVideo { void SendVideoPacket(std::unique_ptr packet, StorageType storage); - void SendVideoPacketAsRed(std::unique_ptr media_packet, - StorageType media_packet_storage, - bool protect); + void SendVideoPacketAsRedMaybeWithUlpfec( + std::unique_ptr media_packet, + StorageType media_packet_storage, + bool protect_media_packet); + + // TODO(brandtr): Remove the FlexFEC functions when FlexfecSender has been + // moved to PacedSender. + void SendVideoPacketWithFlexfec(std::unique_ptr media_packet, + StorageType media_packet_storage, + bool protect_media_packet); bool red_enabled() const EXCLUSIVE_LOCKS_REQUIRED(crit_) { return red_payload_type_ >= 0; @@ -88,6 +96,8 @@ class RTPSenderVideo { return ulpfec_payload_type_ >= 0; } + bool flexfec_enabled() const { return flexfec_sender_ != nullptr; } + RTPSender* const rtp_sender_; Clock* const clock_; @@ -102,9 +112,14 @@ class RTPSenderVideo { // RED/ULPFEC. int red_payload_type_ GUARDED_BY(crit_); int ulpfec_payload_type_ GUARDED_BY(crit_); + UlpfecGenerator ulpfec_generator_ GUARDED_BY(crit_); + + // FlexFEC. + FlexfecSender* const flexfec_sender_; + + // FEC parameters, applicable to either ULPFEC or FlexFEC. FecProtectionParams delta_fec_params_ GUARDED_BY(crit_); FecProtectionParams key_fec_params_ GUARDED_BY(crit_); - UlpfecGenerator ulpfec_generator_ GUARDED_BY(crit_); rtc::CriticalSection stats_crit_; // Bitrate used for FEC payload, RED headers, RTP headers for FEC packets