diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 5f4f971087..8ec534e0c9 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -36,13 +36,9 @@ namespace webrtc { namespace webrtc_internal_rtp_video_sender { -RtpStreamSender::RtpStreamSender( - std::unique_ptr rtp_rtcp, - std::unique_ptr sender_video, - std::unique_ptr fec_generator) - : rtp_rtcp(std::move(rtp_rtcp)), - sender_video(std::move(sender_video)), - fec_generator(std::move(fec_generator)) {} +RtpStreamSender::RtpStreamSender(std::unique_ptr rtp_rtcp, + std::unique_ptr sender_video) + : rtp_rtcp(std::move(rtp_rtcp)), sender_video(std::move(sender_video)) {} RtpStreamSender::~RtpStreamSender() = default; @@ -117,67 +113,6 @@ bool ShouldDisableRedAndUlpfec(bool flexfec_enabled, return should_disable_red_and_ulpfec; } -// TODO(brandtr): Update this function when we support multistream protection. -std::unique_ptr MaybeCreateFecGenerator( - Clock* clock, - const RtpConfig& rtp, - const std::map& suspended_ssrcs, - int simulcast_index) { - // If flexfec is configured that takes priority. - if (rtp.flexfec.payload_type >= 0) { - RTC_DCHECK_GE(rtp.flexfec.payload_type, 0); - RTC_DCHECK_LE(rtp.flexfec.payload_type, 127); - if (rtp.flexfec.ssrc == 0) { - RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " - "Therefore disabling FlexFEC."; - return nullptr; - } - if (rtp.flexfec.protected_media_ssrcs.empty()) { - RTC_LOG(LS_WARNING) - << "FlexFEC is enabled, but no protected media SSRC given. " - "Therefore disabling FlexFEC."; - return nullptr; - } - - if (rtp.flexfec.protected_media_ssrcs.size() > 1) { - RTC_LOG(LS_WARNING) - << "The supplied FlexfecConfig contained multiple protected " - "media streams, but our implementation currently only " - "supports protecting a single media stream. " - "To avoid confusion, disabling FlexFEC completely."; - return nullptr; - } - - if (absl::c_find(rtp.flexfec.protected_media_ssrcs, - rtp.ssrcs[simulcast_index]) == - rtp.flexfec.protected_media_ssrcs.end()) { - // Media SSRC not among flexfec protected SSRCs. - return nullptr; - } - - const RtpState* rtp_state = nullptr; - auto it = suspended_ssrcs.find(rtp.flexfec.ssrc); - if (it != suspended_ssrcs.end()) { - rtp_state = &it->second; - } - - RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size()); - return std::make_unique( - rtp.flexfec.payload_type, rtp.flexfec.ssrc, - rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions, - RTPSender::FecExtensionSizes(), rtp_state, clock); - } else if (rtp.ulpfec.red_payload_type >= 0 && - rtp.ulpfec.ulpfec_payload_type >= 0 && - !ShouldDisableRedAndUlpfec(/*flexfec_enabled=*/false, rtp)) { - // Flexfec not configured, but ulpfec is and is not disabled. - return std::make_unique( - rtp.ulpfec.red_payload_type, rtp.ulpfec.ulpfec_payload_type, clock); - } - - // Not a single FEC is given. - return nullptr; -} - std::vector CreateRtpStreamSenders( Clock* clock, const RtpConfig& rtp_config, @@ -186,7 +121,7 @@ std::vector CreateRtpStreamSenders( Transport* send_transport, RtcpBandwidthObserver* bandwidth_callback, RtpTransportControllerSendInterface* transport, - const std::map& suspended_ssrcs, + FlexfecSender* flexfec_sender, RtcEventLog* event_log, RateLimiter* retransmission_rate_limiter, OverheadObserver* overhead_observer, @@ -225,17 +160,18 @@ std::vector CreateRtpStreamSenders( configuration.rtcp_report_interval_ms = rtcp_report_interval_ms; std::vector rtp_streams; - + const std::vector& flexfec_protected_ssrcs = + rtp_config.flexfec.protected_media_ssrcs; RTC_DCHECK(rtp_config.rtx.ssrcs.empty() || rtp_config.rtx.ssrcs.size() == rtp_config.rtx.ssrcs.size()); for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) { - RTPSenderVideo::Config video_config; configuration.local_media_ssrc = rtp_config.ssrcs[i]; - - std::unique_ptr fec_generator = - MaybeCreateFecGenerator(clock, rtp_config, suspended_ssrcs, i); - configuration.fec_generator = fec_generator.get(); - video_config.fec_generator = fec_generator.get(); + bool enable_flexfec = flexfec_sender != nullptr && + std::find(flexfec_protected_ssrcs.begin(), + flexfec_protected_ssrcs.end(), + configuration.local_media_ssrc) != + flexfec_protected_ssrcs.end(); + configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr; if (rtp_config.rtx.ssrcs.size() > i) { configuration.rtx_send_ssrc = rtp_config.rtx.ssrcs[i]; @@ -251,31 +187,75 @@ std::vector CreateRtpStreamSenders( rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize); FieldTrialBasedConfig field_trial_config; + RTPSenderVideo::Config video_config; video_config.clock = configuration.clock; video_config.rtp_sender = rtp_rtcp->RtpSender(); + video_config.flexfec_sender = configuration.flexfec_sender; video_config.frame_encryptor = frame_encryptor; video_config.require_frame_encryption = crypto_options.sframe.require_frame_encryption; video_config.enable_retransmit_all_layers = false; video_config.field_trials = &field_trial_config; - - const bool using_flexfec = - fec_generator && - fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec; const bool should_disable_red_and_ulpfec = - ShouldDisableRedAndUlpfec(using_flexfec, rtp_config); - if (!should_disable_red_and_ulpfec && - rtp_config.ulpfec.red_payload_type != -1) { + ShouldDisableRedAndUlpfec(enable_flexfec, rtp_config); + if (rtp_config.ulpfec.red_payload_type != -1 && + !should_disable_red_and_ulpfec) { video_config.red_payload_type = rtp_config.ulpfec.red_payload_type; } - + if (rtp_config.ulpfec.ulpfec_payload_type != -1 && + !should_disable_red_and_ulpfec) { + video_config.ulpfec_payload_type = rtp_config.ulpfec.ulpfec_payload_type; + } auto sender_video = std::make_unique(video_config); - rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video), - std::move(fec_generator)); + rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video)); } return rtp_streams; } +// TODO(brandtr): Update this function when we support multistream protection. +std::unique_ptr MaybeCreateFlexfecSender( + Clock* clock, + const RtpConfig& rtp, + const std::map& suspended_ssrcs) { + if (rtp.flexfec.payload_type < 0) { + return nullptr; + } + RTC_DCHECK_GE(rtp.flexfec.payload_type, 0); + RTC_DCHECK_LE(rtp.flexfec.payload_type, 127); + if (rtp.flexfec.ssrc == 0) { + RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " + "Therefore disabling FlexFEC."; + return nullptr; + } + if (rtp.flexfec.protected_media_ssrcs.empty()) { + RTC_LOG(LS_WARNING) + << "FlexFEC is enabled, but no protected media SSRC given. " + "Therefore disabling FlexFEC."; + return nullptr; + } + + if (rtp.flexfec.protected_media_ssrcs.size() > 1) { + RTC_LOG(LS_WARNING) + << "The supplied FlexfecConfig contained multiple protected " + "media streams, but our implementation currently only " + "supports protecting a single media stream. " + "To avoid confusion, disabling FlexFEC completely."; + return nullptr; + } + + const RtpState* rtp_state = nullptr; + auto it = suspended_ssrcs.find(rtp.flexfec.ssrc); + if (it != suspended_ssrcs.end()) { + rtp_state = &it->second; + } + + RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size()); + return std::make_unique( + rtp.flexfec.payload_type, rtp.flexfec.ssrc, + rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions, + RTPSender::FecExtensionSizes(), rtp_state, clock); +} + DataRate CalculateOverheadRate(DataRate data_rate, DataSize packet_size, DataSize overhead_per_packet) { @@ -322,6 +302,8 @@ RtpVideoSender::RtpVideoSender( active_(false), module_process_thread_(nullptr), suspended_ssrcs_(std::move(suspended_ssrcs)), + flexfec_sender_( + MaybeCreateFlexfecSender(clock, rtp_config, suspended_ssrcs_)), fec_controller_(std::move(fec_controller)), fec_allowed_(true), rtp_streams_(CreateRtpStreamSenders(clock, @@ -331,7 +313,7 @@ RtpVideoSender::RtpVideoSender( send_transport, transport->GetBandwidthObserver(), transport, - suspended_ssrcs_, + flexfec_sender_.get(), event_log, retransmission_limiter, this, @@ -393,7 +375,6 @@ RtpVideoSender::RtpVideoSender( } } - bool fec_enabled = false; for (const RtpStreamSender& stream : rtp_streams_) { // Simulcast has one module for each layer. Set the CNAME on all modules. stream.rtp_rtcp->SetCNAME(rtp_config_.c_name.c_str()); @@ -403,13 +384,10 @@ RtpVideoSender::RtpVideoSender( stream.rtp_rtcp->SetMaxRtpPacketSize(rtp_config_.max_packet_size); stream.rtp_rtcp->RegisterSendPayloadFrequency(rtp_config_.payload_type, kVideoPayloadTypeFrequency); - if (stream.fec_generator != nullptr) { - fec_enabled = true; - } } // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, // so enable that logic if either of those FEC schemes are enabled. - fec_controller_->SetProtectionMethod(fec_enabled, NackEnabled()); + fec_controller_->SetProtectionMethod(FecEnabled(), NackEnabled()); fec_controller_->SetProtectionCallback(this); // Signal congestion controller this object is ready for OnPacket* callbacks. @@ -577,6 +555,14 @@ void RtpVideoSender::OnBitrateAllocationUpdated( } } +bool RtpVideoSender::FecEnabled() const { + const bool flexfec_enabled = (flexfec_sender_ != nullptr); + const bool ulpfec_enabled = + !webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment") && + (rtp_config_.ulpfec.ulpfec_payload_type >= 0); + return flexfec_enabled || ulpfec_enabled; +} + bool RtpVideoSender::NackEnabled() const { const bool nack_enabled = rtp_config_.nack.rtp_history_ms > 0; return nack_enabled; @@ -671,14 +657,6 @@ std::map RtpVideoSender::GetRtpStates() const { uint32_t ssrc = rtp_config_.ssrcs[i]; RTC_DCHECK_EQ(ssrc, rtp_streams_[i].rtp_rtcp->SSRC()); rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtpState(); - - VideoFecGenerator* fec_generator = rtp_streams_[i].fec_generator.get(); - if (fec_generator && - fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec) { - auto* flexfec_sender = static_cast(fec_generator); - uint32_t ssrc = rtp_config_.flexfec.ssrc; - rtp_states[ssrc] = flexfec_sender->GetRtpState(); - } } for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) { @@ -686,6 +664,11 @@ std::map RtpVideoSender::GetRtpStates() const { rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtxState(); } + if (flexfec_sender_) { + uint32_t ssrc = rtp_config_.flexfec.ssrc; + rtp_states[ssrc] = flexfec_sender_->GetRtpState(); + } + return rtp_states; } diff --git a/call/rtp_video_sender.h b/call/rtp_video_sender.h index ed89028b1e..620c975810 100644 --- a/call/rtp_video_sender.h +++ b/call/rtp_video_sender.h @@ -51,8 +51,7 @@ namespace webrtc_internal_rtp_video_sender { // RtpVideoSender. struct RtpStreamSender { RtpStreamSender(std::unique_ptr rtp_rtcp, - std::unique_ptr sender_video, - std::unique_ptr fec_generator); + std::unique_ptr sender_video); ~RtpStreamSender(); RtpStreamSender(RtpStreamSender&&) = default; @@ -61,7 +60,6 @@ struct RtpStreamSender { // Note: Needs pointer stability. std::unique_ptr rtp_rtcp; std::unique_ptr sender_video; - std::unique_ptr fec_generator; }; } // namespace webrtc_internal_rtp_video_sender @@ -156,6 +154,7 @@ class RtpVideoSender : public RtpVideoSenderInterface, void ConfigureProtection(); void ConfigureSsrcs(); void ConfigureRids(); + bool FecEnabled() const; bool NackEnabled() const; uint32_t GetPacketizationOverheadRate() const; @@ -173,6 +172,8 @@ class RtpVideoSender : public RtpVideoSenderInterface, rtc::ThreadChecker module_process_thread_checker_; std::map suspended_ssrcs_; + std::unique_ptr flexfec_sender_; + const std::unique_ptr fec_controller_; bool fec_allowed_ RTC_GUARDED_BY(crit_); diff --git a/modules/include/module_fec_types.h b/modules/include/module_fec_types.h index f9b35cc288..25d6bc5714 100644 --- a/modules/include/module_fec_types.h +++ b/modules/include/module_fec_types.h @@ -24,9 +24,9 @@ enum FecMaskType { // Struct containing forward error correction settings. struct FecProtectionParams { - int fec_rate = 0; - int max_fec_frames = 0; - FecMaskType fec_mask_type = FecMaskType::kFecMaskRandom; + int fec_rate; + int max_fec_frames; + FecMaskType fec_mask_type; }; } // namespace webrtc diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 5981c30e21..067c83cab3 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -210,7 +210,6 @@ rtc_library("rtp_rtcp") { "source/ulpfec_header_reader_writer.h", "source/ulpfec_receiver_impl.cc", "source/ulpfec_receiver_impl.h", - "source/video_fec_generator.h", "source/video_rtp_depacketizer.h", "source/video_rtp_depacketizer_av1.cc", "source/video_rtp_depacketizer_av1.h", diff --git a/modules/rtp_rtcp/include/flexfec_sender.h b/modules/rtp_rtcp/include/flexfec_sender.h index 4cc8f99ce6..94f3502d31 100644 --- a/modules/rtp_rtcp/include/flexfec_sender.h +++ b/modules/rtp_rtcp/include/flexfec_sender.h @@ -21,9 +21,7 @@ #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_header_extension_size.h" #include "modules/rtp_rtcp/source/ulpfec_generator.h" -#include "modules/rtp_rtcp/source/video_fec_generator.h" #include "rtc_base/random.h" -#include "rtc_base/rate_statistics.h" namespace webrtc { @@ -33,7 +31,7 @@ class RtpPacketToSend; // Note that this class is not thread safe, and thus requires external // synchronization. Currently, this is done using the lock in PayloadRouter. -class FlexfecSender : public VideoFecGenerator { +class FlexfecSender { public: FlexfecSender(int payload_type, uint32_t ssrc, @@ -45,28 +43,26 @@ class FlexfecSender : public VideoFecGenerator { Clock* clock); ~FlexfecSender(); - FecType GetFecType() const override { - return VideoFecGenerator::FecType::kFlexFec; - } - absl::optional FecSsrc() override { return ssrc_; } + uint32_t ssrc() const { return ssrc_; } // Sets the FEC rate, max frames sent before FEC packets are sent, // and what type of generator matrices are used. - void SetProtectionParameters(const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) override; + void SetFecParameters(const FecProtectionParams& params); // Adds a media packet to the internal buffer. When enough media packets // have been added, the FEC packets are generated and stored internally. // These FEC packets are then obtained by calling GetFecPackets(). - void AddPacketAndGenerateFec(const RtpPacketToSend& packet) override; + // Returns true if the media packet was successfully added. + bool AddRtpPacketAndGenerateFec(const RtpPacketToSend& packet); + + // Returns true if there are generated FEC packets available. + bool FecAvailable() const; // Returns generated FlexFEC packets. - std::vector> GetFecPackets() override; + std::vector> GetFecPackets(); // Returns the overhead, per packet, for FlexFEC. - size_t MaxPacketOverhead() const override; - - DataRate CurrentFecRate() const override; + size_t MaxPacketOverhead() const; // Only called on the VideoSendStream queue, after operation has shut down. RtpState GetRtpState(); @@ -91,9 +87,6 @@ class FlexfecSender : public VideoFecGenerator { UlpfecGenerator ulpfec_generator_; const RtpHeaderExtensionMap rtp_header_extension_map_; const size_t header_extensions_size_; - - rtc::CriticalSection crit_; - RateStatistics fec_bitrate_ RTC_GUARDED_BY(crit_); }; } // namespace webrtc diff --git a/modules/rtp_rtcp/include/rtp_rtcp.h b/modules/rtp_rtcp/include/rtp_rtcp.h index e771e2a9a8..b256f381d8 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp.h +++ b/modules/rtp_rtcp/include/rtp_rtcp.h @@ -22,13 +22,13 @@ #include "api/transport/webrtc_key_value_config.h" #include "api/video/video_bitrate_allocation.h" #include "modules/include/module.h" +#include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/include/report_block_data.h" #include "modules/rtp_rtcp/include/rtp_packet_sender.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" #include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" -#include "modules/rtp_rtcp/source/video_fec_generator.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/deprecation.h" @@ -92,9 +92,9 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { // Spread any bursts of packets into smaller bursts to minimize packet loss. RtpPacketSender* paced_sender = nullptr; - // Generates FEC packets. - // TODO(sprang): Wire up to RtpSenderEgress. - VideoFecGenerator* fec_generator = nullptr; + // Generate FlexFEC packets. + // TODO(brandtr): Remove when FlexfecSender is wired up to PacedSender. + FlexfecSender* flexfec_sender = nullptr; BitrateStatisticsObserver* send_bitrate_observer = nullptr; SendSideDelayObserver* send_side_delay_observer = nullptr; diff --git a/modules/rtp_rtcp/source/flexfec_sender.cc b/modules/rtp_rtcp/source/flexfec_sender.cc index 4ff0893ee7..de0d4129ce 100644 --- a/modules/rtp_rtcp/source/flexfec_sender.cc +++ b/modules/rtp_rtcp/source/flexfec_sender.cc @@ -91,13 +91,11 @@ FlexfecSender::FlexfecSender( seq_num_(rtp_state ? rtp_state->sequence_number : random_.Rand(1, kMaxInitRtpSeqNumber)), ulpfec_generator_( - ForwardErrorCorrection::CreateFlexfec(ssrc, protected_media_ssrc), - clock_), + ForwardErrorCorrection::CreateFlexfec(ssrc, protected_media_ssrc)), rtp_header_extension_map_( RegisterSupportedExtensions(rtp_header_extensions)), header_extensions_size_( - RtpHeaderExtensionSize(extension_sizes, rtp_header_extension_map_)), - fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) { + RtpHeaderExtensionSize(extension_sizes, rtp_header_extension_map_)) { // This object should not have been instantiated if FlexFEC is disabled. RTC_DCHECK_GE(payload_type, 0); RTC_DCHECK_LE(payload_type, 127); @@ -107,30 +105,30 @@ FlexfecSender::~FlexfecSender() = default; // We are reusing the implementation from UlpfecGenerator for SetFecParameters, // AddRtpPacketAndGenerateFec, and FecAvailable. -void FlexfecSender::SetProtectionParameters( - const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) { - ulpfec_generator_.SetProtectionParameters(delta_params, key_params); +void FlexfecSender::SetFecParameters(const FecProtectionParams& params) { + ulpfec_generator_.SetFecParameters(params); } -void FlexfecSender::AddPacketAndGenerateFec(const RtpPacketToSend& packet) { +bool FlexfecSender::AddRtpPacketAndGenerateFec(const RtpPacketToSend& packet) { // TODO(brandtr): Generalize this SSRC check when we support multistream // protection. RTC_DCHECK_EQ(packet.Ssrc(), protected_media_ssrc_); - ulpfec_generator_.AddPacketAndGenerateFec(packet); + return ulpfec_generator_.AddRtpPacketAndGenerateFec( + packet.Buffer(), packet.headers_size()) == 0; +} + +bool FlexfecSender::FecAvailable() const { + return ulpfec_generator_.FecAvailable(); } std::vector> FlexfecSender::GetFecPackets() { - RTC_CHECK_RUNS_SERIALIZED(&ulpfec_generator_.race_checker_); std::vector> fec_packets_to_send; fec_packets_to_send.reserve(ulpfec_generator_.generated_fec_packets_.size()); - size_t total_fec_data_bytes = 0; for (const auto* fec_packet : ulpfec_generator_.generated_fec_packets_) { std::unique_ptr fec_packet_to_send( new RtpPacketToSend(&rtp_header_extension_map_)); fec_packet_to_send->set_packet_type( RtpPacketMediaType::kForwardErrorCorrection); - fec_packet_to_send->set_allow_retransmission(false); // RTP header. fec_packet_to_send->SetMarker(false); @@ -159,13 +157,9 @@ std::vector> FlexfecSender::GetFecPackets() { fec_packet_to_send->AllocatePayload(fec_packet->data.size()); memcpy(payload, fec_packet->data.cdata(), fec_packet->data.size()); - total_fec_data_bytes += fec_packet_to_send->size(); fec_packets_to_send.push_back(std::move(fec_packet_to_send)); } - - if (!fec_packets_to_send.empty()) { - ulpfec_generator_.ResetState(); - } + ulpfec_generator_.ResetState(); int64_t now_ms = clock_->TimeInMilliseconds(); if (!fec_packets_to_send.empty() && @@ -176,9 +170,6 @@ std::vector> FlexfecSender::GetFecPackets() { last_generated_packet_ms_ = now_ms; } - rtc::CritScope cs(&crit_); - fec_bitrate_.Update(total_fec_data_bytes, now_ms); - return fec_packets_to_send; } @@ -187,12 +178,6 @@ size_t FlexfecSender::MaxPacketOverhead() const { return header_extensions_size_ + kFlexfecMaxHeaderSize; } -DataRate FlexfecSender::CurrentFecRate() const { - rtc::CritScope cs(&crit_); - return DataRate::BitsPerSec( - fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0)); -} - RtpState FlexfecSender::GetRtpState() { RtpState rtp_state; rtp_state.sequence_number = seq_num_; diff --git a/modules/rtp_rtcp/source/flexfec_sender_unittest.cc b/modules/rtp_rtcp/source/flexfec_sender_unittest.cc index e4501c2c1d..10ec2e7495 100644 --- a/modules/rtp_rtcp/source/flexfec_sender_unittest.cc +++ b/modules/rtp_rtcp/source/flexfec_sender_unittest.cc @@ -55,7 +55,7 @@ std::unique_ptr GenerateSingleFlexfecPacket( params.fec_mask_type = kFecMaskRandom; constexpr size_t kNumPackets = 4; - sender->SetProtectionParameters(params, params); + sender->SetFecParameters(params); AugmentedPacketGenerator packet_generator(kMediaSsrc); packet_generator.NewFrame(kNumPackets); for (size_t i = 0; i < kNumPackets; ++i) { @@ -63,12 +63,13 @@ std::unique_ptr GenerateSingleFlexfecPacket( packet_generator.NextPacket(i, kPayloadLength); RtpPacketToSend rtp_packet(nullptr); // No header extensions. rtp_packet.Parse(packet->data); - sender->AddPacketAndGenerateFec(rtp_packet); + EXPECT_TRUE(sender->AddRtpPacketAndGenerateFec(rtp_packet)); } + EXPECT_TRUE(sender->FecAvailable()); std::vector> fec_packets = sender->GetFecPackets(); + EXPECT_FALSE(sender->FecAvailable()); EXPECT_EQ(1U, fec_packets.size()); - EXPECT_TRUE(sender->GetFecPackets().empty()); return std::move(fec_packets.front()); } @@ -81,7 +82,7 @@ TEST(FlexfecSenderTest, Ssrc) { kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes, nullptr /* rtp_state */, &clock); - EXPECT_EQ(kFlexfecSsrc, sender.FecSsrc()); + EXPECT_EQ(kFlexfecSsrc, sender.ssrc()); } TEST(FlexfecSenderTest, NoFecAvailableBeforeMediaAdded) { @@ -90,7 +91,9 @@ TEST(FlexfecSenderTest, NoFecAvailableBeforeMediaAdded) { kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes, nullptr /* rtp_state */, &clock); - EXPECT_TRUE(sender.GetFecPackets().empty()); + EXPECT_FALSE(sender.FecAvailable()); + auto fec_packets = sender.GetFecPackets(); + EXPECT_EQ(0U, fec_packets.size()); } TEST(FlexfecSenderTest, ProtectOneFrameWithOneFecPacket) { @@ -121,7 +124,7 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithOneFecPacket) { FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc, kNoMid, kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes, nullptr /* rtp_state */, &clock); - sender.SetProtectionParameters(params, params); + sender.SetFecParameters(params); AugmentedPacketGenerator packet_generator(kMediaSsrc); for (size_t i = 0; i < kNumFrames; ++i) { @@ -131,13 +134,14 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithOneFecPacket) { packet_generator.NextPacket(i, kPayloadLength); RtpPacketToSend rtp_packet(nullptr); rtp_packet.Parse(packet->data); - sender.AddPacketAndGenerateFec(rtp_packet); + EXPECT_TRUE(sender.AddRtpPacketAndGenerateFec(rtp_packet)); } } + EXPECT_TRUE(sender.FecAvailable()); std::vector> fec_packets = sender.GetFecPackets(); + EXPECT_FALSE(sender.FecAvailable()); ASSERT_EQ(1U, fec_packets.size()); - EXPECT_TRUE(sender.GetFecPackets().empty()); RtpPacketToSend* fec_packet = fec_packets.front().get(); EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size()); @@ -160,7 +164,7 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithTwoFecPackets) { FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc, kNoMid, kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes, nullptr /* rtp_state */, &clock); - sender.SetProtectionParameters(params, params); + sender.SetFecParameters(params); AugmentedPacketGenerator packet_generator(kMediaSsrc); for (size_t i = 0; i < kNumFrames; ++i) { @@ -170,12 +174,13 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithTwoFecPackets) { packet_generator.NextPacket(i, kPayloadLength); RtpPacketToSend rtp_packet(nullptr); rtp_packet.Parse(packet->data); - sender.AddPacketAndGenerateFec(rtp_packet); + EXPECT_TRUE(sender.AddRtpPacketAndGenerateFec(rtp_packet)); } + EXPECT_TRUE(sender.FecAvailable()); std::vector> fec_packets = sender.GetFecPackets(); + EXPECT_FALSE(sender.FecAvailable()); ASSERT_EQ(1U, fec_packets.size()); - EXPECT_TRUE(sender.GetFecPackets().empty()); RtpPacketToSend* fec_packet = fec_packets.front().get(); EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size()); diff --git a/modules/rtp_rtcp/source/rtcp_receiver.cc b/modules/rtp_rtcp/source/rtcp_receiver.cc index 26465ada40..6b64473eea 100644 --- a/modules/rtp_rtcp/source/rtcp_receiver.cc +++ b/modules/rtp_rtcp/source/rtcp_receiver.cc @@ -72,11 +72,8 @@ std::set GetRegisteredSsrcs(const RtpRtcp::Configuration& config) { if (config.rtx_send_ssrc) { ssrcs.insert(*config.rtx_send_ssrc); } - if (config.fec_generator) { - absl::optional flexfec_ssrc = config.fec_generator->FecSsrc(); - if (flexfec_ssrc) { - ssrcs.insert(*flexfec_ssrc); - } + if (config.flexfec_sender) { + ssrcs.insert(config.flexfec_sender->ssrc()); } return ssrcs; } diff --git a/modules/rtp_rtcp/source/rtp_packet_to_send.h b/modules/rtp_rtcp/source/rtp_packet_to_send.h index 8997bce0d2..57493e3802 100644 --- a/modules/rtp_rtcp/source/rtp_packet_to_send.h +++ b/modules/rtp_rtcp/source/rtp_packet_to_send.h @@ -98,15 +98,11 @@ class RtpPacketToSend : public RtpPacket { VideoTimingExtension::kNetwork2TimestampDeltaOffset); } - // Indicates if packet is the first packet of a video frame. void set_first_packet_of_frame(bool is_first_packet) { is_first_packet_of_frame_ = is_first_packet; } - bool is_first_packet_of_frame() const { return is_first_packet_of_frame_; } - // Indicates if packet contains payload for a video key-frame. - void set_is_key_frame(bool is_key_frame) { is_key_frame_ = is_key_frame; } - bool is_key_frame() const { return is_key_frame_; } + bool is_first_packet_of_frame() const { return is_first_packet_of_frame_; } private: int64_t capture_time_ms_ = 0; @@ -115,7 +111,6 @@ class RtpPacketToSend : public RtpPacket { absl::optional retransmitted_sequence_number_; std::vector application_data_; bool is_first_packet_of_frame_ = false; - bool is_key_frame_ = false; }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc index c48a662fc5..3277c67314 100644 --- a/modules/rtp_rtcp/source/rtp_sender.cc +++ b/modules/rtp_rtcp/source/rtp_sender.cc @@ -102,8 +102,9 @@ RTPSender::RTPSender(const RtpRtcp::Configuration& config, audio_configured_(config.audio), ssrc_(config.local_media_ssrc), rtx_ssrc_(config.rtx_send_ssrc), - flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc() - : absl::nullopt), + flexfec_ssrc_(config.flexfec_sender + ? absl::make_optional(config.flexfec_sender->ssrc()) + : absl::nullopt), packet_history_(packet_history), paced_sender_(packet_sender), sending_media_(true), // Default to sending media. diff --git a/modules/rtp_rtcp/source/rtp_sender_egress.cc b/modules/rtp_rtcp/source/rtp_sender_egress.cc index ec546c47bf..d34d7c633a 100644 --- a/modules/rtp_rtcp/source/rtp_sender_egress.cc +++ b/modules/rtp_rtcp/source/rtp_sender_egress.cc @@ -57,8 +57,9 @@ RtpSenderEgress::RtpSenderEgress(const RtpRtcp::Configuration& config, RtpPacketHistory* packet_history) : ssrc_(config.local_media_ssrc), rtx_ssrc_(config.rtx_send_ssrc), - flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc() - : absl::nullopt), + flexfec_ssrc_(config.flexfec_sender + ? absl::make_optional(config.flexfec_sender->ssrc()) + : absl::nullopt), populate_network2_timestamp_(config.populate_network2_timestamp), send_side_bwe_with_overhead_( IsEnabled("WebRTC-SendSideBwe-WithOverhead", config.field_trials)), diff --git a/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_unittest.cc index 3b85166e61..c3ae539071 100644 --- a/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -272,7 +272,7 @@ class RtpSenderTest : public ::testing::TestWithParam { config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; config.rtx_send_ssrc = kRtxSsrc; - config.fec_generator = &flexfec_sender_; + config.flexfec_sender = &flexfec_sender_; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -1225,7 +1225,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) { config.outgoing_transport = &transport_; config.paced_sender = &mock_paced_sender_; config.local_media_ssrc = kSsrc; - config.fec_generator = &flexfec_sender_; + config.flexfec_sender = &flexfec_sender_; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -1239,7 +1239,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) { RTPSenderVideo::Config video_config; video_config.clock = &fake_clock_; video_config.rtp_sender = rtp_sender(); - video_config.fec_generator = &flexfec_sender; + video_config.flexfec_sender = &flexfec_sender; video_config.field_trials = &field_trials; RTPSenderVideo rtp_sender_video(video_config); @@ -1311,7 +1311,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) { config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; - config.fec_generator = &flexfec_sender; + config.flexfec_sender = &flexfec_sender; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -1323,7 +1323,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) { RTPSenderVideo::Config video_config; video_config.clock = &fake_clock_; video_config.rtp_sender = rtp_sender(); - video_config.fec_generator = &flexfec_sender; + video_config.flexfec_sender = &flexfec_sender; video_config.field_trials = &field_trials; RTPSenderVideo rtp_sender_video(video_config); @@ -1583,7 +1583,7 @@ TEST_P(RtpSenderTest, FecOverheadRate) { config.outgoing_transport = &transport_; config.paced_sender = &mock_paced_sender_; config.local_media_ssrc = kSsrc; - config.fec_generator = &flexfec_sender; + config.flexfec_sender = &flexfec_sender; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -1595,7 +1595,7 @@ TEST_P(RtpSenderTest, FecOverheadRate) { RTPSenderVideo::Config video_config; video_config.clock = &fake_clock_; video_config.rtp_sender = rtp_sender(); - video_config.fec_generator = &flexfec_sender; + video_config.flexfec_sender = &flexfec_sender; video_config.field_trials = &field_trials; RTPSenderVideo rtp_sender_video(video_config); // Parameters selected to generate a single FEC packet per media packet. @@ -1777,14 +1777,12 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacksUlpfec) { const uint8_t kPayloadType = 127; const VideoCodecType kCodecType = VideoCodecType::kVideoCodecGeneric; FieldTrialBasedConfig field_trials; - UlpfecGenerator ulpfec_generator(kRedPayloadType, kUlpfecPayloadType, - &fake_clock_); RTPSenderVideo::Config video_config; video_config.clock = &fake_clock_; video_config.rtp_sender = rtp_sender(); video_config.field_trials = &field_trials; video_config.red_payload_type = kRedPayloadType; - video_config.fec_generator = &ulpfec_generator; + video_config.ulpfec_payload_type = kUlpfecPayloadType; RTPSenderVideo rtp_sender_video(video_config); uint8_t payload[] = {47, 11, 32, 93, 89}; rtp_sender_context_->packet_history_.SetStorePacketsStatus( @@ -2120,7 +2118,7 @@ TEST_P(RtpSenderTest, SendPacketUpdatesStats) { config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; config.rtx_send_ssrc = kRtxSsrc; - config.fec_generator = &flexfec_sender_; + config.flexfec_sender = &flexfec_sender_; config.send_side_delay_observer = &send_side_delay_observer; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index d892a78d08..efc674c695 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -259,7 +259,11 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) current_playout_delay_{-1, -1}, playout_delay_pending_(false), red_payload_type_(config.red_payload_type), - fec_generator_(config.fec_generator), + ulpfec_payload_type_(config.ulpfec_payload_type), + flexfec_sender_(config.flexfec_sender), + delta_fec_params_{0, 1, kFecMaskRandom}, + key_fec_params_{0, 1, kFecMaskRandom}, + fec_bitrate_(1000, RateStatistics::kBpsScale), video_bitrate_(1000, RateStatistics::kBpsScale), packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale), frame_encryptor_(config.frame_encryptor), @@ -275,6 +279,83 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) RTPSenderVideo::~RTPSenderVideo() {} +void RTPSenderVideo::AppendAsRedMaybeWithUlpfec( + std::unique_ptr media_packet, + bool protect_media_packet, + std::vector>* packets) { + std::unique_ptr red_packet( + new RtpPacketToSend(*media_packet)); + BuildRedPayload(*media_packet, red_packet.get()); + red_packet->SetPayloadType(*red_payload_type_); + + std::vector> fec_packets; + if (ulpfec_enabled()) { + if (protect_media_packet) { + if (exclude_transport_sequence_number_from_fec_experiment_) { + // See comments at the top of the file why experiment + // "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in + // conjunction with datagram transport. + // TODO(sukhanov): We may also need to implement it for flexfec_sender + // if we decide to keep this approach in the future. + uint16_t transport_senquence_number; + if (media_packet->GetExtension( + &transport_senquence_number)) { + if (!media_packet->RemoveExtension( + webrtc::TransportSequenceNumber::kId)) { + RTC_NOTREACHED() + << "Failed to remove transport sequence number, packet=" + << media_packet->ToString(); + } + } + } + + ulpfec_generator_.AddRtpPacketAndGenerateFec( + media_packet->Buffer(), media_packet->headers_size()); + } + uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets(); + if (num_fec_packets > 0) { + uint16_t first_fec_sequence_number = + rtp_sender_->AllocateSequenceNumber(num_fec_packets); + fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed( + *red_payload_type_, *ulpfec_payload_type_, first_fec_sequence_number); + RTC_DCHECK_EQ(num_fec_packets, fec_packets.size()); + } + } + + // Send |red_packet| instead of |packet| for allocated sequence number. + red_packet->set_packet_type(RtpPacketMediaType::kVideo); + red_packet->set_allow_retransmission(media_packet->allow_retransmission()); + packets->emplace_back(std::move(red_packet)); + + for (const auto& fec_packet : fec_packets) { + // TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid + // reparsing them. + std::unique_ptr rtp_packet( + new RtpPacketToSend(*media_packet)); + RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length())); + rtp_packet->set_capture_time_ms(media_packet->capture_time_ms()); + rtp_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection); + rtp_packet->set_allow_retransmission(false); + RTC_DCHECK_EQ(fec_packet->length(), rtp_packet->size()); + packets->emplace_back(std::move(rtp_packet)); + } +} + +void RTPSenderVideo::GenerateAndAppendFlexfec( + std::vector>* packets) { + RTC_DCHECK(flexfec_sender_); + + if (flexfec_sender_->FecAvailable()) { + std::vector> fec_packets = + flexfec_sender_->GetFecPackets(); + for (auto& fec_packet : fec_packets) { + fec_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection); + fec_packet->set_allow_retransmission(false); + packets->emplace_back(std::move(fec_packet)); + } + } +} + void RTPSenderVideo::LogAndSendToNetwork( std::vector> packets, size_t unpacketized_payload_size) { @@ -293,9 +374,16 @@ void RTPSenderVideo::LogAndSendToNetwork( rtc::CritScope cs(&stats_crit_); size_t packetized_payload_size = 0; for (const auto& packet : packets) { - if (*packet->packet_type() == RtpPacketMediaType::kVideo) { - video_bitrate_.Update(packet->size(), now_ms); - packetized_payload_size += packet->payload_size(); + switch (*packet->packet_type()) { + case RtpPacketMediaType::kVideo: + video_bitrate_.Update(packet->size(), now_ms); + packetized_payload_size += packet->payload_size(); + break; + case RtpPacketMediaType::kForwardErrorCorrection: + fec_bitrate_.Update(packet->size(), clock_->TimeInMilliseconds()); + break; + default: + continue; } } // AV1 packetizer may produce less packetized bytes than unpacketized. @@ -310,31 +398,39 @@ void RTPSenderVideo::LogAndSendToNetwork( } size_t RTPSenderVideo::FecPacketOverhead() const { - size_t overhead = fec_generator_ ? fec_generator_->MaxPacketOverhead() : 0u; + if (flexfec_enabled()) + return flexfec_sender_->MaxPacketOverhead(); + + size_t overhead = 0; if (red_enabled()) { // The RED overhead is due to a small header. overhead += kRedForFecHeaderLength; - - // TODO(bugs.webrtc.org/11340): Move this into UlpfecGenerator. - if (fec_generator_ && - fec_generator_->GetFecType() == VideoFecGenerator::FecType::kUlpFec) { - // For ULPFEC, the overhead is the FEC headers plus RED for FEC header - // (see above) plus anything in RTP header beyond the 12 bytes base header - // (CSRC list, extensions...) - // This reason for the header extensions to be included here is that - // from an FEC viewpoint, they are part of the payload to be protected. - // (The base RTP header is already protected by the FEC header.) - overhead += rtp_sender_->RtpHeaderLength() - kRtpHeaderSize; - } + } + if (ulpfec_enabled()) { + // For ULPFEC, the overhead is the FEC headers plus RED for FEC header + // (see above) plus anything in RTP header beyond the 12 bytes base header + // (CSRC list, extensions...) + // This reason for the header extensions to be included here is that + // from an FEC viewpoint, they are part of the payload to be protected. + // (The base RTP header is already protected by the FEC header.) + overhead += ulpfec_generator_.MaxPacketOverhead() + + (rtp_sender_->RtpHeaderLength() - kRtpHeaderSize); } return overhead; } void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params, const FecProtectionParams& key_params) { - if (fec_generator_) { - fec_generator_->SetProtectionParameters(delta_params, key_params); + rtc::CritScope cs(&crit_); + delta_fec_params_ = delta_params; + key_fec_params_ = key_params; +} + +absl::optional RTPSenderVideo::FlexfecSsrc() const { + if (flexfec_sender_) { + return flexfec_sender_->ssrc(); } + return absl::nullopt; } void RTPSenderVideo::SetVideoStructure( @@ -445,6 +541,19 @@ bool RTPSenderVideo::SendVideo( transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false; } + if (flexfec_enabled() || ulpfec_enabled()) { + rtc::CritScope cs(&crit_); + // FEC settings. + const FecProtectionParams& fec_params = + video_header.frame_type == VideoFrameType::kVideoFrameKey + ? key_fec_params_ + : delta_fec_params_; + if (flexfec_enabled()) + flexfec_sender_->SetFecParameters(fec_params); + if (ulpfec_enabled()) + ulpfec_generator_.SetFecParameters(fec_params); + } + // Maximum size of packet including rtp headers. // Extra space left in case packet will be resent using fec or rtx. int packet_capacity = rtp_sender_->MaxRtpPacketSize() - FecPacketOverhead() - @@ -636,40 +745,21 @@ bool RTPSenderVideo::SendVideo( packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds()); } - if (protect_packet && fec_generator_) { - if (red_enabled() && - exclude_transport_sequence_number_from_fec_experiment_) { - // See comments at the top of the file why experiment - // "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in - // conjunction with datagram transport. - // TODO(sukhanov): We may also need to implement it for flexfec_sender - // if we decide to keep this approach in the future. - uint16_t transport_senquence_number; - if (packet->GetExtension( - &transport_senquence_number)) { - if (!packet->RemoveExtension(webrtc::TransportSequenceNumber::kId)) { - RTC_NOTREACHED() - << "Failed to remove transport sequence number, packet=" - << packet->ToString(); - } - } - } - - fec_generator_->AddPacketAndGenerateFec(*packet); - } - if (red_enabled()) { - std::unique_ptr red_packet(new RtpPacketToSend(*packet)); - BuildRedPayload(*packet, red_packet.get()); - red_packet->SetPayloadType(*red_payload_type_); - - // Send |red_packet| instead of |packet| for allocated sequence number. - red_packet->set_packet_type(RtpPacketMediaType::kVideo); - red_packet->set_allow_retransmission(packet->allow_retransmission()); - rtp_packets.emplace_back(std::move(red_packet)); + AppendAsRedMaybeWithUlpfec(std::move(packet), protect_packet, + &rtp_packets); } else { packet->set_packet_type(RtpPacketMediaType::kVideo); + const RtpPacketToSend& media_packet = *packet; rtp_packets.emplace_back(std::move(packet)); + if (flexfec_enabled()) { + // TODO(brandtr): Remove the FlexFEC code path when FlexfecSender + // is wired up to PacedSender instead. + if (protect_packet) { + flexfec_sender_->AddRtpPacketAndGenerateFec(media_packet); + } + GenerateAndAppendFlexfec(&rtp_packets); + } } if (first_frame) { @@ -684,22 +774,6 @@ bool RTPSenderVideo::SendVideo( } } - if (fec_generator_) { - // Fetch any FEC packets generated from the media frame and add them to - // the list of packets to send. - auto fec_packets = fec_generator_->GetFecPackets(); - - // TODO(bugs.webrtc.org/11340): Move sequence number assignment into - // UlpfecGenerator. - const bool generate_sequence_numbers = !fec_generator_->FecSsrc(); - for (auto& fec_packet : fec_packets) { - if (generate_sequence_numbers) { - rtp_sender_->AssignSequenceNumber(fec_packet.get()); - } - rtp_packets.emplace_back(std::move(fec_packet)); - } - } - LogAndSendToNetwork(std::move(rtp_packets), unpacketized_payload_size); TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp", @@ -713,7 +787,8 @@ uint32_t RTPSenderVideo::VideoBitrateSent() const { } uint32_t RTPSenderVideo::FecOverheadRate() const { - return fec_generator_ ? fec_generator_->CurrentFecRate().bps() : 0u; + rtc::CritScope cs(&stats_crit_); + return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0); } uint32_t RTPSenderVideo::PacketizationOverheadBps() const { diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index 1b92f52ee4..5c9657e56f 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -22,12 +22,13 @@ #include "api/video/video_codec_type.h" #include "api/video/video_frame_type.h" #include "modules/include/module_common_types.h" +#include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "modules/rtp_rtcp/source/rtp_sender.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" -#include "modules/rtp_rtcp/source/video_fec_generator.h" +#include "modules/rtp_rtcp/source/ulpfec_generator.h" #include "rtc_base/critical_section.h" #include "rtc_base/one_time_event.h" #include "rtc_base/race_checker.h" @@ -67,11 +68,11 @@ class RTPSenderVideo { Clock* clock = nullptr; RTPSender* rtp_sender = nullptr; FlexfecSender* flexfec_sender = nullptr; - VideoFecGenerator* fec_generator = nullptr; FrameEncryptorInterface* frame_encryptor = nullptr; bool require_frame_encryption = false; bool enable_retransmit_all_layers = false; absl::optional red_payload_type; + absl::optional ulpfec_payload_type; const WebRtcKeyValueConfig* field_trials = nullptr; }; @@ -98,9 +99,13 @@ class RTPSenderVideo { // FlexFEC/ULPFEC. // Set FEC rates, max frames before FEC is sent, and type of FEC masks. + // Returns false on failure. void SetFecParameters(const FecProtectionParams& delta_params, const FecProtectionParams& key_params); + // FlexFEC. + absl::optional FlexfecSsrc() const; + uint32_t VideoBitrateSent() const; uint32_t FecOverheadRate() const; @@ -129,12 +134,27 @@ class RTPSenderVideo { size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); + void AppendAsRedMaybeWithUlpfec( + std::unique_ptr media_packet, + bool protect_media_packet, + std::vector>* packets) + RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); + + // TODO(brandtr): Remove the FlexFEC functions when FlexfecSender has been + // moved to PacedSender. + void GenerateAndAppendFlexfec( + std::vector>* packets); + void LogAndSendToNetwork( std::vector> packets, size_t unpacketized_payload_size); bool red_enabled() const { return red_payload_type_.has_value(); } + bool ulpfec_enabled() const { return ulpfec_payload_type_.has_value(); } + + bool flexfec_enabled() const { return flexfec_sender_ != nullptr; } + bool UpdateConditionalRetransmit(uint8_t temporal_id, int64_t expected_retransmission_time_ms) RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_crit_); @@ -165,10 +185,22 @@ class RTPSenderVideo { // Should never be held when calling out of this class. rtc::CriticalSection crit_; + // RED/ULPFEC. const absl::optional red_payload_type_; - VideoFecGenerator* const fec_generator_; + const absl::optional ulpfec_payload_type_; + UlpfecGenerator ulpfec_generator_ RTC_GUARDED_BY(send_checker_); + + // FlexFEC. + FlexfecSender* const flexfec_sender_; + + // FEC parameters, applicable to either ULPFEC or FlexFEC. + FecProtectionParams delta_fec_params_ RTC_GUARDED_BY(crit_); + FecProtectionParams key_fec_params_ RTC_GUARDED_BY(crit_); rtc::CriticalSection stats_crit_; + // Bitrate used for FEC payload, RED headers, RTP headers for FEC packets + // and any padding overhead. + RateStatistics fec_bitrate_ RTC_GUARDED_BY(stats_crit_); // Bitrate used for video payload and RTP headers. RateStatistics video_bitrate_ RTC_GUARDED_BY(stats_crit_); RateStatistics packetization_overhead_bitrate_ RTC_GUARDED_BY(stats_crit_); diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index c04e771748..b185f0a26b 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -130,7 +130,7 @@ class TestRtpSenderVideo : public RTPSenderVideo { Config config; config.clock = clock; config.rtp_sender = rtp_sender; - config.fec_generator = flexfec_sender; + config.flexfec_sender = flexfec_sender; config.field_trials = &field_trials; return config; }()) {} diff --git a/modules/rtp_rtcp/source/ulpfec_generator.cc b/modules/rtp_rtcp/source/ulpfec_generator.cc index 265fa4d1ac..92e65df187 100644 --- a/modules/rtp_rtcp/source/ulpfec_generator.cc +++ b/modules/rtp_rtcp/source/ulpfec_generator.cc @@ -22,7 +22,6 @@ #include "modules/rtp_rtcp/source/forward_error_correction_internal.h" #include "modules/rtp_rtcp/source/rtp_utility.h" #include "rtc_base/checks.h" -#include "rtc_base/critical_section.h" namespace webrtc { @@ -63,119 +62,128 @@ constexpr uint32_t kUnknownSsrc = 0; } // namespace -UlpfecGenerator::Params::Params() = default; -UlpfecGenerator::Params::Params(FecProtectionParams delta_params, - FecProtectionParams keyframe_params) - : delta_params(delta_params), keyframe_params(keyframe_params) {} +RedPacket::RedPacket(size_t length) + : data_(new uint8_t[length]), length_(length), header_length_(0) {} -UlpfecGenerator::UlpfecGenerator(int red_payload_type, - int ulpfec_payload_type, - Clock* clock) - : red_payload_type_(red_payload_type), - ulpfec_payload_type_(ulpfec_payload_type), - clock_(clock), - fec_(ForwardErrorCorrection::CreateUlpfec(kUnknownSsrc)), - num_protected_frames_(0), - min_num_media_packets_(1), - keyframe_in_process_(false), - fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {} +RedPacket::~RedPacket() = default; -// Used by FlexFecSender, payload types are unused. -UlpfecGenerator::UlpfecGenerator(std::unique_ptr fec, - Clock* clock) - : red_payload_type_(0), - ulpfec_payload_type_(0), - clock_(clock), - fec_(std::move(fec)), +void RedPacket::CreateHeader(const uint8_t* rtp_header, + size_t header_length, + int red_payload_type, + int payload_type) { + RTC_DCHECK_LE(header_length + kRedForFecHeaderLength, length_); + memcpy(data_.get(), rtp_header, header_length); + // Replace payload type. + data_[1] &= 0x80; + data_[1] += red_payload_type; + // Add RED header + // f-bit always 0 + data_[header_length] = static_cast(payload_type); + header_length_ = header_length + kRedForFecHeaderLength; +} + +void RedPacket::SetSeqNum(int seq_num) { + RTC_DCHECK_GE(seq_num, 0); + RTC_DCHECK_LT(seq_num, 1 << 16); + + ByteWriter::WriteBigEndian(&data_[2], seq_num); +} + +void RedPacket::AssignPayload(const uint8_t* payload, size_t length) { + RTC_DCHECK_LE(header_length_ + length, length_); + memcpy(data_.get() + header_length_, payload, length); +} + +void RedPacket::ClearMarkerBit() { + data_[1] &= 0x7F; +} + +uint8_t* RedPacket::data() const { + return data_.get(); +} + +size_t RedPacket::length() const { + return length_; +} + +UlpfecGenerator::UlpfecGenerator() + : UlpfecGenerator(ForwardErrorCorrection::CreateUlpfec(kUnknownSsrc)) {} + +UlpfecGenerator::UlpfecGenerator(std::unique_ptr fec) + : fec_(std::move(fec)), + last_media_packet_rtp_header_length_(0), num_protected_frames_(0), - min_num_media_packets_(1), - keyframe_in_process_(false), - fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {} + min_num_media_packets_(1) { + memset(¶ms_, 0, sizeof(params_)); + memset(&new_params_, 0, sizeof(new_params_)); +} UlpfecGenerator::~UlpfecGenerator() = default; -void UlpfecGenerator::SetProtectionParameters( - const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) { - RTC_DCHECK_GE(delta_params.fec_rate, 0); - RTC_DCHECK_LE(delta_params.fec_rate, 255); - RTC_DCHECK_GE(key_params.fec_rate, 0); - RTC_DCHECK_LE(key_params.fec_rate, 255); +void UlpfecGenerator::SetFecParameters(const FecProtectionParams& params) { + RTC_DCHECK_GE(params.fec_rate, 0); + RTC_DCHECK_LE(params.fec_rate, 255); // Store the new params and apply them for the next set of FEC packets being // produced. - rtc::CritScope cs(&crit_); - pending_params_.emplace(delta_params, key_params); + new_params_ = params; + if (params.fec_rate > kHighProtectionThreshold) { + min_num_media_packets_ = kMinMediaPackets; + } else { + min_num_media_packets_ = 1; + } } -void UlpfecGenerator::AddPacketAndGenerateFec(const RtpPacketToSend& packet) { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); +int UlpfecGenerator::AddRtpPacketAndGenerateFec( + const rtc::CopyOnWriteBuffer& data_buffer, + size_t rtp_header_length) { RTC_DCHECK(generated_fec_packets_.empty()); - if (media_packets_.empty()) { - rtc::CritScope cs(&crit_); - if (pending_params_) { - current_params_ = *pending_params_; - pending_params_.reset(); - - if (CurrentParams().fec_rate > kHighProtectionThreshold) { - min_num_media_packets_ = kMinMediaPackets; - } else { - min_num_media_packets_ = 1; - } - } - - keyframe_in_process_ = packet.is_key_frame(); + params_ = new_params_; } - RTC_DCHECK_EQ(packet.is_key_frame(), keyframe_in_process_); - bool complete_frame = false; - const bool marker_bit = packet.Marker(); + const bool marker_bit = (data_buffer[1] & kRtpMarkerBitMask) ? true : false; if (media_packets_.size() < kUlpfecMaxMediaPackets) { // Our packet masks can only protect up to |kUlpfecMaxMediaPackets| packets. - auto fec_packet = std::make_unique(); - fec_packet->data = packet.Buffer(); - media_packets_.push_back(std::move(fec_packet)); - - // Keep a copy of the last RTP packet, so we can copy the RTP header - // from it when creating newly generated ULPFEC+RED packets. - RTC_DCHECK_GE(packet.headers_size(), kRtpHeaderSize); - last_media_packet_ = packet; + std::unique_ptr packet( + new ForwardErrorCorrection::Packet()); + RTC_DCHECK_GE(data_buffer.size(), rtp_header_length); + packet->data = data_buffer; + media_packets_.push_back(std::move(packet)); + // Keep track of the RTP header length, so we can copy the RTP header + // from |packet| to newly generated ULPFEC+RED packets. + RTC_DCHECK_GE(rtp_header_length, kRtpHeaderSize); + last_media_packet_rtp_header_length_ = rtp_header_length; } - if (marker_bit) { ++num_protected_frames_; complete_frame = true; } - - auto params = CurrentParams(); - // Produce FEC over at most |params_.max_fec_frames| frames, or as soon as: // (1) the excess overhead (actual overhead - requested/target overhead) is // less than |kMaxExcessOverhead|, and // (2) at least |min_num_media_packets_| media packets is reached. if (complete_frame && - (num_protected_frames_ == params.max_fec_frames || + (num_protected_frames_ == params_.max_fec_frames || (ExcessOverheadBelowMax() && MinimumMediaPacketsReached()))) { // We are not using Unequal Protection feature of the parity erasure code. constexpr int kNumImportantPackets = 0; constexpr bool kUseUnequalProtection = false; - fec_->EncodeFec(media_packets_, params.fec_rate, kNumImportantPackets, - kUseUnequalProtection, params.fec_mask_type, - &generated_fec_packets_); + int ret = fec_->EncodeFec(media_packets_, params_.fec_rate, + kNumImportantPackets, kUseUnequalProtection, + params_.fec_mask_type, &generated_fec_packets_); if (generated_fec_packets_.empty()) { ResetState(); } + return ret; } + return 0; } bool UlpfecGenerator::ExcessOverheadBelowMax() const { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); - - return ((Overhead() - CurrentParams().fec_rate) < kMaxExcessOverhead); + return ((Overhead() - params_.fec_rate) < kMaxExcessOverhead); } bool UlpfecGenerator::MinimumMediaPacketsReached() const { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); float average_num_packets_per_frame = static_cast(media_packets_.size()) / num_protected_frames_; int num_media_packets = static_cast(media_packets_.size()); @@ -188,79 +196,61 @@ bool UlpfecGenerator::MinimumMediaPacketsReached() const { } } -const FecProtectionParams& UlpfecGenerator::CurrentParams() const { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); - return keyframe_in_process_ ? current_params_.keyframe_params - : current_params_.delta_params; +bool UlpfecGenerator::FecAvailable() const { + return !generated_fec_packets_.empty(); +} + +size_t UlpfecGenerator::NumAvailableFecPackets() const { + return generated_fec_packets_.size(); } size_t UlpfecGenerator::MaxPacketOverhead() const { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); return fec_->MaxPacketOverhead(); } -std::vector> UlpfecGenerator::GetFecPackets() { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); - if (generated_fec_packets_.empty()) { - return std::vector>(); - } - - // Wrap FEC packet (including FEC headers) in a RED packet. Since the - // FEC packets in |generated_fec_packets_| don't have RTP headers, we - // reuse the header from the last media packet. - RTC_CHECK(last_media_packet_.has_value()); - last_media_packet_->SetPayloadSize(0); - - std::vector> fec_packets; - fec_packets.reserve(generated_fec_packets_.size()); - - size_t total_fec_size_bytes = 0; +std::vector> UlpfecGenerator::GetUlpfecPacketsAsRed( + int red_payload_type, + int ulpfec_payload_type, + uint16_t first_seq_num) { + std::vector> red_packets; + red_packets.reserve(generated_fec_packets_.size()); + RTC_DCHECK(!media_packets_.empty()); + ForwardErrorCorrection::Packet* last_media_packet = + media_packets_.back().get(); + uint16_t seq_num = first_seq_num; for (const auto* fec_packet : generated_fec_packets_) { - std::unique_ptr red_packet = - std::make_unique(*last_media_packet_); - red_packet->SetPayloadType(red_payload_type_); - red_packet->SetMarker(false); - uint8_t* payload_buffer = red_packet->SetPayloadSize( - kRedForFecHeaderLength + fec_packet->data.size()); - // Primary RED header with F bit unset. - // See https://tools.ietf.org/html/rfc2198#section-3 - payload_buffer[0] = ulpfec_payload_type_; // RED header. - memcpy(&payload_buffer[1], fec_packet->data.data(), - fec_packet->data.size()); - total_fec_size_bytes += red_packet->size(); - red_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection); - red_packet->set_allow_retransmission(false); - fec_packets.push_back(std::move(red_packet)); + // Wrap FEC packet (including FEC headers) in a RED packet. Since the + // FEC packets in |generated_fec_packets_| don't have RTP headers, we + // reuse the header from the last media packet. + RTC_DCHECK_GT(last_media_packet_rtp_header_length_, 0); + std::unique_ptr red_packet( + new RedPacket(last_media_packet_rtp_header_length_ + + kRedForFecHeaderLength + fec_packet->data.size())); + red_packet->CreateHeader(last_media_packet->data.data(), + last_media_packet_rtp_header_length_, + red_payload_type, ulpfec_payload_type); + red_packet->SetSeqNum(seq_num++); + red_packet->ClearMarkerBit(); + red_packet->AssignPayload(fec_packet->data.data(), fec_packet->data.size()); + red_packets.push_back(std::move(red_packet)); } ResetState(); - rtc::CritScope cs(&crit_); - fec_bitrate_.Update(total_fec_size_bytes, clock_->TimeInMilliseconds()); - - return fec_packets; -} - -DataRate UlpfecGenerator::CurrentFecRate() const { - rtc::CritScope cs(&crit_); - return DataRate::BitsPerSec( - fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0)); + return red_packets; } int UlpfecGenerator::Overhead() const { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); RTC_DCHECK(!media_packets_.empty()); int num_fec_packets = - fec_->NumFecPackets(media_packets_.size(), CurrentParams().fec_rate); - + fec_->NumFecPackets(media_packets_.size(), params_.fec_rate); // Return the overhead in Q8. return (num_fec_packets << 8) / media_packets_.size(); } void UlpfecGenerator::ResetState() { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); media_packets_.clear(); - last_media_packet_.reset(); + last_media_packet_rtp_header_length_ = 0; generated_fec_packets_.clear(); num_protected_frames_ = 0; } diff --git a/modules/rtp_rtcp/source/ulpfec_generator.h b/modules/rtp_rtcp/source/ulpfec_generator.h index 6c65f5f91e..cdfa1ff67d 100644 --- a/modules/rtp_rtcp/source/ulpfec_generator.h +++ b/modules/rtp_rtcp/source/ulpfec_generator.h @@ -20,54 +20,63 @@ #include "modules/include/module_fec_types.h" #include "modules/rtp_rtcp/source/forward_error_correction.h" -#include "modules/rtp_rtcp/source/video_fec_generator.h" -#include "rtc_base/critical_section.h" -#include "rtc_base/race_checker.h" -#include "rtc_base/rate_statistics.h" namespace webrtc { class FlexfecSender; -class UlpfecGenerator : public VideoFecGenerator { +class RedPacket { + public: + explicit RedPacket(size_t length); + ~RedPacket(); + + void CreateHeader(const uint8_t* rtp_header, + size_t header_length, + int red_payload_type, + int payload_type); + void SetSeqNum(int seq_num); + void AssignPayload(const uint8_t* payload, size_t length); + void ClearMarkerBit(); + uint8_t* data() const; + size_t length() const; + + private: + std::unique_ptr data_; + size_t length_; + size_t header_length_; +}; + +class UlpfecGenerator { friend class FlexfecSender; public: - UlpfecGenerator(int red_payload_type, int ulpfec_payload_type, Clock* clock); + UlpfecGenerator(); ~UlpfecGenerator(); - FecType GetFecType() const override { - return VideoFecGenerator::FecType::kUlpFec; - } - absl::optional FecSsrc() override { return absl::nullopt; } - - void SetProtectionParameters(const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) override; + void SetFecParameters(const FecProtectionParams& params); // Adds a media packet to the internal buffer. When enough media packets // have been added, the FEC packets are generated and stored internally. // These FEC packets are then obtained by calling GetFecPacketsAsRed(). - void AddPacketAndGenerateFec(const RtpPacketToSend& packet) override; + int AddRtpPacketAndGenerateFec(const rtc::CopyOnWriteBuffer& data_buffer, + size_t rtp_header_length); + + // Returns true if there are generated FEC packets available. + bool FecAvailable() const; + + size_t NumAvailableFecPackets() const; // Returns the overhead, per packet, for FEC (and possibly RED). - size_t MaxPacketOverhead() const override; + size_t MaxPacketOverhead() const; - std::vector> GetFecPackets() override; - - // Current rate of FEC packets generated, including all RTP-level headers. - DataRate CurrentFecRate() const override; + // Returns generated FEC packets with RED headers added. + std::vector> GetUlpfecPacketsAsRed( + int red_payload_type, + int ulpfec_payload_type, + uint16_t first_seq_num); private: - struct Params { - Params(); - Params(FecProtectionParams delta_params, - FecProtectionParams keyframe_params); - - FecProtectionParams delta_params; - FecProtectionParams keyframe_params; - }; - - UlpfecGenerator(std::unique_ptr fec, Clock* clock); + explicit UlpfecGenerator(std::unique_ptr fec); // Overhead is defined as relative to the number of media packets, and not // relative to total number of packets. This definition is inherited from the @@ -88,31 +97,16 @@ class UlpfecGenerator : public VideoFecGenerator { // (e.g. (2k,2m) vs (k,m)) are generally more effective at recovering losses. bool MinimumMediaPacketsReached() const; - const FecProtectionParams& CurrentParams() const; - void ResetState(); - const int red_payload_type_; - const int ulpfec_payload_type_; - Clock* const clock_; - - rtc::RaceChecker race_checker_; - const std::unique_ptr fec_ - RTC_GUARDED_BY(race_checker_); - ForwardErrorCorrection::PacketList media_packets_ - RTC_GUARDED_BY(race_checker_); - absl::optional last_media_packet_ - RTC_GUARDED_BY(race_checker_); - std::list generated_fec_packets_ - RTC_GUARDED_BY(race_checker_); - int num_protected_frames_ RTC_GUARDED_BY(race_checker_); - int min_num_media_packets_ RTC_GUARDED_BY(race_checker_); - Params current_params_ RTC_GUARDED_BY(race_checker_); - bool keyframe_in_process_ RTC_GUARDED_BY(race_checker_); - - rtc::CriticalSection crit_; - absl::optional pending_params_ RTC_GUARDED_BY(crit_); - RateStatistics fec_bitrate_ RTC_GUARDED_BY(crit_); + std::unique_ptr fec_; + ForwardErrorCorrection::PacketList media_packets_; + size_t last_media_packet_rtp_header_length_; + std::list generated_fec_packets_; + int num_protected_frames_; + int min_num_media_packets_; + FecProtectionParams params_; + FecProtectionParams new_params_; }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc b/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc index db005ddb49..8c1c7ea396 100644 --- a/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc +++ b/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc @@ -35,8 +35,11 @@ void VerifyHeader(uint16_t seq_num, uint32_t timestamp, int red_payload_type, int fec_payload_type, - bool marker_bit, - const rtc::CopyOnWriteBuffer& data) { + RedPacket* packet, + bool marker_bit) { + EXPECT_GT(packet->length(), kRtpHeaderSize); + EXPECT_TRUE(packet->data() != NULL); + uint8_t* data = packet->data(); // Marker bit not set. EXPECT_EQ(marker_bit ? 0x80 : 0, data[1] & 0x80); EXPECT_EQ(red_payload_type, data[1] & 0x7F); @@ -49,12 +52,8 @@ void VerifyHeader(uint16_t seq_num, class UlpfecGeneratorTest : public ::testing::Test { protected: - UlpfecGeneratorTest() - : fake_clock_(1), - ulpfec_generator_(kRedPayloadType, kFecPayloadType, &fake_clock_), - packet_generator_(kMediaSsrc) {} + UlpfecGeneratorTest() : packet_generator_(kMediaSsrc) {} - SimulatedClock fake_clock_; UlpfecGenerator ulpfec_generator_; AugmentedPacketGenerator packet_generator_; }; @@ -82,22 +81,24 @@ TEST_F(UlpfecGeneratorTest, NoEmptyFecWithSeqNumGaps) { protected_packets.push_back({21, 0, 55, 0}); protected_packets.push_back({13, 3, 57, 1}); FecProtectionParams params = {117, 3, kFecMaskBursty}; - ulpfec_generator_.SetProtectionParameters(params, params); + ulpfec_generator_.SetFecParameters(params); + uint8_t packet[28] = {0}; for (Packet p : protected_packets) { - RtpPacketToSend packet(nullptr); - packet.SetMarker(p.marker_bit); - packet.AllocateExtension(RTPExtensionType::kRtpExtensionMid, - p.header_size - packet.headers_size()); - packet.SetSequenceNumber(p.seq_num); - packet.AllocatePayload(p.payload_size); - ulpfec_generator_.AddPacketAndGenerateFec(packet); - - std::vector> fec_packets = - ulpfec_generator_.GetFecPackets(); - if (!p.marker_bit) { - EXPECT_TRUE(fec_packets.empty()); + if (p.marker_bit) { + packet[1] |= 0x80; } else { - EXPECT_FALSE(fec_packets.empty()); + packet[1] &= ~0x80; + } + ByteWriter::WriteBigEndian(&packet[2], p.seq_num); + ulpfec_generator_.AddRtpPacketAndGenerateFec( + rtc::CopyOnWriteBuffer(packet, p.payload_size + p.header_size), + p.header_size); + size_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets(); + if (num_fec_packets > 0) { + std::vector> fec_packets = + ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, + kFecPayloadType, 100); + EXPECT_EQ(num_fec_packets, fec_packets.size()); } } } @@ -112,28 +113,24 @@ TEST_F(UlpfecGeneratorTest, OneFrameFec) { constexpr size_t kNumPackets = 4; FecProtectionParams params = {15, 3, kFecMaskRandom}; packet_generator_.NewFrame(kNumPackets); - // Expecting one FEC packet. - ulpfec_generator_.SetProtectionParameters(params, params); + ulpfec_generator_.SetFecParameters(params); // Expecting one FEC packet. uint32_t last_timestamp = 0; for (size_t i = 0; i < kNumPackets; ++i) { std::unique_ptr packet = packet_generator_.NextPacket(i, 10); - RtpPacketToSend rtp_packet(nullptr); - EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size())); - ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet); + EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(packet->data, + kRtpHeaderSize)); last_timestamp = packet->header.timestamp; } - std::vector> fec_packets = - ulpfec_generator_.GetFecPackets(); - EXPECT_EQ(fec_packets.size(), 1u); - uint16_t seq_num = packet_generator_.NextPacketSeqNum(); - fec_packets[0]->SetSequenceNumber(seq_num); - EXPECT_TRUE(ulpfec_generator_.GetFecPackets().empty()); - - EXPECT_EQ(fec_packets[0]->headers_size(), kRtpHeaderSize); - - VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, false, - fec_packets[0]->Buffer()); + EXPECT_TRUE(ulpfec_generator_.FecAvailable()); + const uint16_t seq_num = packet_generator_.NextPacketSeqNum(); + std::vector> red_packets = + ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType, + seq_num); + EXPECT_FALSE(ulpfec_generator_.FecAvailable()); + ASSERT_EQ(1u, red_packets.size()); + VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, + red_packets.front().get(), false); } TEST_F(UlpfecGeneratorTest, TwoFrameFec) { @@ -148,27 +145,27 @@ TEST_F(UlpfecGeneratorTest, TwoFrameFec) { constexpr size_t kNumFrames = 2; FecProtectionParams params = {15, 3, kFecMaskRandom}; - // Expecting one FEC packet. - ulpfec_generator_.SetProtectionParameters(params, params); + ulpfec_generator_.SetFecParameters(params); // Expecting one FEC packet. uint32_t last_timestamp = 0; for (size_t i = 0; i < kNumFrames; ++i) { packet_generator_.NewFrame(kNumPackets); for (size_t j = 0; j < kNumPackets; ++j) { std::unique_ptr packet = packet_generator_.NextPacket(i * kNumPackets + j, 10); - RtpPacketToSend rtp_packet(nullptr); - EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size())); - ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet); + EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec( + packet->data, kRtpHeaderSize)); last_timestamp = packet->header.timestamp; } } - std::vector> fec_packets = - ulpfec_generator_.GetFecPackets(); - EXPECT_EQ(fec_packets.size(), 1u); + EXPECT_TRUE(ulpfec_generator_.FecAvailable()); const uint16_t seq_num = packet_generator_.NextPacketSeqNum(); - fec_packets[0]->SetSequenceNumber(seq_num); - VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, false, - fec_packets[0]->Buffer()); + std::vector> red_packets = + ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType, + seq_num); + EXPECT_FALSE(ulpfec_generator_.FecAvailable()); + ASSERT_EQ(1u, red_packets.size()); + VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, + red_packets.front().get(), false); } TEST_F(UlpfecGeneratorTest, MixedMediaRtpHeaderLengths) { @@ -177,43 +174,34 @@ TEST_F(UlpfecGeneratorTest, MixedMediaRtpHeaderLengths) { // Only one frame required to generate FEC. FecProtectionParams params = {127, 1, kFecMaskRandom}; - ulpfec_generator_.SetProtectionParameters(params, params); + ulpfec_generator_.SetFecParameters(params); // Fill up internal buffer with media packets with short RTP header length. packet_generator_.NewFrame(kUlpfecMaxMediaPackets + 1); for (size_t i = 0; i < kUlpfecMaxMediaPackets; ++i) { std::unique_ptr packet = packet_generator_.NextPacket(i, 10); - RtpPacketToSend rtp_packet(nullptr); - EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size())); - EXPECT_EQ(rtp_packet.headers_size(), kShortRtpHeaderLength); - ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet); - EXPECT_TRUE(ulpfec_generator_.GetFecPackets().empty()); + EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec( + packet->data, kShortRtpHeaderLength)); + EXPECT_FALSE(ulpfec_generator_.FecAvailable()); } // Kick off FEC generation with media packet with long RTP header length. // Since the internal buffer is full, this packet will not be protected. std::unique_ptr packet = packet_generator_.NextPacket(kUlpfecMaxMediaPackets, 10); - RtpPacketToSend rtp_packet(nullptr); - EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size())); - EXPECT_TRUE(rtp_packet.SetPayloadSize(0) != nullptr); - const uint32_t csrcs[]{1}; - rtp_packet.SetCsrcs(csrcs); - - EXPECT_EQ(rtp_packet.headers_size(), kLongRtpHeaderLength); - - ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet); - std::vector> fec_packets = - ulpfec_generator_.GetFecPackets(); - EXPECT_FALSE(fec_packets.empty()); + EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec( + packet->data, kLongRtpHeaderLength)); + EXPECT_TRUE(ulpfec_generator_.FecAvailable()); // Ensure that the RED header is placed correctly, i.e. the correct // RTP header length was used in the RED packet creation. - uint16_t seq_num = packet_generator_.NextPacketSeqNum(); - for (const auto& fec_packet : fec_packets) { - fec_packet->SetSequenceNumber(seq_num++); - EXPECT_EQ(kFecPayloadType, fec_packet->data()[kShortRtpHeaderLength]); + const uint16_t seq_num = packet_generator_.NextPacketSeqNum(); + std::vector> red_packets = + ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType, + seq_num); + for (const auto& red_packet : red_packets) { + EXPECT_EQ(kFecPayloadType, red_packet->data()[kShortRtpHeaderLength]); } } diff --git a/modules/rtp_rtcp/source/video_fec_generator.h b/modules/rtp_rtcp/source/video_fec_generator.h deleted file mode 100644 index 3731449b5c..0000000000 --- a/modules/rtp_rtcp/source/video_fec_generator.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_ -#define MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_ - -#include -#include - -#include "api/units/data_rate.h" -#include "modules/include/module_fec_types.h" -#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" - -namespace webrtc { - -class VideoFecGenerator { - public: - VideoFecGenerator() = default; - virtual ~VideoFecGenerator() = default; - - enum class FecType { kFlexFec, kUlpFec }; - virtual FecType GetFecType() const = 0; - // Returns the SSRC used for FEC packets (i.e. FlexFec SSRC). - virtual absl::optional FecSsrc() = 0; - // Returns the overhead, in bytes per packet, for FEC (and possibly RED). - virtual size_t MaxPacketOverhead() const = 0; - // Current rate of FEC packets generated, including all RTP-level headers. - virtual DataRate CurrentFecRate() const = 0; - // Set FEC rates, max frames before FEC is sent, and type of FEC masks. - virtual void SetProtectionParameters( - const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) = 0; - // Called on new media packet to be protected. The generator may choose - // to generate FEC packets at this time, if so they will be stored in an - // internal buffer. - virtual void AddPacketAndGenerateFec(const RtpPacketToSend& packet) = 0; - // Get (and remove) and FEC packets pending in the generator. These packets - // will lack sequence numbers, that needs to be set externally. - // TODO(bugs.webrtc.org/11340): Actually FlexFec sets seq#, fix that! - virtual std::vector> GetFecPackets() = 0; -}; - -} // namespace webrtc -#endif // MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_ diff --git a/test/fuzzers/BUILD.gn b/test/fuzzers/BUILD.gn index 6dd8173806..a15e5f0a03 100644 --- a/test/fuzzers/BUILD.gn +++ b/test/fuzzers/BUILD.gn @@ -148,7 +148,6 @@ webrtc_fuzzer_test("ulpfec_generator_fuzzer") { "../../modules/rtp_rtcp:rtp_rtcp_format", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", - "../../system_wrappers", ] } diff --git a/test/fuzzers/flexfec_sender_fuzzer.cc b/test/fuzzers/flexfec_sender_fuzzer.cc index 8ddd1c0fe0..4882f7df51 100644 --- a/test/fuzzers/flexfec_sender_fuzzer.cc +++ b/test/fuzzers/flexfec_sender_fuzzer.cc @@ -41,7 +41,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) { FecProtectionParams params = { data[i++], static_cast(data[i++] % 100), data[i++] <= 127 ? kFecMaskRandom : kFecMaskBursty}; - sender.SetProtectionParameters(params, params); + sender.SetFecParameters(params); uint16_t seq_num = data[i++]; while (i + 1 < size) { @@ -59,8 +59,11 @@ void FuzzOneInput(const uint8_t* data, size_t size) { RtpPacketToSend rtp_packet(nullptr); if (!rtp_packet.Parse(packet.get(), kRtpHeaderSize + payload_size)) break; - sender.AddPacketAndGenerateFec(rtp_packet); - sender.GetFecPackets(); + sender.AddRtpPacketAndGenerateFec(rtp_packet); + if (sender.FecAvailable()) { + std::vector> fec_packets = + sender.GetFecPackets(); + } } } diff --git a/test/fuzzers/ulpfec_generator_fuzzer.cc b/test/fuzzers/ulpfec_generator_fuzzer.cc index 4d46fa420d..306f7a0da9 100644 --- a/test/fuzzers/ulpfec_generator_fuzzer.cc +++ b/test/fuzzers/ulpfec_generator_fuzzer.cc @@ -16,7 +16,6 @@ #include "modules/rtp_rtcp/source/ulpfec_generator.h" #include "rtc_base/checks.h" #include "rtc_base/copy_on_write_buffer.h" -#include "system_wrappers/include/clock.h" namespace webrtc { @@ -26,14 +25,13 @@ constexpr uint8_t kRedPayloadType = 97; } // namespace void FuzzOneInput(const uint8_t* data, size_t size) { - SimulatedClock clock(1); - UlpfecGenerator generator(kRedPayloadType, kFecPayloadType, &clock); + UlpfecGenerator generator; size_t i = 0; if (size < 4) return; FecProtectionParams params = { data[i++] % 128, static_cast(data[i++] % 10), kFecMaskBursty}; - generator.SetProtectionParameters(params, params); + generator.SetFecParameters(params); uint16_t seq_num = data[i++]; uint16_t prev_seq_num = 0; while (i + 3 < size) { @@ -53,13 +51,16 @@ void FuzzOneInput(const uint8_t* data, size_t size) { // number became out of order. if (protect && IsNewerSequenceNumber(seq_num, prev_seq_num) && seq_num < prev_seq_num + kUlpfecMaxMediaPackets) { - RtpPacketToSend rtp_packet(nullptr); - rtp_packet.Parse(packet); - generator.AddPacketAndGenerateFec(rtp_packet); + generator.AddRtpPacketAndGenerateFec(packet, rtp_header_length); prev_seq_num = seq_num; } - - generator.GetFecPackets(); + const size_t num_fec_packets = generator.NumAvailableFecPackets(); + if (num_fec_packets > 0) { + std::vector> fec_packets = + generator.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType, + 100); + RTC_CHECK_EQ(num_fec_packets, fec_packets.size()); + } } } } // namespace webrtc