diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 9eb789cbfe..fba646ef9f 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -36,9 +36,13 @@ namespace webrtc { namespace webrtc_internal_rtp_video_sender { -RtpStreamSender::RtpStreamSender(std::unique_ptr rtp_rtcp, - std::unique_ptr sender_video) - : rtp_rtcp(std::move(rtp_rtcp)), sender_video(std::move(sender_video)) {} +RtpStreamSender::RtpStreamSender( + std::unique_ptr rtp_rtcp, + std::unique_ptr sender_video, + std::unique_ptr fec_generator) + : rtp_rtcp(std::move(rtp_rtcp)), + sender_video(std::move(sender_video)), + fec_generator(std::move(fec_generator)) {} RtpStreamSender::~RtpStreamSender() = default; @@ -113,6 +117,67 @@ bool ShouldDisableRedAndUlpfec(bool flexfec_enabled, return should_disable_red_and_ulpfec; } +// TODO(brandtr): Update this function when we support multistream protection. +std::unique_ptr MaybeCreateFecGenerator( + Clock* clock, + const RtpConfig& rtp, + const std::map& suspended_ssrcs, + int simulcast_index) { + // If flexfec is configured that takes priority. + if (rtp.flexfec.payload_type >= 0) { + RTC_DCHECK_GE(rtp.flexfec.payload_type, 0); + RTC_DCHECK_LE(rtp.flexfec.payload_type, 127); + if (rtp.flexfec.ssrc == 0) { + RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " + "Therefore disabling FlexFEC."; + return nullptr; + } + if (rtp.flexfec.protected_media_ssrcs.empty()) { + RTC_LOG(LS_WARNING) + << "FlexFEC is enabled, but no protected media SSRC given. " + "Therefore disabling FlexFEC."; + return nullptr; + } + + if (rtp.flexfec.protected_media_ssrcs.size() > 1) { + RTC_LOG(LS_WARNING) + << "The supplied FlexfecConfig contained multiple protected " + "media streams, but our implementation currently only " + "supports protecting a single media stream. " + "To avoid confusion, disabling FlexFEC completely."; + return nullptr; + } + + if (absl::c_find(rtp.flexfec.protected_media_ssrcs, + rtp.ssrcs[simulcast_index]) == + rtp.flexfec.protected_media_ssrcs.end()) { + // Media SSRC not among flexfec protected SSRCs. + return nullptr; + } + + const RtpState* rtp_state = nullptr; + auto it = suspended_ssrcs.find(rtp.flexfec.ssrc); + if (it != suspended_ssrcs.end()) { + rtp_state = &it->second; + } + + RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size()); + return std::make_unique( + rtp.flexfec.payload_type, rtp.flexfec.ssrc, + rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions, + RTPSender::FecExtensionSizes(), rtp_state, clock); + } else if (rtp.ulpfec.red_payload_type >= 0 && + rtp.ulpfec.ulpfec_payload_type >= 0 && + !ShouldDisableRedAndUlpfec(/*flexfec_enabled=*/false, rtp)) { + // Flexfec not configured, but ulpfec is and is not disabled. + return std::make_unique( + rtp.ulpfec.red_payload_type, rtp.ulpfec.ulpfec_payload_type, clock); + } + + // Not a single FEC is given. + return nullptr; +} + std::vector CreateRtpStreamSenders( Clock* clock, const RtpConfig& rtp_config, @@ -121,7 +186,7 @@ std::vector CreateRtpStreamSenders( Transport* send_transport, RtcpBandwidthObserver* bandwidth_callback, RtpTransportControllerSendInterface* transport, - FlexfecSender* flexfec_sender, + const std::map& suspended_ssrcs, RtcEventLog* event_log, RateLimiter* retransmission_rate_limiter, OverheadObserver* overhead_observer, @@ -161,18 +226,17 @@ std::vector CreateRtpStreamSenders( configuration.rtcp_report_interval_ms = rtcp_report_interval_ms; std::vector rtp_streams; - const std::vector& flexfec_protected_ssrcs = - rtp_config.flexfec.protected_media_ssrcs; + RTC_DCHECK(rtp_config.rtx.ssrcs.empty() || rtp_config.rtx.ssrcs.size() == rtp_config.rtx.ssrcs.size()); for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) { + RTPSenderVideo::Config video_config; configuration.local_media_ssrc = rtp_config.ssrcs[i]; - bool enable_flexfec = flexfec_sender != nullptr && - std::find(flexfec_protected_ssrcs.begin(), - flexfec_protected_ssrcs.end(), - configuration.local_media_ssrc) != - flexfec_protected_ssrcs.end(); - configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr; + + std::unique_ptr fec_generator = + MaybeCreateFecGenerator(clock, rtp_config, suspended_ssrcs, i); + configuration.fec_generator = fec_generator.get(); + video_config.fec_generator = fec_generator.get(); if (rtp_config.rtx.ssrcs.size() > i) { configuration.rtx_send_ssrc = rtp_config.rtx.ssrcs[i]; @@ -188,76 +252,31 @@ std::vector CreateRtpStreamSenders( rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize); FieldTrialBasedConfig field_trial_config; - RTPSenderVideo::Config video_config; video_config.clock = configuration.clock; video_config.rtp_sender = rtp_rtcp->RtpSender(); - video_config.flexfec_sender = configuration.flexfec_sender; video_config.frame_encryptor = frame_encryptor; video_config.require_frame_encryption = crypto_options.sframe.require_frame_encryption; video_config.enable_retransmit_all_layers = false; video_config.field_trials = &field_trial_config; + + const bool using_flexfec = + fec_generator && + fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec; const bool should_disable_red_and_ulpfec = - ShouldDisableRedAndUlpfec(enable_flexfec, rtp_config); - if (rtp_config.ulpfec.red_payload_type != -1 && - !should_disable_red_and_ulpfec) { + ShouldDisableRedAndUlpfec(using_flexfec, rtp_config); + if (!should_disable_red_and_ulpfec && + rtp_config.ulpfec.red_payload_type != -1) { video_config.red_payload_type = rtp_config.ulpfec.red_payload_type; } - if (rtp_config.ulpfec.ulpfec_payload_type != -1 && - !should_disable_red_and_ulpfec) { - video_config.ulpfec_payload_type = rtp_config.ulpfec.ulpfec_payload_type; - } video_config.frame_transformer = std::move(frame_transformer); auto sender_video = std::make_unique(video_config); - rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video)); + rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video), + std::move(fec_generator)); } return rtp_streams; } -// TODO(brandtr): Update this function when we support multistream protection. -std::unique_ptr MaybeCreateFlexfecSender( - Clock* clock, - const RtpConfig& rtp, - const std::map& suspended_ssrcs) { - if (rtp.flexfec.payload_type < 0) { - return nullptr; - } - RTC_DCHECK_GE(rtp.flexfec.payload_type, 0); - RTC_DCHECK_LE(rtp.flexfec.payload_type, 127); - if (rtp.flexfec.ssrc == 0) { - RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " - "Therefore disabling FlexFEC."; - return nullptr; - } - if (rtp.flexfec.protected_media_ssrcs.empty()) { - RTC_LOG(LS_WARNING) - << "FlexFEC is enabled, but no protected media SSRC given. " - "Therefore disabling FlexFEC."; - return nullptr; - } - - if (rtp.flexfec.protected_media_ssrcs.size() > 1) { - RTC_LOG(LS_WARNING) - << "The supplied FlexfecConfig contained multiple protected " - "media streams, but our implementation currently only " - "supports protecting a single media stream. " - "To avoid confusion, disabling FlexFEC completely."; - return nullptr; - } - - const RtpState* rtp_state = nullptr; - auto it = suspended_ssrcs.find(rtp.flexfec.ssrc); - if (it != suspended_ssrcs.end()) { - rtp_state = &it->second; - } - - RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size()); - return std::make_unique( - rtp.flexfec.payload_type, rtp.flexfec.ssrc, - rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions, - RTPSender::FecExtensionSizes(), rtp_state, clock); -} - DataRate CalculateOverheadRate(DataRate data_rate, DataSize packet_size, DataSize overhead_per_packet) { @@ -305,8 +324,6 @@ RtpVideoSender::RtpVideoSender( active_(false), module_process_thread_(nullptr), suspended_ssrcs_(std::move(suspended_ssrcs)), - flexfec_sender_( - MaybeCreateFlexfecSender(clock, rtp_config, suspended_ssrcs_)), fec_controller_(std::move(fec_controller)), fec_allowed_(true), rtp_streams_(CreateRtpStreamSenders(clock, @@ -316,7 +333,7 @@ RtpVideoSender::RtpVideoSender( send_transport, transport->GetBandwidthObserver(), transport, - flexfec_sender_.get(), + suspended_ssrcs_, event_log, retransmission_limiter, this, @@ -379,6 +396,7 @@ RtpVideoSender::RtpVideoSender( } } + bool fec_enabled = false; for (const RtpStreamSender& stream : rtp_streams_) { // Simulcast has one module for each layer. Set the CNAME on all modules. stream.rtp_rtcp->SetCNAME(rtp_config_.c_name.c_str()); @@ -388,10 +406,13 @@ RtpVideoSender::RtpVideoSender( stream.rtp_rtcp->SetMaxRtpPacketSize(rtp_config_.max_packet_size); stream.rtp_rtcp->RegisterSendPayloadFrequency(rtp_config_.payload_type, kVideoPayloadTypeFrequency); + if (stream.fec_generator != nullptr) { + fec_enabled = true; + } } // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, // so enable that logic if either of those FEC schemes are enabled. - fec_controller_->SetProtectionMethod(FecEnabled(), NackEnabled()); + fec_controller_->SetProtectionMethod(fec_enabled, NackEnabled()); fec_controller_->SetProtectionCallback(this); // Signal congestion controller this object is ready for OnPacket* callbacks. @@ -559,14 +580,6 @@ void RtpVideoSender::OnBitrateAllocationUpdated( } } -bool RtpVideoSender::FecEnabled() const { - const bool flexfec_enabled = (flexfec_sender_ != nullptr); - const bool ulpfec_enabled = - !webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment") && - (rtp_config_.ulpfec.ulpfec_payload_type >= 0); - return flexfec_enabled || ulpfec_enabled; -} - bool RtpVideoSender::NackEnabled() const { const bool nack_enabled = rtp_config_.nack.rtp_history_ms > 0; return nack_enabled; @@ -661,6 +674,14 @@ std::map RtpVideoSender::GetRtpStates() const { uint32_t ssrc = rtp_config_.ssrcs[i]; RTC_DCHECK_EQ(ssrc, rtp_streams_[i].rtp_rtcp->SSRC()); rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtpState(); + + VideoFecGenerator* fec_generator = rtp_streams_[i].fec_generator.get(); + if (fec_generator && + fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec) { + auto* flexfec_sender = static_cast(fec_generator); + uint32_t ssrc = rtp_config_.flexfec.ssrc; + rtp_states[ssrc] = flexfec_sender->GetRtpState(); + } } for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) { @@ -668,11 +689,6 @@ std::map RtpVideoSender::GetRtpStates() const { rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtxState(); } - if (flexfec_sender_) { - uint32_t ssrc = rtp_config_.flexfec.ssrc; - rtp_states[ssrc] = flexfec_sender_->GetRtpState(); - } - return rtp_states; } diff --git a/call/rtp_video_sender.h b/call/rtp_video_sender.h index 6c941f8acc..f7ebefcbb3 100644 --- a/call/rtp_video_sender.h +++ b/call/rtp_video_sender.h @@ -51,7 +51,8 @@ namespace webrtc_internal_rtp_video_sender { // RtpVideoSender. struct RtpStreamSender { RtpStreamSender(std::unique_ptr rtp_rtcp, - std::unique_ptr sender_video); + std::unique_ptr sender_video, + std::unique_ptr fec_generator); ~RtpStreamSender(); RtpStreamSender(RtpStreamSender&&) = default; @@ -60,6 +61,7 @@ struct RtpStreamSender { // Note: Needs pointer stability. std::unique_ptr rtp_rtcp; std::unique_ptr sender_video; + std::unique_ptr fec_generator; }; } // namespace webrtc_internal_rtp_video_sender @@ -155,7 +157,6 @@ class RtpVideoSender : public RtpVideoSenderInterface, void ConfigureProtection(); void ConfigureSsrcs(); void ConfigureRids(); - bool FecEnabled() const; bool NackEnabled() const; uint32_t GetPacketizationOverheadRate() const; @@ -173,8 +174,6 @@ class RtpVideoSender : public RtpVideoSenderInterface, rtc::ThreadChecker module_process_thread_checker_; std::map suspended_ssrcs_; - std::unique_ptr flexfec_sender_; - const std::unique_ptr fec_controller_; bool fec_allowed_ RTC_GUARDED_BY(crit_); diff --git a/modules/include/module_fec_types.h b/modules/include/module_fec_types.h index 25d6bc5714..f9b35cc288 100644 --- a/modules/include/module_fec_types.h +++ b/modules/include/module_fec_types.h @@ -24,9 +24,9 @@ enum FecMaskType { // Struct containing forward error correction settings. struct FecProtectionParams { - int fec_rate; - int max_fec_frames; - FecMaskType fec_mask_type; + int fec_rate = 0; + int max_fec_frames = 0; + FecMaskType fec_mask_type = FecMaskType::kFecMaskRandom; }; } // namespace webrtc diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 19a2c137a9..2826d0f09b 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -214,6 +214,7 @@ rtc_library("rtp_rtcp") { "source/ulpfec_header_reader_writer.h", "source/ulpfec_receiver_impl.cc", "source/ulpfec_receiver_impl.h", + "source/video_fec_generator.h", "source/video_rtp_depacketizer.h", "source/video_rtp_depacketizer_av1.cc", "source/video_rtp_depacketizer_av1.h", diff --git a/modules/rtp_rtcp/include/flexfec_sender.h b/modules/rtp_rtcp/include/flexfec_sender.h index 94f3502d31..4cc8f99ce6 100644 --- a/modules/rtp_rtcp/include/flexfec_sender.h +++ b/modules/rtp_rtcp/include/flexfec_sender.h @@ -21,7 +21,9 @@ #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_header_extension_size.h" #include "modules/rtp_rtcp/source/ulpfec_generator.h" +#include "modules/rtp_rtcp/source/video_fec_generator.h" #include "rtc_base/random.h" +#include "rtc_base/rate_statistics.h" namespace webrtc { @@ -31,7 +33,7 @@ class RtpPacketToSend; // Note that this class is not thread safe, and thus requires external // synchronization. Currently, this is done using the lock in PayloadRouter. -class FlexfecSender { +class FlexfecSender : public VideoFecGenerator { public: FlexfecSender(int payload_type, uint32_t ssrc, @@ -43,26 +45,28 @@ class FlexfecSender { Clock* clock); ~FlexfecSender(); - uint32_t ssrc() const { return ssrc_; } + FecType GetFecType() const override { + return VideoFecGenerator::FecType::kFlexFec; + } + absl::optional FecSsrc() override { return ssrc_; } // Sets the FEC rate, max frames sent before FEC packets are sent, // and what type of generator matrices are used. - void SetFecParameters(const FecProtectionParams& params); + void SetProtectionParameters(const FecProtectionParams& delta_params, + const FecProtectionParams& key_params) override; // Adds a media packet to the internal buffer. When enough media packets // have been added, the FEC packets are generated and stored internally. // These FEC packets are then obtained by calling GetFecPackets(). - // Returns true if the media packet was successfully added. - bool AddRtpPacketAndGenerateFec(const RtpPacketToSend& packet); - - // Returns true if there are generated FEC packets available. - bool FecAvailable() const; + void AddPacketAndGenerateFec(const RtpPacketToSend& packet) override; // Returns generated FlexFEC packets. - std::vector> GetFecPackets(); + std::vector> GetFecPackets() override; // Returns the overhead, per packet, for FlexFEC. - size_t MaxPacketOverhead() const; + size_t MaxPacketOverhead() const override; + + DataRate CurrentFecRate() const override; // Only called on the VideoSendStream queue, after operation has shut down. RtpState GetRtpState(); @@ -87,6 +91,9 @@ class FlexfecSender { UlpfecGenerator ulpfec_generator_; const RtpHeaderExtensionMap rtp_header_extension_map_; const size_t header_extensions_size_; + + rtc::CriticalSection crit_; + RateStatistics fec_bitrate_ RTC_GUARDED_BY(crit_); }; } // namespace webrtc diff --git a/modules/rtp_rtcp/include/rtp_rtcp.h b/modules/rtp_rtcp/include/rtp_rtcp.h index bb99b1ac27..e897718b2d 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp.h +++ b/modules/rtp_rtcp/include/rtp_rtcp.h @@ -24,13 +24,13 @@ #include "api/transport/webrtc_key_value_config.h" #include "api/video/video_bitrate_allocation.h" #include "modules/include/module.h" -#include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/include/report_block_data.h" #include "modules/rtp_rtcp/include/rtp_packet_sender.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" #include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" +#include "modules/rtp_rtcp/source/video_fec_generator.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/deprecation.h" @@ -94,9 +94,9 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { // Spread any bursts of packets into smaller bursts to minimize packet loss. RtpPacketSender* paced_sender = nullptr; - // Generate FlexFEC packets. - // TODO(brandtr): Remove when FlexfecSender is wired up to PacedSender. - FlexfecSender* flexfec_sender = nullptr; + // Generates FEC packets. + // TODO(sprang): Wire up to RtpSenderEgress. + VideoFecGenerator* fec_generator = nullptr; BitrateStatisticsObserver* send_bitrate_observer = nullptr; SendSideDelayObserver* send_side_delay_observer = nullptr; diff --git a/modules/rtp_rtcp/source/flexfec_sender.cc b/modules/rtp_rtcp/source/flexfec_sender.cc index de0d4129ce..4ff0893ee7 100644 --- a/modules/rtp_rtcp/source/flexfec_sender.cc +++ b/modules/rtp_rtcp/source/flexfec_sender.cc @@ -91,11 +91,13 @@ FlexfecSender::FlexfecSender( seq_num_(rtp_state ? rtp_state->sequence_number : random_.Rand(1, kMaxInitRtpSeqNumber)), ulpfec_generator_( - ForwardErrorCorrection::CreateFlexfec(ssrc, protected_media_ssrc)), + ForwardErrorCorrection::CreateFlexfec(ssrc, protected_media_ssrc), + clock_), rtp_header_extension_map_( RegisterSupportedExtensions(rtp_header_extensions)), header_extensions_size_( - RtpHeaderExtensionSize(extension_sizes, rtp_header_extension_map_)) { + RtpHeaderExtensionSize(extension_sizes, rtp_header_extension_map_)), + fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) { // This object should not have been instantiated if FlexFEC is disabled. RTC_DCHECK_GE(payload_type, 0); RTC_DCHECK_LE(payload_type, 127); @@ -105,30 +107,30 @@ FlexfecSender::~FlexfecSender() = default; // We are reusing the implementation from UlpfecGenerator for SetFecParameters, // AddRtpPacketAndGenerateFec, and FecAvailable. -void FlexfecSender::SetFecParameters(const FecProtectionParams& params) { - ulpfec_generator_.SetFecParameters(params); +void FlexfecSender::SetProtectionParameters( + const FecProtectionParams& delta_params, + const FecProtectionParams& key_params) { + ulpfec_generator_.SetProtectionParameters(delta_params, key_params); } -bool FlexfecSender::AddRtpPacketAndGenerateFec(const RtpPacketToSend& packet) { +void FlexfecSender::AddPacketAndGenerateFec(const RtpPacketToSend& packet) { // TODO(brandtr): Generalize this SSRC check when we support multistream // protection. RTC_DCHECK_EQ(packet.Ssrc(), protected_media_ssrc_); - return ulpfec_generator_.AddRtpPacketAndGenerateFec( - packet.Buffer(), packet.headers_size()) == 0; -} - -bool FlexfecSender::FecAvailable() const { - return ulpfec_generator_.FecAvailable(); + ulpfec_generator_.AddPacketAndGenerateFec(packet); } std::vector> FlexfecSender::GetFecPackets() { + RTC_CHECK_RUNS_SERIALIZED(&ulpfec_generator_.race_checker_); std::vector> fec_packets_to_send; fec_packets_to_send.reserve(ulpfec_generator_.generated_fec_packets_.size()); + size_t total_fec_data_bytes = 0; for (const auto* fec_packet : ulpfec_generator_.generated_fec_packets_) { std::unique_ptr fec_packet_to_send( new RtpPacketToSend(&rtp_header_extension_map_)); fec_packet_to_send->set_packet_type( RtpPacketMediaType::kForwardErrorCorrection); + fec_packet_to_send->set_allow_retransmission(false); // RTP header. fec_packet_to_send->SetMarker(false); @@ -157,9 +159,13 @@ std::vector> FlexfecSender::GetFecPackets() { fec_packet_to_send->AllocatePayload(fec_packet->data.size()); memcpy(payload, fec_packet->data.cdata(), fec_packet->data.size()); + total_fec_data_bytes += fec_packet_to_send->size(); fec_packets_to_send.push_back(std::move(fec_packet_to_send)); } - ulpfec_generator_.ResetState(); + + if (!fec_packets_to_send.empty()) { + ulpfec_generator_.ResetState(); + } int64_t now_ms = clock_->TimeInMilliseconds(); if (!fec_packets_to_send.empty() && @@ -170,6 +176,9 @@ std::vector> FlexfecSender::GetFecPackets() { last_generated_packet_ms_ = now_ms; } + rtc::CritScope cs(&crit_); + fec_bitrate_.Update(total_fec_data_bytes, now_ms); + return fec_packets_to_send; } @@ -178,6 +187,12 @@ size_t FlexfecSender::MaxPacketOverhead() const { return header_extensions_size_ + kFlexfecMaxHeaderSize; } +DataRate FlexfecSender::CurrentFecRate() const { + rtc::CritScope cs(&crit_); + return DataRate::BitsPerSec( + fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0)); +} + RtpState FlexfecSender::GetRtpState() { RtpState rtp_state; rtp_state.sequence_number = seq_num_; diff --git a/modules/rtp_rtcp/source/flexfec_sender_unittest.cc b/modules/rtp_rtcp/source/flexfec_sender_unittest.cc index 10ec2e7495..e4501c2c1d 100644 --- a/modules/rtp_rtcp/source/flexfec_sender_unittest.cc +++ b/modules/rtp_rtcp/source/flexfec_sender_unittest.cc @@ -55,7 +55,7 @@ std::unique_ptr GenerateSingleFlexfecPacket( params.fec_mask_type = kFecMaskRandom; constexpr size_t kNumPackets = 4; - sender->SetFecParameters(params); + sender->SetProtectionParameters(params, params); AugmentedPacketGenerator packet_generator(kMediaSsrc); packet_generator.NewFrame(kNumPackets); for (size_t i = 0; i < kNumPackets; ++i) { @@ -63,13 +63,12 @@ std::unique_ptr GenerateSingleFlexfecPacket( packet_generator.NextPacket(i, kPayloadLength); RtpPacketToSend rtp_packet(nullptr); // No header extensions. rtp_packet.Parse(packet->data); - EXPECT_TRUE(sender->AddRtpPacketAndGenerateFec(rtp_packet)); + sender->AddPacketAndGenerateFec(rtp_packet); } - EXPECT_TRUE(sender->FecAvailable()); std::vector> fec_packets = sender->GetFecPackets(); - EXPECT_FALSE(sender->FecAvailable()); EXPECT_EQ(1U, fec_packets.size()); + EXPECT_TRUE(sender->GetFecPackets().empty()); return std::move(fec_packets.front()); } @@ -82,7 +81,7 @@ TEST(FlexfecSenderTest, Ssrc) { kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes, nullptr /* rtp_state */, &clock); - EXPECT_EQ(kFlexfecSsrc, sender.ssrc()); + EXPECT_EQ(kFlexfecSsrc, sender.FecSsrc()); } TEST(FlexfecSenderTest, NoFecAvailableBeforeMediaAdded) { @@ -91,9 +90,7 @@ TEST(FlexfecSenderTest, NoFecAvailableBeforeMediaAdded) { kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes, nullptr /* rtp_state */, &clock); - EXPECT_FALSE(sender.FecAvailable()); - auto fec_packets = sender.GetFecPackets(); - EXPECT_EQ(0U, fec_packets.size()); + EXPECT_TRUE(sender.GetFecPackets().empty()); } TEST(FlexfecSenderTest, ProtectOneFrameWithOneFecPacket) { @@ -124,7 +121,7 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithOneFecPacket) { FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc, kNoMid, kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes, nullptr /* rtp_state */, &clock); - sender.SetFecParameters(params); + sender.SetProtectionParameters(params, params); AugmentedPacketGenerator packet_generator(kMediaSsrc); for (size_t i = 0; i < kNumFrames; ++i) { @@ -134,14 +131,13 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithOneFecPacket) { packet_generator.NextPacket(i, kPayloadLength); RtpPacketToSend rtp_packet(nullptr); rtp_packet.Parse(packet->data); - EXPECT_TRUE(sender.AddRtpPacketAndGenerateFec(rtp_packet)); + sender.AddPacketAndGenerateFec(rtp_packet); } } - EXPECT_TRUE(sender.FecAvailable()); std::vector> fec_packets = sender.GetFecPackets(); - EXPECT_FALSE(sender.FecAvailable()); ASSERT_EQ(1U, fec_packets.size()); + EXPECT_TRUE(sender.GetFecPackets().empty()); RtpPacketToSend* fec_packet = fec_packets.front().get(); EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size()); @@ -164,7 +160,7 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithTwoFecPackets) { FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc, kNoMid, kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes, nullptr /* rtp_state */, &clock); - sender.SetFecParameters(params); + sender.SetProtectionParameters(params, params); AugmentedPacketGenerator packet_generator(kMediaSsrc); for (size_t i = 0; i < kNumFrames; ++i) { @@ -174,13 +170,12 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithTwoFecPackets) { packet_generator.NextPacket(i, kPayloadLength); RtpPacketToSend rtp_packet(nullptr); rtp_packet.Parse(packet->data); - EXPECT_TRUE(sender.AddRtpPacketAndGenerateFec(rtp_packet)); + sender.AddPacketAndGenerateFec(rtp_packet); } - EXPECT_TRUE(sender.FecAvailable()); std::vector> fec_packets = sender.GetFecPackets(); - EXPECT_FALSE(sender.FecAvailable()); ASSERT_EQ(1U, fec_packets.size()); + EXPECT_TRUE(sender.GetFecPackets().empty()); RtpPacketToSend* fec_packet = fec_packets.front().get(); EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size()); diff --git a/modules/rtp_rtcp/source/rtcp_receiver.cc b/modules/rtp_rtcp/source/rtcp_receiver.cc index 6b64473eea..26465ada40 100644 --- a/modules/rtp_rtcp/source/rtcp_receiver.cc +++ b/modules/rtp_rtcp/source/rtcp_receiver.cc @@ -72,8 +72,11 @@ std::set GetRegisteredSsrcs(const RtpRtcp::Configuration& config) { if (config.rtx_send_ssrc) { ssrcs.insert(*config.rtx_send_ssrc); } - if (config.flexfec_sender) { - ssrcs.insert(config.flexfec_sender->ssrc()); + if (config.fec_generator) { + absl::optional flexfec_ssrc = config.fec_generator->FecSsrc(); + if (flexfec_ssrc) { + ssrcs.insert(*flexfec_ssrc); + } } return ssrcs; } diff --git a/modules/rtp_rtcp/source/rtp_packet_to_send.h b/modules/rtp_rtcp/source/rtp_packet_to_send.h index 57493e3802..8997bce0d2 100644 --- a/modules/rtp_rtcp/source/rtp_packet_to_send.h +++ b/modules/rtp_rtcp/source/rtp_packet_to_send.h @@ -98,12 +98,16 @@ class RtpPacketToSend : public RtpPacket { VideoTimingExtension::kNetwork2TimestampDeltaOffset); } + // Indicates if packet is the first packet of a video frame. void set_first_packet_of_frame(bool is_first_packet) { is_first_packet_of_frame_ = is_first_packet; } - bool is_first_packet_of_frame() const { return is_first_packet_of_frame_; } + // Indicates if packet contains payload for a video key-frame. + void set_is_key_frame(bool is_key_frame) { is_key_frame_ = is_key_frame; } + bool is_key_frame() const { return is_key_frame_; } + private: int64_t capture_time_ms_ = 0; absl::optional packet_type_; @@ -111,6 +115,7 @@ class RtpPacketToSend : public RtpPacket { absl::optional retransmitted_sequence_number_; std::vector application_data_; bool is_first_packet_of_frame_ = false; + bool is_key_frame_ = false; }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc index 3277c67314..c48a662fc5 100644 --- a/modules/rtp_rtcp/source/rtp_sender.cc +++ b/modules/rtp_rtcp/source/rtp_sender.cc @@ -102,9 +102,8 @@ RTPSender::RTPSender(const RtpRtcp::Configuration& config, audio_configured_(config.audio), ssrc_(config.local_media_ssrc), rtx_ssrc_(config.rtx_send_ssrc), - flexfec_ssrc_(config.flexfec_sender - ? absl::make_optional(config.flexfec_sender->ssrc()) - : absl::nullopt), + flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc() + : absl::nullopt), packet_history_(packet_history), paced_sender_(packet_sender), sending_media_(true), // Default to sending media. diff --git a/modules/rtp_rtcp/source/rtp_sender_egress.cc b/modules/rtp_rtcp/source/rtp_sender_egress.cc index d34d7c633a..ec546c47bf 100644 --- a/modules/rtp_rtcp/source/rtp_sender_egress.cc +++ b/modules/rtp_rtcp/source/rtp_sender_egress.cc @@ -57,9 +57,8 @@ RtpSenderEgress::RtpSenderEgress(const RtpRtcp::Configuration& config, RtpPacketHistory* packet_history) : ssrc_(config.local_media_ssrc), rtx_ssrc_(config.rtx_send_ssrc), - flexfec_ssrc_(config.flexfec_sender - ? absl::make_optional(config.flexfec_sender->ssrc()) - : absl::nullopt), + flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc() + : absl::nullopt), populate_network2_timestamp_(config.populate_network2_timestamp), send_side_bwe_with_overhead_( IsEnabled("WebRTC-SendSideBwe-WithOverhead", config.field_trials)), diff --git a/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_unittest.cc index c3ae539071..3b85166e61 100644 --- a/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -272,7 +272,7 @@ class RtpSenderTest : public ::testing::TestWithParam { config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; config.rtx_send_ssrc = kRtxSsrc; - config.flexfec_sender = &flexfec_sender_; + config.fec_generator = &flexfec_sender_; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -1225,7 +1225,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) { config.outgoing_transport = &transport_; config.paced_sender = &mock_paced_sender_; config.local_media_ssrc = kSsrc; - config.flexfec_sender = &flexfec_sender_; + config.fec_generator = &flexfec_sender_; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -1239,7 +1239,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) { RTPSenderVideo::Config video_config; video_config.clock = &fake_clock_; video_config.rtp_sender = rtp_sender(); - video_config.flexfec_sender = &flexfec_sender; + video_config.fec_generator = &flexfec_sender; video_config.field_trials = &field_trials; RTPSenderVideo rtp_sender_video(video_config); @@ -1311,7 +1311,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) { config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; - config.flexfec_sender = &flexfec_sender; + config.fec_generator = &flexfec_sender; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -1323,7 +1323,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) { RTPSenderVideo::Config video_config; video_config.clock = &fake_clock_; video_config.rtp_sender = rtp_sender(); - video_config.flexfec_sender = &flexfec_sender; + video_config.fec_generator = &flexfec_sender; video_config.field_trials = &field_trials; RTPSenderVideo rtp_sender_video(video_config); @@ -1583,7 +1583,7 @@ TEST_P(RtpSenderTest, FecOverheadRate) { config.outgoing_transport = &transport_; config.paced_sender = &mock_paced_sender_; config.local_media_ssrc = kSsrc; - config.flexfec_sender = &flexfec_sender; + config.fec_generator = &flexfec_sender; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -1595,7 +1595,7 @@ TEST_P(RtpSenderTest, FecOverheadRate) { RTPSenderVideo::Config video_config; video_config.clock = &fake_clock_; video_config.rtp_sender = rtp_sender(); - video_config.flexfec_sender = &flexfec_sender; + video_config.fec_generator = &flexfec_sender; video_config.field_trials = &field_trials; RTPSenderVideo rtp_sender_video(video_config); // Parameters selected to generate a single FEC packet per media packet. @@ -1777,12 +1777,14 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacksUlpfec) { const uint8_t kPayloadType = 127; const VideoCodecType kCodecType = VideoCodecType::kVideoCodecGeneric; FieldTrialBasedConfig field_trials; + UlpfecGenerator ulpfec_generator(kRedPayloadType, kUlpfecPayloadType, + &fake_clock_); RTPSenderVideo::Config video_config; video_config.clock = &fake_clock_; video_config.rtp_sender = rtp_sender(); video_config.field_trials = &field_trials; video_config.red_payload_type = kRedPayloadType; - video_config.ulpfec_payload_type = kUlpfecPayloadType; + video_config.fec_generator = &ulpfec_generator; RTPSenderVideo rtp_sender_video(video_config); uint8_t payload[] = {47, 11, 32, 93, 89}; rtp_sender_context_->packet_history_.SetStorePacketsStatus( @@ -2118,7 +2120,7 @@ TEST_P(RtpSenderTest, SendPacketUpdatesStats) { config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; config.rtx_send_ssrc = kRtxSsrc; - config.flexfec_sender = &flexfec_sender_; + config.fec_generator = &flexfec_sender_; config.send_side_delay_observer = &send_side_delay_observer; config.event_log = &mock_rtc_event_log_; config.send_packet_observer = &send_packet_observer_; diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 5ab7831d2d..ec5cf8fd13 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -260,11 +260,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) current_playout_delay_{-1, -1}, playout_delay_pending_(false), red_payload_type_(config.red_payload_type), - ulpfec_payload_type_(config.ulpfec_payload_type), - flexfec_sender_(config.flexfec_sender), - delta_fec_params_{0, 1, kFecMaskRandom}, - key_fec_params_{0, 1, kFecMaskRandom}, - fec_bitrate_(1000, RateStatistics::kBpsScale), + fec_generator_(config.fec_generator), video_bitrate_(1000, RateStatistics::kBpsScale), packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale), frame_encryptor_(config.frame_encryptor), @@ -293,83 +289,6 @@ RTPSenderVideo::~RTPSenderVideo() { frame_transformer_delegate_->Reset(); } -void RTPSenderVideo::AppendAsRedMaybeWithUlpfec( - std::unique_ptr media_packet, - bool protect_media_packet, - std::vector>* packets) { - std::unique_ptr red_packet( - new RtpPacketToSend(*media_packet)); - BuildRedPayload(*media_packet, red_packet.get()); - red_packet->SetPayloadType(*red_payload_type_); - - std::vector> fec_packets; - if (ulpfec_enabled()) { - if (protect_media_packet) { - if (exclude_transport_sequence_number_from_fec_experiment_) { - // See comments at the top of the file why experiment - // "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in - // conjunction with datagram transport. - // TODO(sukhanov): We may also need to implement it for flexfec_sender - // if we decide to keep this approach in the future. - uint16_t transport_senquence_number; - if (media_packet->GetExtension( - &transport_senquence_number)) { - if (!media_packet->RemoveExtension( - webrtc::TransportSequenceNumber::kId)) { - RTC_NOTREACHED() - << "Failed to remove transport sequence number, packet=" - << media_packet->ToString(); - } - } - } - - ulpfec_generator_.AddRtpPacketAndGenerateFec( - media_packet->Buffer(), media_packet->headers_size()); - } - uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets(); - if (num_fec_packets > 0) { - uint16_t first_fec_sequence_number = - rtp_sender_->AllocateSequenceNumber(num_fec_packets); - fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed( - *red_payload_type_, *ulpfec_payload_type_, first_fec_sequence_number); - RTC_DCHECK_EQ(num_fec_packets, fec_packets.size()); - } - } - - // Send |red_packet| instead of |packet| for allocated sequence number. - red_packet->set_packet_type(RtpPacketMediaType::kVideo); - red_packet->set_allow_retransmission(media_packet->allow_retransmission()); - packets->emplace_back(std::move(red_packet)); - - for (const auto& fec_packet : fec_packets) { - // TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid - // reparsing them. - std::unique_ptr rtp_packet( - new RtpPacketToSend(*media_packet)); - RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length())); - rtp_packet->set_capture_time_ms(media_packet->capture_time_ms()); - rtp_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection); - rtp_packet->set_allow_retransmission(false); - RTC_DCHECK_EQ(fec_packet->length(), rtp_packet->size()); - packets->emplace_back(std::move(rtp_packet)); - } -} - -void RTPSenderVideo::GenerateAndAppendFlexfec( - std::vector>* packets) { - RTC_DCHECK(flexfec_sender_); - - if (flexfec_sender_->FecAvailable()) { - std::vector> fec_packets = - flexfec_sender_->GetFecPackets(); - for (auto& fec_packet : fec_packets) { - fec_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection); - fec_packet->set_allow_retransmission(false); - packets->emplace_back(std::move(fec_packet)); - } - } -} - void RTPSenderVideo::LogAndSendToNetwork( std::vector> packets, size_t unpacketized_payload_size) { @@ -388,16 +307,9 @@ void RTPSenderVideo::LogAndSendToNetwork( rtc::CritScope cs(&stats_crit_); size_t packetized_payload_size = 0; for (const auto& packet : packets) { - switch (*packet->packet_type()) { - case RtpPacketMediaType::kVideo: - video_bitrate_.Update(packet->size(), now_ms); - packetized_payload_size += packet->payload_size(); - break; - case RtpPacketMediaType::kForwardErrorCorrection: - fec_bitrate_.Update(packet->size(), clock_->TimeInMilliseconds()); - break; - default: - continue; + if (*packet->packet_type() == RtpPacketMediaType::kVideo) { + video_bitrate_.Update(packet->size(), now_ms); + packetized_payload_size += packet->payload_size(); } } // AV1 packetizer may produce less packetized bytes than unpacketized. @@ -412,39 +324,31 @@ void RTPSenderVideo::LogAndSendToNetwork( } size_t RTPSenderVideo::FecPacketOverhead() const { - if (flexfec_enabled()) - return flexfec_sender_->MaxPacketOverhead(); - - size_t overhead = 0; + size_t overhead = fec_generator_ ? fec_generator_->MaxPacketOverhead() : 0u; if (red_enabled()) { // The RED overhead is due to a small header. overhead += kRedForFecHeaderLength; - } - if (ulpfec_enabled()) { - // For ULPFEC, the overhead is the FEC headers plus RED for FEC header - // (see above) plus anything in RTP header beyond the 12 bytes base header - // (CSRC list, extensions...) - // This reason for the header extensions to be included here is that - // from an FEC viewpoint, they are part of the payload to be protected. - // (The base RTP header is already protected by the FEC header.) - overhead += ulpfec_generator_.MaxPacketOverhead() + - (rtp_sender_->RtpHeaderLength() - kRtpHeaderSize); + + // TODO(bugs.webrtc.org/11340): Move this into UlpfecGenerator. + if (fec_generator_ && + fec_generator_->GetFecType() == VideoFecGenerator::FecType::kUlpFec) { + // For ULPFEC, the overhead is the FEC headers plus RED for FEC header + // (see above) plus anything in RTP header beyond the 12 bytes base header + // (CSRC list, extensions...) + // This reason for the header extensions to be included here is that + // from an FEC viewpoint, they are part of the payload to be protected. + // (The base RTP header is already protected by the FEC header.) + overhead += rtp_sender_->RtpHeaderLength() - kRtpHeaderSize; + } } return overhead; } void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params, const FecProtectionParams& key_params) { - rtc::CritScope cs(&crit_); - delta_fec_params_ = delta_params; - key_fec_params_ = key_params; -} - -absl::optional RTPSenderVideo::FlexfecSsrc() const { - if (flexfec_sender_) { - return flexfec_sender_->ssrc(); + if (fec_generator_) { + fec_generator_->SetProtectionParameters(delta_params, key_params); } - return absl::nullopt; } void RTPSenderVideo::SetVideoStructure( @@ -565,19 +469,6 @@ bool RTPSenderVideo::SendVideo( transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false; } - if (flexfec_enabled() || ulpfec_enabled()) { - rtc::CritScope cs(&crit_); - // FEC settings. - const FecProtectionParams& fec_params = - video_header.frame_type == VideoFrameType::kVideoFrameKey - ? key_fec_params_ - : delta_fec_params_; - if (flexfec_enabled()) - flexfec_sender_->SetFecParameters(fec_params); - if (ulpfec_enabled()) - ulpfec_generator_.SetFecParameters(fec_params); - } - // Maximum size of packet including rtp headers. // Extra space left in case packet will be resent using fec or rtx. int packet_capacity = rtp_sender_->MaxRtpPacketSize() - FecPacketOverhead() - @@ -769,21 +660,40 @@ bool RTPSenderVideo::SendVideo( packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds()); } + if (protect_packet && fec_generator_) { + if (red_enabled() && + exclude_transport_sequence_number_from_fec_experiment_) { + // See comments at the top of the file why experiment + // "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in + // conjunction with datagram transport. + // TODO(sukhanov): We may also need to implement it for flexfec_sender + // if we decide to keep this approach in the future. + uint16_t transport_senquence_number; + if (packet->GetExtension( + &transport_senquence_number)) { + if (!packet->RemoveExtension(webrtc::TransportSequenceNumber::kId)) { + RTC_NOTREACHED() + << "Failed to remove transport sequence number, packet=" + << packet->ToString(); + } + } + } + + fec_generator_->AddPacketAndGenerateFec(*packet); + } + if (red_enabled()) { - AppendAsRedMaybeWithUlpfec(std::move(packet), protect_packet, - &rtp_packets); + std::unique_ptr red_packet(new RtpPacketToSend(*packet)); + BuildRedPayload(*packet, red_packet.get()); + red_packet->SetPayloadType(*red_payload_type_); + + // Send |red_packet| instead of |packet| for allocated sequence number. + red_packet->set_packet_type(RtpPacketMediaType::kVideo); + red_packet->set_allow_retransmission(packet->allow_retransmission()); + rtp_packets.emplace_back(std::move(red_packet)); } else { packet->set_packet_type(RtpPacketMediaType::kVideo); - const RtpPacketToSend& media_packet = *packet; rtp_packets.emplace_back(std::move(packet)); - if (flexfec_enabled()) { - // TODO(brandtr): Remove the FlexFEC code path when FlexfecSender - // is wired up to PacedSender instead. - if (protect_packet) { - flexfec_sender_->AddRtpPacketAndGenerateFec(media_packet); - } - GenerateAndAppendFlexfec(&rtp_packets); - } } if (first_frame) { @@ -798,6 +708,22 @@ bool RTPSenderVideo::SendVideo( } } + if (fec_generator_) { + // Fetch any FEC packets generated from the media frame and add them to + // the list of packets to send. + auto fec_packets = fec_generator_->GetFecPackets(); + + // TODO(bugs.webrtc.org/11340): Move sequence number assignment into + // UlpfecGenerator. + const bool generate_sequence_numbers = !fec_generator_->FecSsrc(); + for (auto& fec_packet : fec_packets) { + if (generate_sequence_numbers) { + rtp_sender_->AssignSequenceNumber(fec_packet.get()); + } + rtp_packets.emplace_back(std::move(fec_packet)); + } + } + LogAndSendToNetwork(std::move(rtp_packets), unpacketized_payload_size); TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp", @@ -830,8 +756,7 @@ uint32_t RTPSenderVideo::VideoBitrateSent() const { } uint32_t RTPSenderVideo::FecOverheadRate() const { - rtc::CritScope cs(&stats_crit_); - return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0); + return fec_generator_ ? fec_generator_->CurrentFecRate().bps() : 0u; } uint32_t RTPSenderVideo::PacketizationOverheadBps() const { diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index 5fb669974a..440a0600bf 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -24,14 +24,13 @@ #include "api/video/video_codec_type.h" #include "api/video/video_frame_type.h" #include "modules/include/module_common_types.h" -#include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "modules/rtp_rtcp/source/rtp_sender.h" #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" -#include "modules/rtp_rtcp/source/ulpfec_generator.h" +#include "modules/rtp_rtcp/source/video_fec_generator.h" #include "rtc_base/critical_section.h" #include "rtc_base/one_time_event.h" #include "rtc_base/race_checker.h" @@ -71,11 +70,11 @@ class RTPSenderVideo { Clock* clock = nullptr; RTPSender* rtp_sender = nullptr; FlexfecSender* flexfec_sender = nullptr; + VideoFecGenerator* fec_generator = nullptr; FrameEncryptorInterface* frame_encryptor = nullptr; bool require_frame_encryption = false; bool enable_retransmit_all_layers = false; absl::optional red_payload_type; - absl::optional ulpfec_payload_type; const WebRtcKeyValueConfig* field_trials = nullptr; rtc::scoped_refptr frame_transformer; }; @@ -115,13 +114,9 @@ class RTPSenderVideo { // FlexFEC/ULPFEC. // Set FEC rates, max frames before FEC is sent, and type of FEC masks. - // Returns false on failure. void SetFecParameters(const FecProtectionParams& delta_params, const FecProtectionParams& key_params); - // FlexFEC. - absl::optional FlexfecSsrc() const; - uint32_t VideoBitrateSent() const; uint32_t FecOverheadRate() const; @@ -150,27 +145,12 @@ class RTPSenderVideo { size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); - void AppendAsRedMaybeWithUlpfec( - std::unique_ptr media_packet, - bool protect_media_packet, - std::vector>* packets) - RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); - - // TODO(brandtr): Remove the FlexFEC functions when FlexfecSender has been - // moved to PacedSender. - void GenerateAndAppendFlexfec( - std::vector>* packets); - void LogAndSendToNetwork( std::vector> packets, size_t unpacketized_payload_size); bool red_enabled() const { return red_payload_type_.has_value(); } - bool ulpfec_enabled() const { return ulpfec_payload_type_.has_value(); } - - bool flexfec_enabled() const { return flexfec_sender_ != nullptr; } - bool UpdateConditionalRetransmit(uint8_t temporal_id, int64_t expected_retransmission_time_ms) RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_crit_); @@ -201,22 +181,10 @@ class RTPSenderVideo { // Should never be held when calling out of this class. rtc::CriticalSection crit_; - // RED/ULPFEC. const absl::optional red_payload_type_; - const absl::optional ulpfec_payload_type_; - UlpfecGenerator ulpfec_generator_ RTC_GUARDED_BY(send_checker_); - - // FlexFEC. - FlexfecSender* const flexfec_sender_; - - // FEC parameters, applicable to either ULPFEC or FlexFEC. - FecProtectionParams delta_fec_params_ RTC_GUARDED_BY(crit_); - FecProtectionParams key_fec_params_ RTC_GUARDED_BY(crit_); + VideoFecGenerator* const fec_generator_; rtc::CriticalSection stats_crit_; - // Bitrate used for FEC payload, RED headers, RTP headers for FEC packets - // and any padding overhead. - RateStatistics fec_bitrate_ RTC_GUARDED_BY(stats_crit_); // Bitrate used for video payload and RTP headers. RateStatistics video_bitrate_ RTC_GUARDED_BY(stats_crit_); RateStatistics packetization_overhead_bitrate_ RTC_GUARDED_BY(stats_crit_); diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index a25c7b7d2a..5be9c9ed0d 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -131,7 +131,7 @@ class TestRtpSenderVideo : public RTPSenderVideo { Config config; config.clock = clock; config.rtp_sender = rtp_sender; - config.flexfec_sender = flexfec_sender; + config.fec_generator = flexfec_sender; config.field_trials = &field_trials; return config; }()) {} diff --git a/modules/rtp_rtcp/source/ulpfec_generator.cc b/modules/rtp_rtcp/source/ulpfec_generator.cc index 92e65df187..265fa4d1ac 100644 --- a/modules/rtp_rtcp/source/ulpfec_generator.cc +++ b/modules/rtp_rtcp/source/ulpfec_generator.cc @@ -22,6 +22,7 @@ #include "modules/rtp_rtcp/source/forward_error_correction_internal.h" #include "modules/rtp_rtcp/source/rtp_utility.h" #include "rtc_base/checks.h" +#include "rtc_base/critical_section.h" namespace webrtc { @@ -62,128 +63,119 @@ constexpr uint32_t kUnknownSsrc = 0; } // namespace -RedPacket::RedPacket(size_t length) - : data_(new uint8_t[length]), length_(length), header_length_(0) {} +UlpfecGenerator::Params::Params() = default; +UlpfecGenerator::Params::Params(FecProtectionParams delta_params, + FecProtectionParams keyframe_params) + : delta_params(delta_params), keyframe_params(keyframe_params) {} -RedPacket::~RedPacket() = default; - -void RedPacket::CreateHeader(const uint8_t* rtp_header, - size_t header_length, - int red_payload_type, - int payload_type) { - RTC_DCHECK_LE(header_length + kRedForFecHeaderLength, length_); - memcpy(data_.get(), rtp_header, header_length); - // Replace payload type. - data_[1] &= 0x80; - data_[1] += red_payload_type; - // Add RED header - // f-bit always 0 - data_[header_length] = static_cast(payload_type); - header_length_ = header_length + kRedForFecHeaderLength; -} - -void RedPacket::SetSeqNum(int seq_num) { - RTC_DCHECK_GE(seq_num, 0); - RTC_DCHECK_LT(seq_num, 1 << 16); - - ByteWriter::WriteBigEndian(&data_[2], seq_num); -} - -void RedPacket::AssignPayload(const uint8_t* payload, size_t length) { - RTC_DCHECK_LE(header_length_ + length, length_); - memcpy(data_.get() + header_length_, payload, length); -} - -void RedPacket::ClearMarkerBit() { - data_[1] &= 0x7F; -} - -uint8_t* RedPacket::data() const { - return data_.get(); -} - -size_t RedPacket::length() const { - return length_; -} - -UlpfecGenerator::UlpfecGenerator() - : UlpfecGenerator(ForwardErrorCorrection::CreateUlpfec(kUnknownSsrc)) {} - -UlpfecGenerator::UlpfecGenerator(std::unique_ptr fec) - : fec_(std::move(fec)), - last_media_packet_rtp_header_length_(0), +UlpfecGenerator::UlpfecGenerator(int red_payload_type, + int ulpfec_payload_type, + Clock* clock) + : red_payload_type_(red_payload_type), + ulpfec_payload_type_(ulpfec_payload_type), + clock_(clock), + fec_(ForwardErrorCorrection::CreateUlpfec(kUnknownSsrc)), num_protected_frames_(0), - min_num_media_packets_(1) { - memset(¶ms_, 0, sizeof(params_)); - memset(&new_params_, 0, sizeof(new_params_)); -} + min_num_media_packets_(1), + keyframe_in_process_(false), + fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {} + +// Used by FlexFecSender, payload types are unused. +UlpfecGenerator::UlpfecGenerator(std::unique_ptr fec, + Clock* clock) + : red_payload_type_(0), + ulpfec_payload_type_(0), + clock_(clock), + fec_(std::move(fec)), + num_protected_frames_(0), + min_num_media_packets_(1), + keyframe_in_process_(false), + fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {} UlpfecGenerator::~UlpfecGenerator() = default; -void UlpfecGenerator::SetFecParameters(const FecProtectionParams& params) { - RTC_DCHECK_GE(params.fec_rate, 0); - RTC_DCHECK_LE(params.fec_rate, 255); +void UlpfecGenerator::SetProtectionParameters( + const FecProtectionParams& delta_params, + const FecProtectionParams& key_params) { + RTC_DCHECK_GE(delta_params.fec_rate, 0); + RTC_DCHECK_LE(delta_params.fec_rate, 255); + RTC_DCHECK_GE(key_params.fec_rate, 0); + RTC_DCHECK_LE(key_params.fec_rate, 255); // Store the new params and apply them for the next set of FEC packets being // produced. - new_params_ = params; - if (params.fec_rate > kHighProtectionThreshold) { - min_num_media_packets_ = kMinMediaPackets; - } else { - min_num_media_packets_ = 1; - } + rtc::CritScope cs(&crit_); + pending_params_.emplace(delta_params, key_params); } -int UlpfecGenerator::AddRtpPacketAndGenerateFec( - const rtc::CopyOnWriteBuffer& data_buffer, - size_t rtp_header_length) { +void UlpfecGenerator::AddPacketAndGenerateFec(const RtpPacketToSend& packet) { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); RTC_DCHECK(generated_fec_packets_.empty()); + if (media_packets_.empty()) { - params_ = new_params_; + rtc::CritScope cs(&crit_); + if (pending_params_) { + current_params_ = *pending_params_; + pending_params_.reset(); + + if (CurrentParams().fec_rate > kHighProtectionThreshold) { + min_num_media_packets_ = kMinMediaPackets; + } else { + min_num_media_packets_ = 1; + } + } + + keyframe_in_process_ = packet.is_key_frame(); } + RTC_DCHECK_EQ(packet.is_key_frame(), keyframe_in_process_); + bool complete_frame = false; - const bool marker_bit = (data_buffer[1] & kRtpMarkerBitMask) ? true : false; + const bool marker_bit = packet.Marker(); if (media_packets_.size() < kUlpfecMaxMediaPackets) { // Our packet masks can only protect up to |kUlpfecMaxMediaPackets| packets. - std::unique_ptr packet( - new ForwardErrorCorrection::Packet()); - RTC_DCHECK_GE(data_buffer.size(), rtp_header_length); - packet->data = data_buffer; - media_packets_.push_back(std::move(packet)); - // Keep track of the RTP header length, so we can copy the RTP header - // from |packet| to newly generated ULPFEC+RED packets. - RTC_DCHECK_GE(rtp_header_length, kRtpHeaderSize); - last_media_packet_rtp_header_length_ = rtp_header_length; + auto fec_packet = std::make_unique(); + fec_packet->data = packet.Buffer(); + media_packets_.push_back(std::move(fec_packet)); + + // Keep a copy of the last RTP packet, so we can copy the RTP header + // from it when creating newly generated ULPFEC+RED packets. + RTC_DCHECK_GE(packet.headers_size(), kRtpHeaderSize); + last_media_packet_ = packet; } + if (marker_bit) { ++num_protected_frames_; complete_frame = true; } + + auto params = CurrentParams(); + // Produce FEC over at most |params_.max_fec_frames| frames, or as soon as: // (1) the excess overhead (actual overhead - requested/target overhead) is // less than |kMaxExcessOverhead|, and // (2) at least |min_num_media_packets_| media packets is reached. if (complete_frame && - (num_protected_frames_ == params_.max_fec_frames || + (num_protected_frames_ == params.max_fec_frames || (ExcessOverheadBelowMax() && MinimumMediaPacketsReached()))) { // We are not using Unequal Protection feature of the parity erasure code. constexpr int kNumImportantPackets = 0; constexpr bool kUseUnequalProtection = false; - int ret = fec_->EncodeFec(media_packets_, params_.fec_rate, - kNumImportantPackets, kUseUnequalProtection, - params_.fec_mask_type, &generated_fec_packets_); + fec_->EncodeFec(media_packets_, params.fec_rate, kNumImportantPackets, + kUseUnequalProtection, params.fec_mask_type, + &generated_fec_packets_); if (generated_fec_packets_.empty()) { ResetState(); } - return ret; } - return 0; } bool UlpfecGenerator::ExcessOverheadBelowMax() const { - return ((Overhead() - params_.fec_rate) < kMaxExcessOverhead); + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + + return ((Overhead() - CurrentParams().fec_rate) < kMaxExcessOverhead); } bool UlpfecGenerator::MinimumMediaPacketsReached() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); float average_num_packets_per_frame = static_cast(media_packets_.size()) / num_protected_frames_; int num_media_packets = static_cast(media_packets_.size()); @@ -196,61 +188,79 @@ bool UlpfecGenerator::MinimumMediaPacketsReached() const { } } -bool UlpfecGenerator::FecAvailable() const { - return !generated_fec_packets_.empty(); -} - -size_t UlpfecGenerator::NumAvailableFecPackets() const { - return generated_fec_packets_.size(); +const FecProtectionParams& UlpfecGenerator::CurrentParams() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + return keyframe_in_process_ ? current_params_.keyframe_params + : current_params_.delta_params; } size_t UlpfecGenerator::MaxPacketOverhead() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); return fec_->MaxPacketOverhead(); } -std::vector> UlpfecGenerator::GetUlpfecPacketsAsRed( - int red_payload_type, - int ulpfec_payload_type, - uint16_t first_seq_num) { - std::vector> red_packets; - red_packets.reserve(generated_fec_packets_.size()); - RTC_DCHECK(!media_packets_.empty()); - ForwardErrorCorrection::Packet* last_media_packet = - media_packets_.back().get(); - uint16_t seq_num = first_seq_num; +std::vector> UlpfecGenerator::GetFecPackets() { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + if (generated_fec_packets_.empty()) { + return std::vector>(); + } + + // Wrap FEC packet (including FEC headers) in a RED packet. Since the + // FEC packets in |generated_fec_packets_| don't have RTP headers, we + // reuse the header from the last media packet. + RTC_CHECK(last_media_packet_.has_value()); + last_media_packet_->SetPayloadSize(0); + + std::vector> fec_packets; + fec_packets.reserve(generated_fec_packets_.size()); + + size_t total_fec_size_bytes = 0; for (const auto* fec_packet : generated_fec_packets_) { - // Wrap FEC packet (including FEC headers) in a RED packet. Since the - // FEC packets in |generated_fec_packets_| don't have RTP headers, we - // reuse the header from the last media packet. - RTC_DCHECK_GT(last_media_packet_rtp_header_length_, 0); - std::unique_ptr red_packet( - new RedPacket(last_media_packet_rtp_header_length_ + - kRedForFecHeaderLength + fec_packet->data.size())); - red_packet->CreateHeader(last_media_packet->data.data(), - last_media_packet_rtp_header_length_, - red_payload_type, ulpfec_payload_type); - red_packet->SetSeqNum(seq_num++); - red_packet->ClearMarkerBit(); - red_packet->AssignPayload(fec_packet->data.data(), fec_packet->data.size()); - red_packets.push_back(std::move(red_packet)); + std::unique_ptr red_packet = + std::make_unique(*last_media_packet_); + red_packet->SetPayloadType(red_payload_type_); + red_packet->SetMarker(false); + uint8_t* payload_buffer = red_packet->SetPayloadSize( + kRedForFecHeaderLength + fec_packet->data.size()); + // Primary RED header with F bit unset. + // See https://tools.ietf.org/html/rfc2198#section-3 + payload_buffer[0] = ulpfec_payload_type_; // RED header. + memcpy(&payload_buffer[1], fec_packet->data.data(), + fec_packet->data.size()); + total_fec_size_bytes += red_packet->size(); + red_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection); + red_packet->set_allow_retransmission(false); + fec_packets.push_back(std::move(red_packet)); } ResetState(); - return red_packets; + rtc::CritScope cs(&crit_); + fec_bitrate_.Update(total_fec_size_bytes, clock_->TimeInMilliseconds()); + + return fec_packets; +} + +DataRate UlpfecGenerator::CurrentFecRate() const { + rtc::CritScope cs(&crit_); + return DataRate::BitsPerSec( + fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0)); } int UlpfecGenerator::Overhead() const { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); RTC_DCHECK(!media_packets_.empty()); int num_fec_packets = - fec_->NumFecPackets(media_packets_.size(), params_.fec_rate); + fec_->NumFecPackets(media_packets_.size(), CurrentParams().fec_rate); + // Return the overhead in Q8. return (num_fec_packets << 8) / media_packets_.size(); } void UlpfecGenerator::ResetState() { + RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); media_packets_.clear(); - last_media_packet_rtp_header_length_ = 0; + last_media_packet_.reset(); generated_fec_packets_.clear(); num_protected_frames_ = 0; } diff --git a/modules/rtp_rtcp/source/ulpfec_generator.h b/modules/rtp_rtcp/source/ulpfec_generator.h index cdfa1ff67d..6c65f5f91e 100644 --- a/modules/rtp_rtcp/source/ulpfec_generator.h +++ b/modules/rtp_rtcp/source/ulpfec_generator.h @@ -20,63 +20,54 @@ #include "modules/include/module_fec_types.h" #include "modules/rtp_rtcp/source/forward_error_correction.h" +#include "modules/rtp_rtcp/source/video_fec_generator.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/rate_statistics.h" namespace webrtc { class FlexfecSender; -class RedPacket { - public: - explicit RedPacket(size_t length); - ~RedPacket(); - - void CreateHeader(const uint8_t* rtp_header, - size_t header_length, - int red_payload_type, - int payload_type); - void SetSeqNum(int seq_num); - void AssignPayload(const uint8_t* payload, size_t length); - void ClearMarkerBit(); - uint8_t* data() const; - size_t length() const; - - private: - std::unique_ptr data_; - size_t length_; - size_t header_length_; -}; - -class UlpfecGenerator { +class UlpfecGenerator : public VideoFecGenerator { friend class FlexfecSender; public: - UlpfecGenerator(); + UlpfecGenerator(int red_payload_type, int ulpfec_payload_type, Clock* clock); ~UlpfecGenerator(); - void SetFecParameters(const FecProtectionParams& params); + FecType GetFecType() const override { + return VideoFecGenerator::FecType::kUlpFec; + } + absl::optional FecSsrc() override { return absl::nullopt; } + + void SetProtectionParameters(const FecProtectionParams& delta_params, + const FecProtectionParams& key_params) override; // Adds a media packet to the internal buffer. When enough media packets // have been added, the FEC packets are generated and stored internally. // These FEC packets are then obtained by calling GetFecPacketsAsRed(). - int AddRtpPacketAndGenerateFec(const rtc::CopyOnWriteBuffer& data_buffer, - size_t rtp_header_length); - - // Returns true if there are generated FEC packets available. - bool FecAvailable() const; - - size_t NumAvailableFecPackets() const; + void AddPacketAndGenerateFec(const RtpPacketToSend& packet) override; // Returns the overhead, per packet, for FEC (and possibly RED). - size_t MaxPacketOverhead() const; + size_t MaxPacketOverhead() const override; - // Returns generated FEC packets with RED headers added. - std::vector> GetUlpfecPacketsAsRed( - int red_payload_type, - int ulpfec_payload_type, - uint16_t first_seq_num); + std::vector> GetFecPackets() override; + + // Current rate of FEC packets generated, including all RTP-level headers. + DataRate CurrentFecRate() const override; private: - explicit UlpfecGenerator(std::unique_ptr fec); + struct Params { + Params(); + Params(FecProtectionParams delta_params, + FecProtectionParams keyframe_params); + + FecProtectionParams delta_params; + FecProtectionParams keyframe_params; + }; + + UlpfecGenerator(std::unique_ptr fec, Clock* clock); // Overhead is defined as relative to the number of media packets, and not // relative to total number of packets. This definition is inherited from the @@ -97,16 +88,31 @@ class UlpfecGenerator { // (e.g. (2k,2m) vs (k,m)) are generally more effective at recovering losses. bool MinimumMediaPacketsReached() const; + const FecProtectionParams& CurrentParams() const; + void ResetState(); - std::unique_ptr fec_; - ForwardErrorCorrection::PacketList media_packets_; - size_t last_media_packet_rtp_header_length_; - std::list generated_fec_packets_; - int num_protected_frames_; - int min_num_media_packets_; - FecProtectionParams params_; - FecProtectionParams new_params_; + const int red_payload_type_; + const int ulpfec_payload_type_; + Clock* const clock_; + + rtc::RaceChecker race_checker_; + const std::unique_ptr fec_ + RTC_GUARDED_BY(race_checker_); + ForwardErrorCorrection::PacketList media_packets_ + RTC_GUARDED_BY(race_checker_); + absl::optional last_media_packet_ + RTC_GUARDED_BY(race_checker_); + std::list generated_fec_packets_ + RTC_GUARDED_BY(race_checker_); + int num_protected_frames_ RTC_GUARDED_BY(race_checker_); + int min_num_media_packets_ RTC_GUARDED_BY(race_checker_); + Params current_params_ RTC_GUARDED_BY(race_checker_); + bool keyframe_in_process_ RTC_GUARDED_BY(race_checker_); + + rtc::CriticalSection crit_; + absl::optional pending_params_ RTC_GUARDED_BY(crit_); + RateStatistics fec_bitrate_ RTC_GUARDED_BY(crit_); }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc b/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc index 8c1c7ea396..db005ddb49 100644 --- a/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc +++ b/modules/rtp_rtcp/source/ulpfec_generator_unittest.cc @@ -35,11 +35,8 @@ void VerifyHeader(uint16_t seq_num, uint32_t timestamp, int red_payload_type, int fec_payload_type, - RedPacket* packet, - bool marker_bit) { - EXPECT_GT(packet->length(), kRtpHeaderSize); - EXPECT_TRUE(packet->data() != NULL); - uint8_t* data = packet->data(); + bool marker_bit, + const rtc::CopyOnWriteBuffer& data) { // Marker bit not set. EXPECT_EQ(marker_bit ? 0x80 : 0, data[1] & 0x80); EXPECT_EQ(red_payload_type, data[1] & 0x7F); @@ -52,8 +49,12 @@ void VerifyHeader(uint16_t seq_num, class UlpfecGeneratorTest : public ::testing::Test { protected: - UlpfecGeneratorTest() : packet_generator_(kMediaSsrc) {} + UlpfecGeneratorTest() + : fake_clock_(1), + ulpfec_generator_(kRedPayloadType, kFecPayloadType, &fake_clock_), + packet_generator_(kMediaSsrc) {} + SimulatedClock fake_clock_; UlpfecGenerator ulpfec_generator_; AugmentedPacketGenerator packet_generator_; }; @@ -81,24 +82,22 @@ TEST_F(UlpfecGeneratorTest, NoEmptyFecWithSeqNumGaps) { protected_packets.push_back({21, 0, 55, 0}); protected_packets.push_back({13, 3, 57, 1}); FecProtectionParams params = {117, 3, kFecMaskBursty}; - ulpfec_generator_.SetFecParameters(params); - uint8_t packet[28] = {0}; + ulpfec_generator_.SetProtectionParameters(params, params); for (Packet p : protected_packets) { - if (p.marker_bit) { - packet[1] |= 0x80; + RtpPacketToSend packet(nullptr); + packet.SetMarker(p.marker_bit); + packet.AllocateExtension(RTPExtensionType::kRtpExtensionMid, + p.header_size - packet.headers_size()); + packet.SetSequenceNumber(p.seq_num); + packet.AllocatePayload(p.payload_size); + ulpfec_generator_.AddPacketAndGenerateFec(packet); + + std::vector> fec_packets = + ulpfec_generator_.GetFecPackets(); + if (!p.marker_bit) { + EXPECT_TRUE(fec_packets.empty()); } else { - packet[1] &= ~0x80; - } - ByteWriter::WriteBigEndian(&packet[2], p.seq_num); - ulpfec_generator_.AddRtpPacketAndGenerateFec( - rtc::CopyOnWriteBuffer(packet, p.payload_size + p.header_size), - p.header_size); - size_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets(); - if (num_fec_packets > 0) { - std::vector> fec_packets = - ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, - kFecPayloadType, 100); - EXPECT_EQ(num_fec_packets, fec_packets.size()); + EXPECT_FALSE(fec_packets.empty()); } } } @@ -113,24 +112,28 @@ TEST_F(UlpfecGeneratorTest, OneFrameFec) { constexpr size_t kNumPackets = 4; FecProtectionParams params = {15, 3, kFecMaskRandom}; packet_generator_.NewFrame(kNumPackets); - ulpfec_generator_.SetFecParameters(params); // Expecting one FEC packet. + // Expecting one FEC packet. + ulpfec_generator_.SetProtectionParameters(params, params); uint32_t last_timestamp = 0; for (size_t i = 0; i < kNumPackets; ++i) { std::unique_ptr packet = packet_generator_.NextPacket(i, 10); - EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(packet->data, - kRtpHeaderSize)); + RtpPacketToSend rtp_packet(nullptr); + EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size())); + ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet); last_timestamp = packet->header.timestamp; } - EXPECT_TRUE(ulpfec_generator_.FecAvailable()); - const uint16_t seq_num = packet_generator_.NextPacketSeqNum(); - std::vector> red_packets = - ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType, - seq_num); - EXPECT_FALSE(ulpfec_generator_.FecAvailable()); - ASSERT_EQ(1u, red_packets.size()); - VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, - red_packets.front().get(), false); + std::vector> fec_packets = + ulpfec_generator_.GetFecPackets(); + EXPECT_EQ(fec_packets.size(), 1u); + uint16_t seq_num = packet_generator_.NextPacketSeqNum(); + fec_packets[0]->SetSequenceNumber(seq_num); + EXPECT_TRUE(ulpfec_generator_.GetFecPackets().empty()); + + EXPECT_EQ(fec_packets[0]->headers_size(), kRtpHeaderSize); + + VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, false, + fec_packets[0]->Buffer()); } TEST_F(UlpfecGeneratorTest, TwoFrameFec) { @@ -145,27 +148,27 @@ TEST_F(UlpfecGeneratorTest, TwoFrameFec) { constexpr size_t kNumFrames = 2; FecProtectionParams params = {15, 3, kFecMaskRandom}; - ulpfec_generator_.SetFecParameters(params); // Expecting one FEC packet. + // Expecting one FEC packet. + ulpfec_generator_.SetProtectionParameters(params, params); uint32_t last_timestamp = 0; for (size_t i = 0; i < kNumFrames; ++i) { packet_generator_.NewFrame(kNumPackets); for (size_t j = 0; j < kNumPackets; ++j) { std::unique_ptr packet = packet_generator_.NextPacket(i * kNumPackets + j, 10); - EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec( - packet->data, kRtpHeaderSize)); + RtpPacketToSend rtp_packet(nullptr); + EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size())); + ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet); last_timestamp = packet->header.timestamp; } } - EXPECT_TRUE(ulpfec_generator_.FecAvailable()); + std::vector> fec_packets = + ulpfec_generator_.GetFecPackets(); + EXPECT_EQ(fec_packets.size(), 1u); const uint16_t seq_num = packet_generator_.NextPacketSeqNum(); - std::vector> red_packets = - ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType, - seq_num); - EXPECT_FALSE(ulpfec_generator_.FecAvailable()); - ASSERT_EQ(1u, red_packets.size()); - VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, - red_packets.front().get(), false); + fec_packets[0]->SetSequenceNumber(seq_num); + VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, false, + fec_packets[0]->Buffer()); } TEST_F(UlpfecGeneratorTest, MixedMediaRtpHeaderLengths) { @@ -174,34 +177,43 @@ TEST_F(UlpfecGeneratorTest, MixedMediaRtpHeaderLengths) { // Only one frame required to generate FEC. FecProtectionParams params = {127, 1, kFecMaskRandom}; - ulpfec_generator_.SetFecParameters(params); + ulpfec_generator_.SetProtectionParameters(params, params); // Fill up internal buffer with media packets with short RTP header length. packet_generator_.NewFrame(kUlpfecMaxMediaPackets + 1); for (size_t i = 0; i < kUlpfecMaxMediaPackets; ++i) { std::unique_ptr packet = packet_generator_.NextPacket(i, 10); - EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec( - packet->data, kShortRtpHeaderLength)); - EXPECT_FALSE(ulpfec_generator_.FecAvailable()); + RtpPacketToSend rtp_packet(nullptr); + EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size())); + EXPECT_EQ(rtp_packet.headers_size(), kShortRtpHeaderLength); + ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet); + EXPECT_TRUE(ulpfec_generator_.GetFecPackets().empty()); } // Kick off FEC generation with media packet with long RTP header length. // Since the internal buffer is full, this packet will not be protected. std::unique_ptr packet = packet_generator_.NextPacket(kUlpfecMaxMediaPackets, 10); - EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec( - packet->data, kLongRtpHeaderLength)); - EXPECT_TRUE(ulpfec_generator_.FecAvailable()); + RtpPacketToSend rtp_packet(nullptr); + EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size())); + EXPECT_TRUE(rtp_packet.SetPayloadSize(0) != nullptr); + const uint32_t csrcs[]{1}; + rtp_packet.SetCsrcs(csrcs); + + EXPECT_EQ(rtp_packet.headers_size(), kLongRtpHeaderLength); + + ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet); + std::vector> fec_packets = + ulpfec_generator_.GetFecPackets(); + EXPECT_FALSE(fec_packets.empty()); // Ensure that the RED header is placed correctly, i.e. the correct // RTP header length was used in the RED packet creation. - const uint16_t seq_num = packet_generator_.NextPacketSeqNum(); - std::vector> red_packets = - ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType, - seq_num); - for (const auto& red_packet : red_packets) { - EXPECT_EQ(kFecPayloadType, red_packet->data()[kShortRtpHeaderLength]); + uint16_t seq_num = packet_generator_.NextPacketSeqNum(); + for (const auto& fec_packet : fec_packets) { + fec_packet->SetSequenceNumber(seq_num++); + EXPECT_EQ(kFecPayloadType, fec_packet->data()[kShortRtpHeaderLength]); } } diff --git a/modules/rtp_rtcp/source/video_fec_generator.h b/modules/rtp_rtcp/source/video_fec_generator.h new file mode 100644 index 0000000000..3731449b5c --- /dev/null +++ b/modules/rtp_rtcp/source/video_fec_generator.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_ +#define MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_ + +#include +#include + +#include "api/units/data_rate.h" +#include "modules/include/module_fec_types.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" + +namespace webrtc { + +class VideoFecGenerator { + public: + VideoFecGenerator() = default; + virtual ~VideoFecGenerator() = default; + + enum class FecType { kFlexFec, kUlpFec }; + virtual FecType GetFecType() const = 0; + // Returns the SSRC used for FEC packets (i.e. FlexFec SSRC). + virtual absl::optional FecSsrc() = 0; + // Returns the overhead, in bytes per packet, for FEC (and possibly RED). + virtual size_t MaxPacketOverhead() const = 0; + // Current rate of FEC packets generated, including all RTP-level headers. + virtual DataRate CurrentFecRate() const = 0; + // Set FEC rates, max frames before FEC is sent, and type of FEC masks. + virtual void SetProtectionParameters( + const FecProtectionParams& delta_params, + const FecProtectionParams& key_params) = 0; + // Called on new media packet to be protected. The generator may choose + // to generate FEC packets at this time, if so they will be stored in an + // internal buffer. + virtual void AddPacketAndGenerateFec(const RtpPacketToSend& packet) = 0; + // Get (and remove) and FEC packets pending in the generator. These packets + // will lack sequence numbers, that needs to be set externally. + // TODO(bugs.webrtc.org/11340): Actually FlexFec sets seq#, fix that! + virtual std::vector> GetFecPackets() = 0; +}; + +} // namespace webrtc +#endif // MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_ diff --git a/test/fuzzers/BUILD.gn b/test/fuzzers/BUILD.gn index 4caac345cc..123e54840b 100644 --- a/test/fuzzers/BUILD.gn +++ b/test/fuzzers/BUILD.gn @@ -149,6 +149,7 @@ webrtc_fuzzer_test("ulpfec_generator_fuzzer") { "../../modules/rtp_rtcp:rtp_rtcp_format", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", + "../../system_wrappers", ] } diff --git a/test/fuzzers/flexfec_sender_fuzzer.cc b/test/fuzzers/flexfec_sender_fuzzer.cc index 4882f7df51..8ddd1c0fe0 100644 --- a/test/fuzzers/flexfec_sender_fuzzer.cc +++ b/test/fuzzers/flexfec_sender_fuzzer.cc @@ -41,7 +41,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) { FecProtectionParams params = { data[i++], static_cast(data[i++] % 100), data[i++] <= 127 ? kFecMaskRandom : kFecMaskBursty}; - sender.SetFecParameters(params); + sender.SetProtectionParameters(params, params); uint16_t seq_num = data[i++]; while (i + 1 < size) { @@ -59,11 +59,8 @@ void FuzzOneInput(const uint8_t* data, size_t size) { RtpPacketToSend rtp_packet(nullptr); if (!rtp_packet.Parse(packet.get(), kRtpHeaderSize + payload_size)) break; - sender.AddRtpPacketAndGenerateFec(rtp_packet); - if (sender.FecAvailable()) { - std::vector> fec_packets = - sender.GetFecPackets(); - } + sender.AddPacketAndGenerateFec(rtp_packet); + sender.GetFecPackets(); } } diff --git a/test/fuzzers/ulpfec_generator_fuzzer.cc b/test/fuzzers/ulpfec_generator_fuzzer.cc index 306f7a0da9..9426ef0ad3 100644 --- a/test/fuzzers/ulpfec_generator_fuzzer.cc +++ b/test/fuzzers/ulpfec_generator_fuzzer.cc @@ -16,6 +16,7 @@ #include "modules/rtp_rtcp/source/ulpfec_generator.h" #include "rtc_base/checks.h" #include "rtc_base/copy_on_write_buffer.h" +#include "system_wrappers/include/clock.h" namespace webrtc { @@ -25,13 +26,14 @@ constexpr uint8_t kRedPayloadType = 97; } // namespace void FuzzOneInput(const uint8_t* data, size_t size) { - UlpfecGenerator generator; + SimulatedClock clock(1); + UlpfecGenerator generator(kRedPayloadType, kFecPayloadType, &clock); size_t i = 0; if (size < 4) return; FecProtectionParams params = { data[i++] % 128, static_cast(data[i++] % 10), kFecMaskBursty}; - generator.SetFecParameters(params); + generator.SetProtectionParameters(params, params); uint16_t seq_num = data[i++]; uint16_t prev_seq_num = 0; while (i + 3 < size) { @@ -41,6 +43,9 @@ void FuzzOneInput(const uint8_t* data, size_t size) { break; rtc::CopyOnWriteBuffer packet(&data[i], payload_size + rtp_header_length); packet.EnsureCapacity(IP_PACKET_SIZE); + // Write a valid parsable header (version = 2, no padding, no extensions, + // no CSRCs). + ByteWriter::WriteBigEndian(&packet[0], 2 << 6); // Make sure sequence numbers are increasing. ByteWriter::WriteBigEndian(&packet[2], seq_num++); i += payload_size + rtp_header_length; @@ -51,16 +56,15 @@ void FuzzOneInput(const uint8_t* data, size_t size) { // number became out of order. if (protect && IsNewerSequenceNumber(seq_num, prev_seq_num) && seq_num < prev_seq_num + kUlpfecMaxMediaPackets) { - generator.AddRtpPacketAndGenerateFec(packet, rtp_header_length); + RtpPacketToSend rtp_packet(nullptr); + // Check that we actually have a parsable packet, we want to fuzz FEC + // logic, not RTP header parsing. + RTC_CHECK(rtp_packet.Parse(packet)); + generator.AddPacketAndGenerateFec(rtp_packet); prev_seq_num = seq_num; } - const size_t num_fec_packets = generator.NumAvailableFecPackets(); - if (num_fec_packets > 0) { - std::vector> fec_packets = - generator.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType, - 100); - RTC_CHECK_EQ(num_fec_packets, fec_packets.size()); - } + + generator.GetFecPackets(); } } } // namespace webrtc