admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reland "Reland "Refactors UlpFec and FlexFec to use a common interface.""

Browse Source 
This is a reland of 49734dc0faa69616a58a1a95c7fc61a4610793cf

Patchset 2 contains a fix for the fuzzer set up. Since we now parse
an RtpPacket out of the fuzzer data, the header needs to be correct,
otherwise we fail before even reaching the FEC code that we actually
want to test.

Bug: webrtc:11340, chromium:1052323, chromium:1055974
TBR=stefan@webrtc.org

Original change's description:
> Reland "Refactors UlpFec and FlexFec to use a common interface."
>
> This is a reland of 11af1d7444fd7438766b7bc52cbd64752d72e32e
>
> Original change's description:
> > Refactors UlpFec and FlexFec to use a common interface.
> >
> > The new VideoFecGenerator is now injected into RtpSenderVideo,
> > and generalizes the usage.
> > This also prepares for being able to genera FEC in the RTP egress
> > module.
> >
> > Bug: webrtc:11340
> > Change-Id: I8aa873129b2fb4131eb3399ee88f6ea2747155a3
> > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168347
> > Reviewed-by: Stefan Holmer <stefan@webrtc.org>
> > Reviewed-by: Sebastian Jansson <srte@webrtc.org>
> > Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> > Commit-Queue: Erik Språng <sprang@webrtc.org>
> > Cr-Commit-Position: refs/heads/master@{#30515}
>
> Bug: webrtc:11340, chromium:1052323
> Change-Id: Id646047365f1c46cca9e6f3e8eefa5151207b4a0
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168608
> Commit-Queue: Erik Språng <sprang@webrtc.org>
> Reviewed-by: Stefan Holmer <stefan@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#30593}

Bug: webrtc:11340, chromium:1052323
Change-Id: Ib8925f44e2edfcfeadc95c845c3bfc23822604ed
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169222
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30724}
This commit is contained in:
Erik Språng 2020-03-05 10:14:04 +01:00 committed by Commit Bot
parent 269d68f521
commit f87536c9de
23 changed files with 578 additions and 563 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
call/rtp_video_sender.cc
View File

@ -36,9 +36,13 @@ namespace webrtc {
namespace webrtc_internal_rtp_video_sender {
RtpStreamSender::RtpStreamSender(std::unique_ptr<RtpRtcp> rtp_rtcp,
std::unique_ptr<RTPSenderVideo> sender_video)
: rtp_rtcp(std::move(rtp_rtcp)), sender_video(std::move(sender_video)) {}
RtpStreamSender::RtpStreamSender(
std::unique_ptr<RtpRtcp> rtp_rtcp,
std::unique_ptr<RTPSenderVideo> sender_video,
std::unique_ptr<VideoFecGenerator> fec_generator)
: rtp_rtcp(std::move(rtp_rtcp)),
sender_video(std::move(sender_video)),
fec_generator(std::move(fec_generator)) {}
RtpStreamSender::~RtpStreamSender() = default;
@ -113,6 +117,67 @@ bool ShouldDisableRedAndUlpfec(bool flexfec_enabled,
return should_disable_red_and_ulpfec;
}
// TODO(brandtr): Update this function when we support multistream protection.
std::unique_ptr<VideoFecGenerator> MaybeCreateFecGenerator(
Clock* clock,
const RtpConfig& rtp,
const std::map<uint32_t, RtpState>& suspended_ssrcs,
int simulcast_index) {
// If flexfec is configured that takes priority.
if (rtp.flexfec.payload_type >= 0) {
RTC_DCHECK_GE(rtp.flexfec.payload_type, 0);
RTC_DCHECK_LE(rtp.flexfec.payload_type, 127);
if (rtp.flexfec.ssrc == 0) {
RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. "
"Therefore disabling FlexFEC.";
return nullptr;
}
if (rtp.flexfec.protected_media_ssrcs.empty()) {
RTC_LOG(LS_WARNING)
<< "FlexFEC is enabled, but no protected media SSRC given. "
"Therefore disabling FlexFEC.";
return nullptr;
}
if (rtp.flexfec.protected_media_ssrcs.size() > 1) {
RTC_LOG(LS_WARNING)
<< "The supplied FlexfecConfig contained multiple protected "
"media streams, but our implementation currently only "
"supports protecting a single media stream. "
"To avoid confusion, disabling FlexFEC completely.";
return nullptr;
}
if (absl::c_find(rtp.flexfec.protected_media_ssrcs,
rtp.ssrcs[simulcast_index]) ==
rtp.flexfec.protected_media_ssrcs.end()) {
// Media SSRC not among flexfec protected SSRCs.
return nullptr;
}
const RtpState* rtp_state = nullptr;
auto it = suspended_ssrcs.find(rtp.flexfec.ssrc);
if (it != suspended_ssrcs.end()) {
rtp_state = &it->second;
}
RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size());
return std::make_unique<FlexfecSender>(
rtp.flexfec.payload_type, rtp.flexfec.ssrc,
rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions,
RTPSender::FecExtensionSizes(), rtp_state, clock);
} else if (rtp.ulpfec.red_payload_type >= 0 &&
rtp.ulpfec.ulpfec_payload_type >= 0 &&
!ShouldDisableRedAndUlpfec(/*flexfec_enabled=*/false, rtp)) {
// Flexfec not configured, but ulpfec is and is not disabled.
return std::make_unique<UlpfecGenerator>(
rtp.ulpfec.red_payload_type, rtp.ulpfec.ulpfec_payload_type, clock);
}
// Not a single FEC is given.
return nullptr;
}
std::vector<RtpStreamSender> CreateRtpStreamSenders(
Clock* clock,
const RtpConfig& rtp_config,
@ -121,7 +186,7 @@ std::vector<RtpStreamSender> CreateRtpStreamSenders(
Transport* send_transport,
RtcpBandwidthObserver* bandwidth_callback,
RtpTransportControllerSendInterface* transport,
FlexfecSender* flexfec_sender,
const std::map<uint32_t, RtpState>& suspended_ssrcs,
RtcEventLog* event_log,
RateLimiter* retransmission_rate_limiter,
OverheadObserver* overhead_observer,
@ -161,18 +226,17 @@ std::vector<RtpStreamSender> CreateRtpStreamSenders(
configuration.rtcp_report_interval_ms = rtcp_report_interval_ms;
std::vector<RtpStreamSender> rtp_streams;
const std::vector<uint32_t>& flexfec_protected_ssrcs =
rtp_config.flexfec.protected_media_ssrcs;
RTC_DCHECK(rtp_config.rtx.ssrcs.empty() ||
rtp_config.rtx.ssrcs.size() == rtp_config.rtx.ssrcs.size());
for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) {
RTPSenderVideo::Config video_config;
configuration.local_media_ssrc = rtp_config.ssrcs[i];
bool enable_flexfec = flexfec_sender != nullptr &&
std::find(flexfec_protected_ssrcs.begin(),
flexfec_protected_ssrcs.end(),
configuration.local_media_ssrc) !=
flexfec_protected_ssrcs.end();
configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr;
std::unique_ptr<VideoFecGenerator> fec_generator =
MaybeCreateFecGenerator(clock, rtp_config, suspended_ssrcs, i);
configuration.fec_generator = fec_generator.get();
video_config.fec_generator = fec_generator.get();
if (rtp_config.rtx.ssrcs.size() > i) {
configuration.rtx_send_ssrc = rtp_config.rtx.ssrcs[i];
@ -188,76 +252,31 @@ std::vector<RtpStreamSender> CreateRtpStreamSenders(
rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize);
FieldTrialBasedConfig field_trial_config;
RTPSenderVideo::Config video_config;
video_config.clock = configuration.clock;
video_config.rtp_sender = rtp_rtcp->RtpSender();
video_config.flexfec_sender = configuration.flexfec_sender;
video_config.frame_encryptor = frame_encryptor;
video_config.require_frame_encryption =
crypto_options.sframe.require_frame_encryption;
video_config.enable_retransmit_all_layers = false;
video_config.field_trials = &field_trial_config;
const bool using_flexfec =
fec_generator &&
fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec;
const bool should_disable_red_and_ulpfec =
ShouldDisableRedAndUlpfec(enable_flexfec, rtp_config);
if (rtp_config.ulpfec.red_payload_type != -1 &&
!should_disable_red_and_ulpfec) {
ShouldDisableRedAndUlpfec(using_flexfec, rtp_config);
if (!should_disable_red_and_ulpfec &&
rtp_config.ulpfec.red_payload_type != -1) {
video_config.red_payload_type = rtp_config.ulpfec.red_payload_type;
}
if (rtp_config.ulpfec.ulpfec_payload_type != -1 &&
!should_disable_red_and_ulpfec) {
video_config.ulpfec_payload_type = rtp_config.ulpfec.ulpfec_payload_type;
}
video_config.frame_transformer = std::move(frame_transformer);
auto sender_video = std::make_unique<RTPSenderVideo>(video_config);
rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video));
rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video),
std::move(fec_generator));
}
return rtp_streams;
}
// TODO(brandtr): Update this function when we support multistream protection.
std::unique_ptr<FlexfecSender> MaybeCreateFlexfecSender(
Clock* clock,
const RtpConfig& rtp,
const std::map<uint32_t, RtpState>& suspended_ssrcs) {
if (rtp.flexfec.payload_type < 0) {
return nullptr;
}
RTC_DCHECK_GE(rtp.flexfec.payload_type, 0);
RTC_DCHECK_LE(rtp.flexfec.payload_type, 127);
if (rtp.flexfec.ssrc == 0) {
RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. "
"Therefore disabling FlexFEC.";
return nullptr;
}
if (rtp.flexfec.protected_media_ssrcs.empty()) {
RTC_LOG(LS_WARNING)
<< "FlexFEC is enabled, but no protected media SSRC given. "
"Therefore disabling FlexFEC.";
return nullptr;
}
if (rtp.flexfec.protected_media_ssrcs.size() > 1) {
RTC_LOG(LS_WARNING)
<< "The supplied FlexfecConfig contained multiple protected "
"media streams, but our implementation currently only "
"supports protecting a single media stream. "
"To avoid confusion, disabling FlexFEC completely.";
return nullptr;
}
const RtpState* rtp_state = nullptr;
auto it = suspended_ssrcs.find(rtp.flexfec.ssrc);
if (it != suspended_ssrcs.end()) {
rtp_state = &it->second;
}
RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size());
return std::make_unique<FlexfecSender>(
rtp.flexfec.payload_type, rtp.flexfec.ssrc,
rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions,
RTPSender::FecExtensionSizes(), rtp_state, clock);
}
DataRate CalculateOverheadRate(DataRate data_rate,
DataSize packet_size,
DataSize overhead_per_packet) {
@ -305,8 +324,6 @@ RtpVideoSender::RtpVideoSender(
active_(false),
module_process_thread_(nullptr),
suspended_ssrcs_(std::move(suspended_ssrcs)),
flexfec_sender_(
MaybeCreateFlexfecSender(clock, rtp_config, suspended_ssrcs_)),
fec_controller_(std::move(fec_controller)),
fec_allowed_(true),
rtp_streams_(CreateRtpStreamSenders(clock,
@ -316,7 +333,7 @@ RtpVideoSender::RtpVideoSender(
send_transport,
transport->GetBandwidthObserver(),
transport,
flexfec_sender_.get(),
suspended_ssrcs_,
event_log,
retransmission_limiter,
this,
@ -379,6 +396,7 @@ RtpVideoSender::RtpVideoSender(
}
}
bool fec_enabled = false;
for (const RtpStreamSender& stream : rtp_streams_) {
// Simulcast has one module for each layer. Set the CNAME on all modules.
stream.rtp_rtcp->SetCNAME(rtp_config_.c_name.c_str());
@ -388,10 +406,13 @@ RtpVideoSender::RtpVideoSender(
stream.rtp_rtcp->SetMaxRtpPacketSize(rtp_config_.max_packet_size);
stream.rtp_rtcp->RegisterSendPayloadFrequency(rtp_config_.payload_type,
kVideoPayloadTypeFrequency);
if (stream.fec_generator != nullptr) {
fec_enabled = true;
}
}
// Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic,
// so enable that logic if either of those FEC schemes are enabled.
fec_controller_->SetProtectionMethod(FecEnabled(), NackEnabled());
fec_controller_->SetProtectionMethod(fec_enabled, NackEnabled());
fec_controller_->SetProtectionCallback(this);
// Signal congestion controller this object is ready for OnPacket* callbacks.
@ -559,14 +580,6 @@ void RtpVideoSender::OnBitrateAllocationUpdated(
}
}
bool RtpVideoSender::FecEnabled() const {
const bool flexfec_enabled = (flexfec_sender_ != nullptr);
const bool ulpfec_enabled =
!webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment") &&
(rtp_config_.ulpfec.ulpfec_payload_type >= 0);
return flexfec_enabled || ulpfec_enabled;
}
bool RtpVideoSender::NackEnabled() const {
const bool nack_enabled = rtp_config_.nack.rtp_history_ms > 0;
return nack_enabled;
@ -661,6 +674,14 @@ std::map<uint32_t, RtpState> RtpVideoSender::GetRtpStates() const {
uint32_t ssrc = rtp_config_.ssrcs[i];
RTC_DCHECK_EQ(ssrc, rtp_streams_[i].rtp_rtcp->SSRC());
rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtpState();
VideoFecGenerator* fec_generator = rtp_streams_[i].fec_generator.get();
if (fec_generator &&
fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec) {
auto* flexfec_sender = static_cast<FlexfecSender*>(fec_generator);
uint32_t ssrc = rtp_config_.flexfec.ssrc;
rtp_states[ssrc] = flexfec_sender->GetRtpState();
}
}
for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) {
@ -668,11 +689,6 @@ std::map<uint32_t, RtpState> RtpVideoSender::GetRtpStates() const {
rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtxState();
}
if (flexfec_sender_) {
uint32_t ssrc = rtp_config_.flexfec.ssrc;
rtp_states[ssrc] = flexfec_sender_->GetRtpState();
}
return rtp_states;
}

7
call/rtp_video_sender.h
View File

@ -51,7 +51,8 @@ namespace webrtc_internal_rtp_video_sender {
// RtpVideoSender.
struct RtpStreamSender {
RtpStreamSender(std::unique_ptr<RtpRtcp> rtp_rtcp,
std::unique_ptr<RTPSenderVideo> sender_video);
std::unique_ptr<RTPSenderVideo> sender_video,
std::unique_ptr<VideoFecGenerator> fec_generator);
~RtpStreamSender();
RtpStreamSender(RtpStreamSender&&) = default;
@ -60,6 +61,7 @@ struct RtpStreamSender {
// Note: Needs pointer stability.
std::unique_ptr<RtpRtcp> rtp_rtcp;
std::unique_ptr<RTPSenderVideo> sender_video;
std::unique_ptr<VideoFecGenerator> fec_generator;
};
} // namespace webrtc_internal_rtp_video_sender
@ -155,7 +157,6 @@ class RtpVideoSender : public RtpVideoSenderInterface,
void ConfigureProtection();
void ConfigureSsrcs();
void ConfigureRids();
bool FecEnabled() const;
bool NackEnabled() const;
uint32_t GetPacketizationOverheadRate() const;
@ -173,8 +174,6 @@ class RtpVideoSender : public RtpVideoSenderInterface,
rtc::ThreadChecker module_process_thread_checker_;
std::map<uint32_t, RtpState> suspended_ssrcs_;
std::unique_ptr<FlexfecSender> flexfec_sender_;
const std::unique_ptr<FecController> fec_controller_;
bool fec_allowed_ RTC_GUARDED_BY(crit_);

6
modules/include/module_fec_types.h
View File

@ -24,9 +24,9 @@ enum FecMaskType {
// Struct containing forward error correction settings.
struct FecProtectionParams {
int fec_rate;
int max_fec_frames;
FecMaskType fec_mask_type;
int fec_rate = 0;
int max_fec_frames = 0;
FecMaskType fec_mask_type = FecMaskType::kFecMaskRandom;
};
} // namespace webrtc

1
modules/rtp_rtcp/BUILD.gn
View File

@ -214,6 +214,7 @@ rtc_library("rtp_rtcp") {
"source/ulpfec_header_reader_writer.h",
"source/ulpfec_receiver_impl.cc",
"source/ulpfec_receiver_impl.h",
"source/video_fec_generator.h",
"source/video_rtp_depacketizer.h",
"source/video_rtp_depacketizer_av1.cc",
"source/video_rtp_depacketizer_av1.h",

27
modules/rtp_rtcp/include/flexfec_sender.h
View File

@ -21,7 +21,9 @@
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/rtp_header_extension_size.h"
#include "modules/rtp_rtcp/source/ulpfec_generator.h"
#include "modules/rtp_rtcp/source/video_fec_generator.h"
#include "rtc_base/random.h"
#include "rtc_base/rate_statistics.h"
namespace webrtc {
@ -31,7 +33,7 @@ class RtpPacketToSend;
// Note that this class is not thread safe, and thus requires external
// synchronization. Currently, this is done using the lock in PayloadRouter.
class FlexfecSender {
class FlexfecSender : public VideoFecGenerator {
public:
FlexfecSender(int payload_type,
uint32_t ssrc,
@ -43,26 +45,28 @@ class FlexfecSender {
Clock* clock);
~FlexfecSender();
uint32_t ssrc() const { return ssrc_; }
FecType GetFecType() const override {
return VideoFecGenerator::FecType::kFlexFec;
}
absl::optional<uint32_t> FecSsrc() override { return ssrc_; }
// Sets the FEC rate, max frames sent before FEC packets are sent,
// and what type of generator matrices are used.
void SetFecParameters(const FecProtectionParams& params);
void SetProtectionParameters(const FecProtectionParams& delta_params,
const FecProtectionParams& key_params) override;
// Adds a media packet to the internal buffer. When enough media packets
// have been added, the FEC packets are generated and stored internally.
// These FEC packets are then obtained by calling GetFecPackets().
// Returns true if the media packet was successfully added.
bool AddRtpPacketAndGenerateFec(const RtpPacketToSend& packet);
// Returns true if there are generated FEC packets available.
bool FecAvailable() const;
void AddPacketAndGenerateFec(const RtpPacketToSend& packet) override;
// Returns generated FlexFEC packets.
std::vector<std::unique_ptr<RtpPacketToSend>> GetFecPackets();
std::vector<std::unique_ptr<RtpPacketToSend>> GetFecPackets() override;
// Returns the overhead, per packet, for FlexFEC.
size_t MaxPacketOverhead() const;
size_t MaxPacketOverhead() const override;
DataRate CurrentFecRate() const override;
// Only called on the VideoSendStream queue, after operation has shut down.
RtpState GetRtpState();
@ -87,6 +91,9 @@ class FlexfecSender {
UlpfecGenerator ulpfec_generator_;
const RtpHeaderExtensionMap rtp_header_extension_map_;
const size_t header_extensions_size_;
rtc::CriticalSection crit_;
RateStatistics fec_bitrate_ RTC_GUARDED_BY(crit_);
};
} // namespace webrtc

8
modules/rtp_rtcp/include/rtp_rtcp.h
View File

@ -24,13 +24,13 @@
#include "api/transport/webrtc_key_value_config.h"
#include "api/video/video_bitrate_allocation.h"
#include "modules/include/module.h"
#include "modules/rtp_rtcp/include/flexfec_sender.h"
#include "modules/rtp_rtcp/include/receive_statistics.h"
#include "modules/rtp_rtcp/include/report_block_data.h"
#include "modules/rtp_rtcp/include/rtp_packet_sender.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h"
#include "modules/rtp_rtcp/source/video_fec_generator.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/deprecation.h"
@ -94,9 +94,9 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface {
// Spread any bursts of packets into smaller bursts to minimize packet loss.
RtpPacketSender* paced_sender = nullptr;
// Generate FlexFEC packets.
// TODO(brandtr): Remove when FlexfecSender is wired up to PacedSender.
FlexfecSender* flexfec_sender = nullptr;
// Generates FEC packets.
// TODO(sprang): Wire up to RtpSenderEgress.
VideoFecGenerator* fec_generator = nullptr;
BitrateStatisticsObserver* send_bitrate_observer = nullptr;
SendSideDelayObserver* send_side_delay_observer = nullptr;

39
modules/rtp_rtcp/source/flexfec_sender.cc
View File

@ -91,11 +91,13 @@ FlexfecSender::FlexfecSender(
seq_num_(rtp_state ? rtp_state->sequence_number
: random_.Rand(1, kMaxInitRtpSeqNumber)),
ulpfec_generator_(
ForwardErrorCorrection::CreateFlexfec(ssrc, protected_media_ssrc)),
ForwardErrorCorrection::CreateFlexfec(ssrc, protected_media_ssrc),
clock_),
rtp_header_extension_map_(
RegisterSupportedExtensions(rtp_header_extensions)),
header_extensions_size_(
RtpHeaderExtensionSize(extension_sizes, rtp_header_extension_map_)) {
RtpHeaderExtensionSize(extension_sizes, rtp_header_extension_map_)),
fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {
// This object should not have been instantiated if FlexFEC is disabled.
RTC_DCHECK_GE(payload_type, 0);
RTC_DCHECK_LE(payload_type, 127);
@ -105,30 +107,30 @@ FlexfecSender::~FlexfecSender() = default;
// We are reusing the implementation from UlpfecGenerator for SetFecParameters,
// AddRtpPacketAndGenerateFec, and FecAvailable.
void FlexfecSender::SetFecParameters(const FecProtectionParams& params) {
ulpfec_generator_.SetFecParameters(params);
void FlexfecSender::SetProtectionParameters(
const FecProtectionParams& delta_params,
const FecProtectionParams& key_params) {
ulpfec_generator_.SetProtectionParameters(delta_params, key_params);
}
bool FlexfecSender::AddRtpPacketAndGenerateFec(const RtpPacketToSend& packet) {
void FlexfecSender::AddPacketAndGenerateFec(const RtpPacketToSend& packet) {
// TODO(brandtr): Generalize this SSRC check when we support multistream
// protection.
RTC_DCHECK_EQ(packet.Ssrc(), protected_media_ssrc_);
return ulpfec_generator_.AddRtpPacketAndGenerateFec(
packet.Buffer(), packet.headers_size()) == 0;
}
bool FlexfecSender::FecAvailable() const {
return ulpfec_generator_.FecAvailable();
ulpfec_generator_.AddPacketAndGenerateFec(packet);
}
std::vector<std::unique_ptr<RtpPacketToSend>> FlexfecSender::GetFecPackets() {
RTC_CHECK_RUNS_SERIALIZED(&ulpfec_generator_.race_checker_);
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets_to_send;
fec_packets_to_send.reserve(ulpfec_generator_.generated_fec_packets_.size());
size_t total_fec_data_bytes = 0;
for (const auto* fec_packet : ulpfec_generator_.generated_fec_packets_) {
std::unique_ptr<RtpPacketToSend> fec_packet_to_send(
new RtpPacketToSend(&rtp_header_extension_map_));
fec_packet_to_send->set_packet_type(
RtpPacketMediaType::kForwardErrorCorrection);
fec_packet_to_send->set_allow_retransmission(false);
// RTP header.
fec_packet_to_send->SetMarker(false);
@ -157,9 +159,13 @@ std::vector<std::unique_ptr<RtpPacketToSend>> FlexfecSender::GetFecPackets() {
fec_packet_to_send->AllocatePayload(fec_packet->data.size());
memcpy(payload, fec_packet->data.cdata(), fec_packet->data.size());
total_fec_data_bytes += fec_packet_to_send->size();
fec_packets_to_send.push_back(std::move(fec_packet_to_send));
}
ulpfec_generator_.ResetState();
if (!fec_packets_to_send.empty()) {
ulpfec_generator_.ResetState();
}
int64_t now_ms = clock_->TimeInMilliseconds();
if (!fec_packets_to_send.empty() &&
@ -170,6 +176,9 @@ std::vector<std::unique_ptr<RtpPacketToSend>> FlexfecSender::GetFecPackets() {
last_generated_packet_ms_ = now_ms;
}
rtc::CritScope cs(&crit_);
fec_bitrate_.Update(total_fec_data_bytes, now_ms);
return fec_packets_to_send;
}
@ -178,6 +187,12 @@ size_t FlexfecSender::MaxPacketOverhead() const {
return header_extensions_size_ + kFlexfecMaxHeaderSize;
}
DataRate FlexfecSender::CurrentFecRate() const {
rtc::CritScope cs(&crit_);
return DataRate::BitsPerSec(
fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0));
}
RtpState FlexfecSender::GetRtpState() {
RtpState rtp_state;
rtp_state.sequence_number = seq_num_;

27
modules/rtp_rtcp/source/flexfec_sender_unittest.cc
View File

@ -55,7 +55,7 @@ std::unique_ptr<RtpPacketToSend> GenerateSingleFlexfecPacket(
params.fec_mask_type = kFecMaskRandom;
constexpr size_t kNumPackets = 4;
sender->SetFecParameters(params);
sender->SetProtectionParameters(params, params);
AugmentedPacketGenerator packet_generator(kMediaSsrc);
packet_generator.NewFrame(kNumPackets);
for (size_t i = 0; i < kNumPackets; ++i) {
@ -63,13 +63,12 @@ std::unique_ptr<RtpPacketToSend> GenerateSingleFlexfecPacket(
packet_generator.NextPacket(i, kPayloadLength);
RtpPacketToSend rtp_packet(nullptr); // No header extensions.
rtp_packet.Parse(packet->data);
EXPECT_TRUE(sender->AddRtpPacketAndGenerateFec(rtp_packet));
sender->AddPacketAndGenerateFec(rtp_packet);
}
EXPECT_TRUE(sender->FecAvailable());
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
sender->GetFecPackets();
EXPECT_FALSE(sender->FecAvailable());
EXPECT_EQ(1U, fec_packets.size());
EXPECT_TRUE(sender->GetFecPackets().empty());
return std::move(fec_packets.front());
}
@ -82,7 +81,7 @@ TEST(FlexfecSenderTest, Ssrc) {
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
nullptr /* rtp_state */, &clock);
EXPECT_EQ(kFlexfecSsrc, sender.ssrc());
EXPECT_EQ(kFlexfecSsrc, sender.FecSsrc());
}
TEST(FlexfecSenderTest, NoFecAvailableBeforeMediaAdded) {
@ -91,9 +90,7 @@ TEST(FlexfecSenderTest, NoFecAvailableBeforeMediaAdded) {
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
nullptr /* rtp_state */, &clock);
EXPECT_FALSE(sender.FecAvailable());
auto fec_packets = sender.GetFecPackets();
EXPECT_EQ(0U, fec_packets.size());
EXPECT_TRUE(sender.GetFecPackets().empty());
}
TEST(FlexfecSenderTest, ProtectOneFrameWithOneFecPacket) {
@ -124,7 +121,7 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithOneFecPacket) {
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc, kNoMid,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
nullptr /* rtp_state */, &clock);
sender.SetFecParameters(params);
sender.SetProtectionParameters(params, params);
AugmentedPacketGenerator packet_generator(kMediaSsrc);
for (size_t i = 0; i < kNumFrames; ++i) {
@ -134,14 +131,13 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithOneFecPacket) {
packet_generator.NextPacket(i, kPayloadLength);
RtpPacketToSend rtp_packet(nullptr);
rtp_packet.Parse(packet->data);
EXPECT_TRUE(sender.AddRtpPacketAndGenerateFec(rtp_packet));
sender.AddPacketAndGenerateFec(rtp_packet);
}
}
EXPECT_TRUE(sender.FecAvailable());
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
sender.GetFecPackets();
EXPECT_FALSE(sender.FecAvailable());
ASSERT_EQ(1U, fec_packets.size());
EXPECT_TRUE(sender.GetFecPackets().empty());
RtpPacketToSend* fec_packet = fec_packets.front().get();
EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size());
@ -164,7 +160,7 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithTwoFecPackets) {
FlexfecSender sender(kFlexfecPayloadType, kFlexfecSsrc, kMediaSsrc, kNoMid,
kNoRtpHeaderExtensions, kNoRtpHeaderExtensionSizes,
nullptr /* rtp_state */, &clock);
sender.SetFecParameters(params);
sender.SetProtectionParameters(params, params);
AugmentedPacketGenerator packet_generator(kMediaSsrc);
for (size_t i = 0; i < kNumFrames; ++i) {
@ -174,13 +170,12 @@ TEST(FlexfecSenderTest, ProtectTwoFramesWithTwoFecPackets) {
packet_generator.NextPacket(i, kPayloadLength);
RtpPacketToSend rtp_packet(nullptr);
rtp_packet.Parse(packet->data);
EXPECT_TRUE(sender.AddRtpPacketAndGenerateFec(rtp_packet));
sender.AddPacketAndGenerateFec(rtp_packet);
}
EXPECT_TRUE(sender.FecAvailable());
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
sender.GetFecPackets();
EXPECT_FALSE(sender.FecAvailable());
ASSERT_EQ(1U, fec_packets.size());
EXPECT_TRUE(sender.GetFecPackets().empty());
RtpPacketToSend* fec_packet = fec_packets.front().get();
EXPECT_EQ(kRtpHeaderSize, fec_packet->headers_size());

7
modules/rtp_rtcp/source/rtcp_receiver.cc
View File

@ -72,8 +72,11 @@ std::set<uint32_t> GetRegisteredSsrcs(const RtpRtcp::Configuration& config) {
if (config.rtx_send_ssrc) {
ssrcs.insert(*config.rtx_send_ssrc);
}
if (config.flexfec_sender) {
ssrcs.insert(config.flexfec_sender->ssrc());
if (config.fec_generator) {
absl::optional<uint32_t> flexfec_ssrc = config.fec_generator->FecSsrc();
if (flexfec_ssrc) {
ssrcs.insert(*flexfec_ssrc);
}
}
return ssrcs;
}

7
modules/rtp_rtcp/source/rtp_packet_to_send.h
View File

@ -98,12 +98,16 @@ class RtpPacketToSend : public RtpPacket {
VideoTimingExtension::kNetwork2TimestampDeltaOffset);
}
// Indicates if packet is the first packet of a video frame.
void set_first_packet_of_frame(bool is_first_packet) {
is_first_packet_of_frame_ = is_first_packet;
}
bool is_first_packet_of_frame() const { return is_first_packet_of_frame_; }
// Indicates if packet contains payload for a video key-frame.
void set_is_key_frame(bool is_key_frame) { is_key_frame_ = is_key_frame; }
bool is_key_frame() const { return is_key_frame_; }
private:
int64_t capture_time_ms_ = 0;
absl::optional<RtpPacketMediaType> packet_type_;
@ -111,6 +115,7 @@ class RtpPacketToSend : public RtpPacket {
absl::optional<uint16_t> retransmitted_sequence_number_;
std::vector<uint8_t> application_data_;
bool is_first_packet_of_frame_ = false;
bool is_key_frame_ = false;
};
} // namespace webrtc

5
modules/rtp_rtcp/source/rtp_sender.cc
View File

@ -102,9 +102,8 @@ RTPSender::RTPSender(const RtpRtcp::Configuration& config,
audio_configured_(config.audio),
ssrc_(config.local_media_ssrc),
rtx_ssrc_(config.rtx_send_ssrc),
flexfec_ssrc_(config.flexfec_sender
? absl::make_optional(config.flexfec_sender->ssrc())
: absl::nullopt),
flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc()
: absl::nullopt),
packet_history_(packet_history),
paced_sender_(packet_sender),
sending_media_(true), // Default to sending media.

5
modules/rtp_rtcp/source/rtp_sender_egress.cc
View File

@ -57,9 +57,8 @@ RtpSenderEgress::RtpSenderEgress(const RtpRtcp::Configuration& config,
RtpPacketHistory* packet_history)
: ssrc_(config.local_media_ssrc),
rtx_ssrc_(config.rtx_send_ssrc),
flexfec_ssrc_(config.flexfec_sender
? absl::make_optional(config.flexfec_sender->ssrc())
: absl::nullopt),
flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc()
: absl::nullopt),
populate_network2_timestamp_(config.populate_network2_timestamp),
send_side_bwe_with_overhead_(
IsEnabled("WebRTC-SendSideBwe-WithOverhead", config.field_trials)),

20
modules/rtp_rtcp/source/rtp_sender_unittest.cc
View File

@ -272,7 +272,7 @@ class RtpSenderTest : public ::testing::TestWithParam<TestConfig> {
config.outgoing_transport = &transport_;
config.local_media_ssrc = kSsrc;
config.rtx_send_ssrc = kRtxSsrc;
config.flexfec_sender = &flexfec_sender_;
config.fec_generator = &flexfec_sender_;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
config.retransmission_rate_limiter = &retransmission_rate_limiter_;
@ -1225,7 +1225,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) {
config.outgoing_transport = &transport_;
config.paced_sender = &mock_paced_sender_;
config.local_media_ssrc = kSsrc;
config.flexfec_sender = &flexfec_sender_;
config.fec_generator = &flexfec_sender_;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
config.retransmission_rate_limiter = &retransmission_rate_limiter_;
@ -1239,7 +1239,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) {
RTPSenderVideo::Config video_config;
video_config.clock = &fake_clock_;
video_config.rtp_sender = rtp_sender();
video_config.flexfec_sender = &flexfec_sender;
video_config.fec_generator = &flexfec_sender;
video_config.field_trials = &field_trials;
RTPSenderVideo rtp_sender_video(video_config);
@ -1311,7 +1311,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) {
config.clock = &fake_clock_;
config.outgoing_transport = &transport_;
config.local_media_ssrc = kSsrc;
config.flexfec_sender = &flexfec_sender;
config.fec_generator = &flexfec_sender;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
config.retransmission_rate_limiter = &retransmission_rate_limiter_;
@ -1323,7 +1323,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) {
RTPSenderVideo::Config video_config;
video_config.clock = &fake_clock_;
video_config.rtp_sender = rtp_sender();
video_config.flexfec_sender = &flexfec_sender;
video_config.fec_generator = &flexfec_sender;
video_config.field_trials = &field_trials;
RTPSenderVideo rtp_sender_video(video_config);
@ -1583,7 +1583,7 @@ TEST_P(RtpSenderTest, FecOverheadRate) {
config.outgoing_transport = &transport_;
config.paced_sender = &mock_paced_sender_;
config.local_media_ssrc = kSsrc;
config.flexfec_sender = &flexfec_sender;
config.fec_generator = &flexfec_sender;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;
config.retransmission_rate_limiter = &retransmission_rate_limiter_;
@ -1595,7 +1595,7 @@ TEST_P(RtpSenderTest, FecOverheadRate) {
RTPSenderVideo::Config video_config;
video_config.clock = &fake_clock_;
video_config.rtp_sender = rtp_sender();
video_config.flexfec_sender = &flexfec_sender;
video_config.fec_generator = &flexfec_sender;
video_config.field_trials = &field_trials;
RTPSenderVideo rtp_sender_video(video_config);
// Parameters selected to generate a single FEC packet per media packet.
@ -1777,12 +1777,14 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacksUlpfec) {
const uint8_t kPayloadType = 127;
const VideoCodecType kCodecType = VideoCodecType::kVideoCodecGeneric;
FieldTrialBasedConfig field_trials;
UlpfecGenerator ulpfec_generator(kRedPayloadType, kUlpfecPayloadType,
&fake_clock_);
RTPSenderVideo::Config video_config;
video_config.clock = &fake_clock_;
video_config.rtp_sender = rtp_sender();
video_config.field_trials = &field_trials;
video_config.red_payload_type = kRedPayloadType;
video_config.ulpfec_payload_type = kUlpfecPayloadType;
video_config.fec_generator = &ulpfec_generator;
RTPSenderVideo rtp_sender_video(video_config);
uint8_t payload[] = {47, 11, 32, 93, 89};
rtp_sender_context_->packet_history_.SetStorePacketsStatus(
@ -2118,7 +2120,7 @@ TEST_P(RtpSenderTest, SendPacketUpdatesStats) {
config.outgoing_transport = &transport_;
config.local_media_ssrc = kSsrc;
config.rtx_send_ssrc = kRtxSsrc;
config.flexfec_sender = &flexfec_sender_;
config.fec_generator = &flexfec_sender_;
config.send_side_delay_observer = &send_side_delay_observer;
config.event_log = &mock_rtc_event_log_;
config.send_packet_observer = &send_packet_observer_;

207
modules/rtp_rtcp/source/rtp_sender_video.cc
View File

@ -260,11 +260,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config)
current_playout_delay_{-1, -1},
playout_delay_pending_(false),
red_payload_type_(config.red_payload_type),
ulpfec_payload_type_(config.ulpfec_payload_type),
flexfec_sender_(config.flexfec_sender),
delta_fec_params_{0, 1, kFecMaskRandom},
key_fec_params_{0, 1, kFecMaskRandom},
fec_bitrate_(1000, RateStatistics::kBpsScale),
fec_generator_(config.fec_generator),
video_bitrate_(1000, RateStatistics::kBpsScale),
packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale),
frame_encryptor_(config.frame_encryptor),
@ -293,83 +289,6 @@ RTPSenderVideo::~RTPSenderVideo() {
frame_transformer_delegate_->Reset();
}
void RTPSenderVideo::AppendAsRedMaybeWithUlpfec(
std::unique_ptr<RtpPacketToSend> media_packet,
bool protect_media_packet,
std::vector<std::unique_ptr<RtpPacketToSend>>* packets) {
std::unique_ptr<RtpPacketToSend> red_packet(
new RtpPacketToSend(*media_packet));
BuildRedPayload(*media_packet, red_packet.get());
red_packet->SetPayloadType(*red_payload_type_);
std::vector<std::unique_ptr<RedPacket>> fec_packets;
if (ulpfec_enabled()) {
if (protect_media_packet) {
if (exclude_transport_sequence_number_from_fec_experiment_) {
// See comments at the top of the file why experiment
// "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in
// conjunction with datagram transport.
// TODO(sukhanov): We may also need to implement it for flexfec_sender
// if we decide to keep this approach in the future.
uint16_t transport_senquence_number;
if (media_packet->GetExtension<webrtc::TransportSequenceNumber>(
&transport_senquence_number)) {
if (!media_packet->RemoveExtension(
webrtc::TransportSequenceNumber::kId)) {
RTC_NOTREACHED()
<< "Failed to remove transport sequence number, packet="
<< media_packet->ToString();
}
}
}
ulpfec_generator_.AddRtpPacketAndGenerateFec(
media_packet->Buffer(), media_packet->headers_size());
}
uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
if (num_fec_packets > 0) {
uint16_t first_fec_sequence_number =
rtp_sender_->AllocateSequenceNumber(num_fec_packets);
fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed(
*red_payload_type_, *ulpfec_payload_type_, first_fec_sequence_number);
RTC_DCHECK_EQ(num_fec_packets, fec_packets.size());
}
}
// Send |red_packet| instead of |packet| for allocated sequence number.
red_packet->set_packet_type(RtpPacketMediaType::kVideo);
red_packet->set_allow_retransmission(media_packet->allow_retransmission());
packets->emplace_back(std::move(red_packet));
for (const auto& fec_packet : fec_packets) {
// TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid
// reparsing them.
std::unique_ptr<RtpPacketToSend> rtp_packet(
new RtpPacketToSend(*media_packet));
RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length()));
rtp_packet->set_capture_time_ms(media_packet->capture_time_ms());
rtp_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection);
rtp_packet->set_allow_retransmission(false);
RTC_DCHECK_EQ(fec_packet->length(), rtp_packet->size());
packets->emplace_back(std::move(rtp_packet));
}
}
void RTPSenderVideo::GenerateAndAppendFlexfec(
std::vector<std::unique_ptr<RtpPacketToSend>>* packets) {
RTC_DCHECK(flexfec_sender_);
if (flexfec_sender_->FecAvailable()) {
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
flexfec_sender_->GetFecPackets();
for (auto& fec_packet : fec_packets) {
fec_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection);
fec_packet->set_allow_retransmission(false);
packets->emplace_back(std::move(fec_packet));
}
}
}
void RTPSenderVideo::LogAndSendToNetwork(
std::vector<std::unique_ptr<RtpPacketToSend>> packets,
size_t unpacketized_payload_size) {
@ -388,16 +307,9 @@ void RTPSenderVideo::LogAndSendToNetwork(
rtc::CritScope cs(&stats_crit_);
size_t packetized_payload_size = 0;
for (const auto& packet : packets) {
switch (*packet->packet_type()) {
case RtpPacketMediaType::kVideo:
video_bitrate_.Update(packet->size(), now_ms);
packetized_payload_size += packet->payload_size();
break;
case RtpPacketMediaType::kForwardErrorCorrection:
fec_bitrate_.Update(packet->size(), clock_->TimeInMilliseconds());
break;
default:
continue;
if (*packet->packet_type() == RtpPacketMediaType::kVideo) {
video_bitrate_.Update(packet->size(), now_ms);
packetized_payload_size += packet->payload_size();
}
}
// AV1 packetizer may produce less packetized bytes than unpacketized.
@ -412,39 +324,31 @@ void RTPSenderVideo::LogAndSendToNetwork(
}
size_t RTPSenderVideo::FecPacketOverhead() const {
if (flexfec_enabled())
return flexfec_sender_->MaxPacketOverhead();
size_t overhead = 0;
size_t overhead = fec_generator_ ? fec_generator_->MaxPacketOverhead() : 0u;
if (red_enabled()) {
// The RED overhead is due to a small header.
overhead += kRedForFecHeaderLength;
}
if (ulpfec_enabled()) {
// For ULPFEC, the overhead is the FEC headers plus RED for FEC header
// (see above) plus anything in RTP header beyond the 12 bytes base header
// (CSRC list, extensions...)
// This reason for the header extensions to be included here is that
// from an FEC viewpoint, they are part of the payload to be protected.
// (The base RTP header is already protected by the FEC header.)
overhead += ulpfec_generator_.MaxPacketOverhead() +
(rtp_sender_->RtpHeaderLength() - kRtpHeaderSize);
// TODO(bugs.webrtc.org/11340): Move this into UlpfecGenerator.
if (fec_generator_ &&
fec_generator_->GetFecType() == VideoFecGenerator::FecType::kUlpFec) {
// For ULPFEC, the overhead is the FEC headers plus RED for FEC header
// (see above) plus anything in RTP header beyond the 12 bytes base header
// (CSRC list, extensions...)
// This reason for the header extensions to be included here is that
// from an FEC viewpoint, they are part of the payload to be protected.
// (The base RTP header is already protected by the FEC header.)
overhead += rtp_sender_->RtpHeaderLength() - kRtpHeaderSize;
}
}
return overhead;
}
void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params,
const FecProtectionParams& key_params) {
rtc::CritScope cs(&crit_);
delta_fec_params_ = delta_params;
key_fec_params_ = key_params;
}
absl::optional<uint32_t> RTPSenderVideo::FlexfecSsrc() const {
if (flexfec_sender_) {
return flexfec_sender_->ssrc();
if (fec_generator_) {
fec_generator_->SetProtectionParameters(delta_params, key_params);
}
return absl::nullopt;
}
void RTPSenderVideo::SetVideoStructure(
@ -565,19 +469,6 @@ bool RTPSenderVideo::SendVideo(
transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false;
}
if (flexfec_enabled() || ulpfec_enabled()) {
rtc::CritScope cs(&crit_);
// FEC settings.
const FecProtectionParams& fec_params =
video_header.frame_type == VideoFrameType::kVideoFrameKey
? key_fec_params_
: delta_fec_params_;
if (flexfec_enabled())
flexfec_sender_->SetFecParameters(fec_params);
if (ulpfec_enabled())
ulpfec_generator_.SetFecParameters(fec_params);
}
// Maximum size of packet including rtp headers.
// Extra space left in case packet will be resent using fec or rtx.
int packet_capacity = rtp_sender_->MaxRtpPacketSize() - FecPacketOverhead() -
@ -769,21 +660,40 @@ bool RTPSenderVideo::SendVideo(
packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
}
if (protect_packet && fec_generator_) {
if (red_enabled() &&
exclude_transport_sequence_number_from_fec_experiment_) {
// See comments at the top of the file why experiment
// "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in
// conjunction with datagram transport.
// TODO(sukhanov): We may also need to implement it for flexfec_sender
// if we decide to keep this approach in the future.
uint16_t transport_senquence_number;
if (packet->GetExtension<webrtc::TransportSequenceNumber>(
&transport_senquence_number)) {
if (!packet->RemoveExtension(webrtc::TransportSequenceNumber::kId)) {
RTC_NOTREACHED()
<< "Failed to remove transport sequence number, packet="
<< packet->ToString();
}
}
}
fec_generator_->AddPacketAndGenerateFec(*packet);
}
if (red_enabled()) {
AppendAsRedMaybeWithUlpfec(std::move(packet), protect_packet,
&rtp_packets);
std::unique_ptr<RtpPacketToSend> red_packet(new RtpPacketToSend(*packet));
BuildRedPayload(*packet, red_packet.get());
red_packet->SetPayloadType(*red_payload_type_);
// Send |red_packet| instead of |packet| for allocated sequence number.
red_packet->set_packet_type(RtpPacketMediaType::kVideo);
red_packet->set_allow_retransmission(packet->allow_retransmission());
rtp_packets.emplace_back(std::move(red_packet));
} else {
packet->set_packet_type(RtpPacketMediaType::kVideo);
const RtpPacketToSend& media_packet = *packet;
rtp_packets.emplace_back(std::move(packet));
if (flexfec_enabled()) {
// TODO(brandtr): Remove the FlexFEC code path when FlexfecSender
// is wired up to PacedSender instead.
if (protect_packet) {
flexfec_sender_->AddRtpPacketAndGenerateFec(media_packet);
}
GenerateAndAppendFlexfec(&rtp_packets);
}
}
if (first_frame) {
@ -798,6 +708,22 @@ bool RTPSenderVideo::SendVideo(
}
}
if (fec_generator_) {
// Fetch any FEC packets generated from the media frame and add them to
// the list of packets to send.
auto fec_packets = fec_generator_->GetFecPackets();
// TODO(bugs.webrtc.org/11340): Move sequence number assignment into
// UlpfecGenerator.
const bool generate_sequence_numbers = !fec_generator_->FecSsrc();
for (auto& fec_packet : fec_packets) {
if (generate_sequence_numbers) {
rtp_sender_->AssignSequenceNumber(fec_packet.get());
}
rtp_packets.emplace_back(std::move(fec_packet));
}
}
LogAndSendToNetwork(std::move(rtp_packets), unpacketized_payload_size);
TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
@ -830,8 +756,7 @@ uint32_t RTPSenderVideo::VideoBitrateSent() const {
}
uint32_t RTPSenderVideo::FecOverheadRate() const {
rtc::CritScope cs(&stats_crit_);
return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
return fec_generator_ ? fec_generator_->CurrentFecRate().bps<uint32_t>() : 0u;
}
uint32_t RTPSenderVideo::PacketizationOverheadBps() const {

38
modules/rtp_rtcp/source/rtp_sender_video.h
View File

@ -24,14 +24,13 @@
#include "api/video/video_codec_type.h"
#include "api/video/video_frame_type.h"
#include "modules/include/module_common_types.h"
#include "modules/rtp_rtcp/include/flexfec_sender.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
#include "modules/rtp_rtcp/source/rtp_sender.h"
#include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
#include "modules/rtp_rtcp/source/ulpfec_generator.h"
#include "modules/rtp_rtcp/source/video_fec_generator.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/one_time_event.h"
#include "rtc_base/race_checker.h"
@ -71,11 +70,11 @@ class RTPSenderVideo {
Clock* clock = nullptr;
RTPSender* rtp_sender = nullptr;
FlexfecSender* flexfec_sender = nullptr;
VideoFecGenerator* fec_generator = nullptr;
FrameEncryptorInterface* frame_encryptor = nullptr;
bool require_frame_encryption = false;
bool enable_retransmit_all_layers = false;
absl::optional<int> red_payload_type;
absl::optional<int> ulpfec_payload_type;
const WebRtcKeyValueConfig* field_trials = nullptr;
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer;
};
@ -115,13 +114,9 @@ class RTPSenderVideo {
// FlexFEC/ULPFEC.
// Set FEC rates, max frames before FEC is sent, and type of FEC masks.
// Returns false on failure.
void SetFecParameters(const FecProtectionParams& delta_params,
const FecProtectionParams& key_params);
// FlexFEC.
absl::optional<uint32_t> FlexfecSsrc() const;
uint32_t VideoBitrateSent() const;
uint32_t FecOverheadRate() const;
@ -150,27 +145,12 @@ class RTPSenderVideo {
size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
void AppendAsRedMaybeWithUlpfec(
std::unique_ptr<RtpPacketToSend> media_packet,
bool protect_media_packet,
std::vector<std::unique_ptr<RtpPacketToSend>>* packets)
RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
// TODO(brandtr): Remove the FlexFEC functions when FlexfecSender has been
// moved to PacedSender.
void GenerateAndAppendFlexfec(
std::vector<std::unique_ptr<RtpPacketToSend>>* packets);
void LogAndSendToNetwork(
std::vector<std::unique_ptr<RtpPacketToSend>> packets,
size_t unpacketized_payload_size);
bool red_enabled() const { return red_payload_type_.has_value(); }
bool ulpfec_enabled() const { return ulpfec_payload_type_.has_value(); }
bool flexfec_enabled() const { return flexfec_sender_ != nullptr; }
bool UpdateConditionalRetransmit(uint8_t temporal_id,
int64_t expected_retransmission_time_ms)
RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_crit_);
@ -201,22 +181,10 @@ class RTPSenderVideo {
// Should never be held when calling out of this class.
rtc::CriticalSection crit_;
// RED/ULPFEC.
const absl::optional<int> red_payload_type_;
const absl::optional<int> ulpfec_payload_type_;
UlpfecGenerator ulpfec_generator_ RTC_GUARDED_BY(send_checker_);
// FlexFEC.
FlexfecSender* const flexfec_sender_;
// FEC parameters, applicable to either ULPFEC or FlexFEC.
FecProtectionParams delta_fec_params_ RTC_GUARDED_BY(crit_);
FecProtectionParams key_fec_params_ RTC_GUARDED_BY(crit_);
VideoFecGenerator* const fec_generator_;
rtc::CriticalSection stats_crit_;
// Bitrate used for FEC payload, RED headers, RTP headers for FEC packets
// and any padding overhead.
RateStatistics fec_bitrate_ RTC_GUARDED_BY(stats_crit_);
// Bitrate used for video payload and RTP headers.
RateStatistics video_bitrate_ RTC_GUARDED_BY(stats_crit_);
RateStatistics packetization_overhead_bitrate_ RTC_GUARDED_BY(stats_crit_);

2
modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
View File

@ -131,7 +131,7 @@ class TestRtpSenderVideo : public RTPSenderVideo {
Config config;
config.clock = clock;
config.rtp_sender = rtp_sender;
config.flexfec_sender = flexfec_sender;
config.fec_generator = flexfec_sender;
config.field_trials = &field_trials;
return config;
}()) {}

242
modules/rtp_rtcp/source/ulpfec_generator.cc
View File

@ -22,6 +22,7 @@
#include "modules/rtp_rtcp/source/forward_error_correction_internal.h"
#include "modules/rtp_rtcp/source/rtp_utility.h"
#include "rtc_base/checks.h"
#include "rtc_base/critical_section.h"
namespace webrtc {
@ -62,128 +63,119 @@ constexpr uint32_t kUnknownSsrc = 0;
} // namespace
RedPacket::RedPacket(size_t length)
: data_(new uint8_t[length]), length_(length), header_length_(0) {}
UlpfecGenerator::Params::Params() = default;
UlpfecGenerator::Params::Params(FecProtectionParams delta_params,
FecProtectionParams keyframe_params)
: delta_params(delta_params), keyframe_params(keyframe_params) {}
RedPacket::~RedPacket() = default;
void RedPacket::CreateHeader(const uint8_t* rtp_header,
size_t header_length,
int red_payload_type,
int payload_type) {
RTC_DCHECK_LE(header_length + kRedForFecHeaderLength, length_);
memcpy(data_.get(), rtp_header, header_length);
// Replace payload type.
data_[1] &= 0x80;
data_[1] += red_payload_type;
// Add RED header
// f-bit always 0
data_[header_length] = static_cast<uint8_t>(payload_type);
header_length_ = header_length + kRedForFecHeaderLength;
}
void RedPacket::SetSeqNum(int seq_num) {
RTC_DCHECK_GE(seq_num, 0);
RTC_DCHECK_LT(seq_num, 1 << 16);
ByteWriter<uint16_t>::WriteBigEndian(&data_[2], seq_num);
}
void RedPacket::AssignPayload(const uint8_t* payload, size_t length) {
RTC_DCHECK_LE(header_length_ + length, length_);
memcpy(data_.get() + header_length_, payload, length);
}
void RedPacket::ClearMarkerBit() {
data_[1] &= 0x7F;
}
uint8_t* RedPacket::data() const {
return data_.get();
}
size_t RedPacket::length() const {
return length_;
}
UlpfecGenerator::UlpfecGenerator()
: UlpfecGenerator(ForwardErrorCorrection::CreateUlpfec(kUnknownSsrc)) {}
UlpfecGenerator::UlpfecGenerator(std::unique_ptr<ForwardErrorCorrection> fec)
: fec_(std::move(fec)),
last_media_packet_rtp_header_length_(0),
UlpfecGenerator::UlpfecGenerator(int red_payload_type,
int ulpfec_payload_type,
Clock* clock)
: red_payload_type_(red_payload_type),
ulpfec_payload_type_(ulpfec_payload_type),
clock_(clock),
fec_(ForwardErrorCorrection::CreateUlpfec(kUnknownSsrc)),
num_protected_frames_(0),
min_num_media_packets_(1) {
memset(&params_, 0, sizeof(params_));
memset(&new_params_, 0, sizeof(new_params_));
}
min_num_media_packets_(1),
keyframe_in_process_(false),
fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {}
// Used by FlexFecSender, payload types are unused.
UlpfecGenerator::UlpfecGenerator(std::unique_ptr<ForwardErrorCorrection> fec,
Clock* clock)
: red_payload_type_(0),
ulpfec_payload_type_(0),
clock_(clock),
fec_(std::move(fec)),
num_protected_frames_(0),
min_num_media_packets_(1),
keyframe_in_process_(false),
fec_bitrate_(/*max_window_size_ms=*/1000, RateStatistics::kBpsScale) {}
UlpfecGenerator::~UlpfecGenerator() = default;
void UlpfecGenerator::SetFecParameters(const FecProtectionParams& params) {
RTC_DCHECK_GE(params.fec_rate, 0);
RTC_DCHECK_LE(params.fec_rate, 255);
void UlpfecGenerator::SetProtectionParameters(
const FecProtectionParams& delta_params,
const FecProtectionParams& key_params) {
RTC_DCHECK_GE(delta_params.fec_rate, 0);
RTC_DCHECK_LE(delta_params.fec_rate, 255);
RTC_DCHECK_GE(key_params.fec_rate, 0);
RTC_DCHECK_LE(key_params.fec_rate, 255);
// Store the new params and apply them for the next set of FEC packets being
// produced.
new_params_ = params;
if (params.fec_rate > kHighProtectionThreshold) {
min_num_media_packets_ = kMinMediaPackets;
} else {
min_num_media_packets_ = 1;
}
rtc::CritScope cs(&crit_);
pending_params_.emplace(delta_params, key_params);
}
int UlpfecGenerator::AddRtpPacketAndGenerateFec(
const rtc::CopyOnWriteBuffer& data_buffer,
size_t rtp_header_length) {
void UlpfecGenerator::AddPacketAndGenerateFec(const RtpPacketToSend& packet) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
RTC_DCHECK(generated_fec_packets_.empty());
if (media_packets_.empty()) {
params_ = new_params_;
rtc::CritScope cs(&crit_);
if (pending_params_) {
current_params_ = *pending_params_;
pending_params_.reset();
if (CurrentParams().fec_rate > kHighProtectionThreshold) {
min_num_media_packets_ = kMinMediaPackets;
} else {
min_num_media_packets_ = 1;
}
}
keyframe_in_process_ = packet.is_key_frame();
}
RTC_DCHECK_EQ(packet.is_key_frame(), keyframe_in_process_);
bool complete_frame = false;
const bool marker_bit = (data_buffer[1] & kRtpMarkerBitMask) ? true : false;
const bool marker_bit = packet.Marker();
if (media_packets_.size() < kUlpfecMaxMediaPackets) {
// Our packet masks can only protect up to |kUlpfecMaxMediaPackets| packets.
std::unique_ptr<ForwardErrorCorrection::Packet> packet(
new ForwardErrorCorrection::Packet());
RTC_DCHECK_GE(data_buffer.size(), rtp_header_length);
packet->data = data_buffer;
media_packets_.push_back(std::move(packet));
// Keep track of the RTP header length, so we can copy the RTP header
// from |packet| to newly generated ULPFEC+RED packets.
RTC_DCHECK_GE(rtp_header_length, kRtpHeaderSize);
last_media_packet_rtp_header_length_ = rtp_header_length;
auto fec_packet = std::make_unique<ForwardErrorCorrection::Packet>();
fec_packet->data = packet.Buffer();
media_packets_.push_back(std::move(fec_packet));
// Keep a copy of the last RTP packet, so we can copy the RTP header
// from it when creating newly generated ULPFEC+RED packets.
RTC_DCHECK_GE(packet.headers_size(), kRtpHeaderSize);
last_media_packet_ = packet;
}
if (marker_bit) {
++num_protected_frames_;
complete_frame = true;
}
auto params = CurrentParams();
// Produce FEC over at most |params_.max_fec_frames| frames, or as soon as:
// (1) the excess overhead (actual overhead - requested/target overhead) is
// less than |kMaxExcessOverhead|, and
// (2) at least |min_num_media_packets_| media packets is reached.
if (complete_frame &&
(num_protected_frames_ == params_.max_fec_frames ||
(num_protected_frames_ == params.max_fec_frames ||
(ExcessOverheadBelowMax() && MinimumMediaPacketsReached()))) {
// We are not using Unequal Protection feature of the parity erasure code.
constexpr int kNumImportantPackets = 0;
constexpr bool kUseUnequalProtection = false;
int ret = fec_->EncodeFec(media_packets_, params_.fec_rate,
kNumImportantPackets, kUseUnequalProtection,
params_.fec_mask_type, &generated_fec_packets_);
fec_->EncodeFec(media_packets_, params.fec_rate, kNumImportantPackets,
kUseUnequalProtection, params.fec_mask_type,
&generated_fec_packets_);
if (generated_fec_packets_.empty()) {
ResetState();
}
return ret;
}
return 0;
}
bool UlpfecGenerator::ExcessOverheadBelowMax() const {
return ((Overhead() - params_.fec_rate) < kMaxExcessOverhead);
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
return ((Overhead() - CurrentParams().fec_rate) < kMaxExcessOverhead);
}
bool UlpfecGenerator::MinimumMediaPacketsReached() const {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
float average_num_packets_per_frame =
static_cast<float>(media_packets_.size()) / num_protected_frames_;
int num_media_packets = static_cast<int>(media_packets_.size());
@ -196,61 +188,79 @@ bool UlpfecGenerator::MinimumMediaPacketsReached() const {
}
}
bool UlpfecGenerator::FecAvailable() const {
return !generated_fec_packets_.empty();
}
size_t UlpfecGenerator::NumAvailableFecPackets() const {
return generated_fec_packets_.size();
const FecProtectionParams& UlpfecGenerator::CurrentParams() const {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
return keyframe_in_process_ ? current_params_.keyframe_params
: current_params_.delta_params;
}
size_t UlpfecGenerator::MaxPacketOverhead() const {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
return fec_->MaxPacketOverhead();
}
std::vector<std::unique_ptr<RedPacket>> UlpfecGenerator::GetUlpfecPacketsAsRed(
int red_payload_type,
int ulpfec_payload_type,
uint16_t first_seq_num) {
std::vector<std::unique_ptr<RedPacket>> red_packets;
red_packets.reserve(generated_fec_packets_.size());
RTC_DCHECK(!media_packets_.empty());
ForwardErrorCorrection::Packet* last_media_packet =
media_packets_.back().get();
uint16_t seq_num = first_seq_num;
std::vector<std::unique_ptr<RtpPacketToSend>> UlpfecGenerator::GetFecPackets() {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
if (generated_fec_packets_.empty()) {
return std::vector<std::unique_ptr<RtpPacketToSend>>();
}
// Wrap FEC packet (including FEC headers) in a RED packet. Since the
// FEC packets in |generated_fec_packets_| don't have RTP headers, we
// reuse the header from the last media packet.
RTC_CHECK(last_media_packet_.has_value());
last_media_packet_->SetPayloadSize(0);
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets;
fec_packets.reserve(generated_fec_packets_.size());
size_t total_fec_size_bytes = 0;
for (const auto* fec_packet : generated_fec_packets_) {
// Wrap FEC packet (including FEC headers) in a RED packet. Since the
// FEC packets in |generated_fec_packets_| don't have RTP headers, we
// reuse the header from the last media packet.
RTC_DCHECK_GT(last_media_packet_rtp_header_length_, 0);
std::unique_ptr<RedPacket> red_packet(
new RedPacket(last_media_packet_rtp_header_length_ +
kRedForFecHeaderLength + fec_packet->data.size()));
red_packet->CreateHeader(last_media_packet->data.data(),
last_media_packet_rtp_header_length_,
red_payload_type, ulpfec_payload_type);
red_packet->SetSeqNum(seq_num++);
red_packet->ClearMarkerBit();
red_packet->AssignPayload(fec_packet->data.data(), fec_packet->data.size());
red_packets.push_back(std::move(red_packet));
std::unique_ptr<RtpPacketToSend> red_packet =
std::make_unique<RtpPacketToSend>(*last_media_packet_);
red_packet->SetPayloadType(red_payload_type_);
red_packet->SetMarker(false);
uint8_t* payload_buffer = red_packet->SetPayloadSize(
kRedForFecHeaderLength + fec_packet->data.size());
// Primary RED header with F bit unset.
// See https://tools.ietf.org/html/rfc2198#section-3
payload_buffer[0] = ulpfec_payload_type_; // RED header.
memcpy(&payload_buffer[1], fec_packet->data.data(),
fec_packet->data.size());
total_fec_size_bytes += red_packet->size();
red_packet->set_packet_type(RtpPacketMediaType::kForwardErrorCorrection);
red_packet->set_allow_retransmission(false);
fec_packets.push_back(std::move(red_packet));
}
ResetState();
return red_packets;
rtc::CritScope cs(&crit_);
fec_bitrate_.Update(total_fec_size_bytes, clock_->TimeInMilliseconds());
return fec_packets;
}
DataRate UlpfecGenerator::CurrentFecRate() const {
rtc::CritScope cs(&crit_);
return DataRate::BitsPerSec(
fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0));
}
int UlpfecGenerator::Overhead() const {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
RTC_DCHECK(!media_packets_.empty());
int num_fec_packets =
fec_->NumFecPackets(media_packets_.size(), params_.fec_rate);
fec_->NumFecPackets(media_packets_.size(), CurrentParams().fec_rate);
// Return the overhead in Q8.
return (num_fec_packets << 8) / media_packets_.size();
}
void UlpfecGenerator::ResetState() {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
media_packets_.clear();
last_media_packet_rtp_header_length_ = 0;
last_media_packet_.reset();
generated_fec_packets_.clear();
num_protected_frames_ = 0;
}

98
modules/rtp_rtcp/source/ulpfec_generator.h
View File

@ -20,63 +20,54 @@
#include "modules/include/module_fec_types.h"
#include "modules/rtp_rtcp/source/forward_error_correction.h"
#include "modules/rtp_rtcp/source/video_fec_generator.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/rate_statistics.h"
namespace webrtc {
class FlexfecSender;
class RedPacket {
public:
explicit RedPacket(size_t length);
~RedPacket();
void CreateHeader(const uint8_t* rtp_header,
size_t header_length,
int red_payload_type,
int payload_type);
void SetSeqNum(int seq_num);
void AssignPayload(const uint8_t* payload, size_t length);
void ClearMarkerBit();
uint8_t* data() const;
size_t length() const;
private:
std::unique_ptr<uint8_t[]> data_;
size_t length_;
size_t header_length_;
};
class UlpfecGenerator {
class UlpfecGenerator : public VideoFecGenerator {
friend class FlexfecSender;
public:
UlpfecGenerator();
UlpfecGenerator(int red_payload_type, int ulpfec_payload_type, Clock* clock);
~UlpfecGenerator();
void SetFecParameters(const FecProtectionParams& params);
FecType GetFecType() const override {
return VideoFecGenerator::FecType::kUlpFec;
}
absl::optional<uint32_t> FecSsrc() override { return absl::nullopt; }
void SetProtectionParameters(const FecProtectionParams& delta_params,
const FecProtectionParams& key_params) override;
// Adds a media packet to the internal buffer. When enough media packets
// have been added, the FEC packets are generated and stored internally.
// These FEC packets are then obtained by calling GetFecPacketsAsRed().
int AddRtpPacketAndGenerateFec(const rtc::CopyOnWriteBuffer& data_buffer,
size_t rtp_header_length);
// Returns true if there are generated FEC packets available.
bool FecAvailable() const;
size_t NumAvailableFecPackets() const;
void AddPacketAndGenerateFec(const RtpPacketToSend& packet) override;
// Returns the overhead, per packet, for FEC (and possibly RED).
size_t MaxPacketOverhead() const;
size_t MaxPacketOverhead() const override;
// Returns generated FEC packets with RED headers added.
std::vector<std::unique_ptr<RedPacket>> GetUlpfecPacketsAsRed(
int red_payload_type,
int ulpfec_payload_type,
uint16_t first_seq_num);
std::vector<std::unique_ptr<RtpPacketToSend>> GetFecPackets() override;
// Current rate of FEC packets generated, including all RTP-level headers.
DataRate CurrentFecRate() const override;
private:
explicit UlpfecGenerator(std::unique_ptr<ForwardErrorCorrection> fec);
struct Params {
Params();
Params(FecProtectionParams delta_params,
FecProtectionParams keyframe_params);
FecProtectionParams delta_params;
FecProtectionParams keyframe_params;
};
UlpfecGenerator(std::unique_ptr<ForwardErrorCorrection> fec, Clock* clock);
// Overhead is defined as relative to the number of media packets, and not
// relative to total number of packets. This definition is inherited from the
@ -97,16 +88,31 @@ class UlpfecGenerator {
// (e.g. (2k,2m) vs (k,m)) are generally more effective at recovering losses.
bool MinimumMediaPacketsReached() const;
const FecProtectionParams& CurrentParams() const;
void ResetState();
std::unique_ptr<ForwardErrorCorrection> fec_;
ForwardErrorCorrection::PacketList media_packets_;
size_t last_media_packet_rtp_header_length_;
std::list<ForwardErrorCorrection::Packet*> generated_fec_packets_;
int num_protected_frames_;
int min_num_media_packets_;
FecProtectionParams params_;
FecProtectionParams new_params_;
const int red_payload_type_;
const int ulpfec_payload_type_;
Clock* const clock_;
rtc::RaceChecker race_checker_;
const std::unique_ptr<ForwardErrorCorrection> fec_
RTC_GUARDED_BY(race_checker_);
ForwardErrorCorrection::PacketList media_packets_
RTC_GUARDED_BY(race_checker_);
absl::optional<RtpPacketToSend> last_media_packet_
RTC_GUARDED_BY(race_checker_);
std::list<ForwardErrorCorrection::Packet*> generated_fec_packets_
RTC_GUARDED_BY(race_checker_);
int num_protected_frames_ RTC_GUARDED_BY(race_checker_);
int min_num_media_packets_ RTC_GUARDED_BY(race_checker_);
Params current_params_ RTC_GUARDED_BY(race_checker_);
bool keyframe_in_process_ RTC_GUARDED_BY(race_checker_);
rtc::CriticalSection crit_;
absl::optional<Params> pending_params_ RTC_GUARDED_BY(crit_);
RateStatistics fec_bitrate_ RTC_GUARDED_BY(crit_);
};
} // namespace webrtc

128
modules/rtp_rtcp/source/ulpfec_generator_unittest.cc
View File

@ -35,11 +35,8 @@ void VerifyHeader(uint16_t seq_num,
uint32_t timestamp,
int red_payload_type,
int fec_payload_type,
RedPacket* packet,
bool marker_bit) {
EXPECT_GT(packet->length(), kRtpHeaderSize);
EXPECT_TRUE(packet->data() != NULL);
uint8_t* data = packet->data();
bool marker_bit,
const rtc::CopyOnWriteBuffer& data) {
// Marker bit not set.
EXPECT_EQ(marker_bit ? 0x80 : 0, data[1] & 0x80);
EXPECT_EQ(red_payload_type, data[1] & 0x7F);
@ -52,8 +49,12 @@ void VerifyHeader(uint16_t seq_num,
class UlpfecGeneratorTest : public ::testing::Test {
protected:
UlpfecGeneratorTest() : packet_generator_(kMediaSsrc) {}
UlpfecGeneratorTest()
: fake_clock_(1),
ulpfec_generator_(kRedPayloadType, kFecPayloadType, &fake_clock_),
packet_generator_(kMediaSsrc) {}
SimulatedClock fake_clock_;
UlpfecGenerator ulpfec_generator_;
AugmentedPacketGenerator packet_generator_;
};
@ -81,24 +82,22 @@ TEST_F(UlpfecGeneratorTest, NoEmptyFecWithSeqNumGaps) {
protected_packets.push_back({21, 0, 55, 0});
protected_packets.push_back({13, 3, 57, 1});
FecProtectionParams params = {117, 3, kFecMaskBursty};
ulpfec_generator_.SetFecParameters(params);
uint8_t packet[28] = {0};
ulpfec_generator_.SetProtectionParameters(params, params);
for (Packet p : protected_packets) {
if (p.marker_bit) {
packet[1] |= 0x80;
RtpPacketToSend packet(nullptr);
packet.SetMarker(p.marker_bit);
packet.AllocateExtension(RTPExtensionType::kRtpExtensionMid,
p.header_size - packet.headers_size());
packet.SetSequenceNumber(p.seq_num);
packet.AllocatePayload(p.payload_size);
ulpfec_generator_.AddPacketAndGenerateFec(packet);
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
ulpfec_generator_.GetFecPackets();
if (!p.marker_bit) {
EXPECT_TRUE(fec_packets.empty());
} else {
packet[1] &= ~0x80;
}
ByteWriter<uint16_t>::WriteBigEndian(&packet[2], p.seq_num);
ulpfec_generator_.AddRtpPacketAndGenerateFec(
rtc::CopyOnWriteBuffer(packet, p.payload_size + p.header_size),
p.header_size);
size_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
if (num_fec_packets > 0) {
std::vector<std::unique_ptr<RedPacket>> fec_packets =
ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType,
kFecPayloadType, 100);
EXPECT_EQ(num_fec_packets, fec_packets.size());
EXPECT_FALSE(fec_packets.empty());
}
}
}
@ -113,24 +112,28 @@ TEST_F(UlpfecGeneratorTest, OneFrameFec) {
constexpr size_t kNumPackets = 4;
FecProtectionParams params = {15, 3, kFecMaskRandom};
packet_generator_.NewFrame(kNumPackets);
ulpfec_generator_.SetFecParameters(params); // Expecting one FEC packet.
// Expecting one FEC packet.
ulpfec_generator_.SetProtectionParameters(params, params);
uint32_t last_timestamp = 0;
for (size_t i = 0; i < kNumPackets; ++i) {
std::unique_ptr<AugmentedPacket> packet =
packet_generator_.NextPacket(i, 10);
EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(packet->data,
kRtpHeaderSize));
RtpPacketToSend rtp_packet(nullptr);
EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size()));
ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet);
last_timestamp = packet->header.timestamp;
}
EXPECT_TRUE(ulpfec_generator_.FecAvailable());
const uint16_t seq_num = packet_generator_.NextPacketSeqNum();
std::vector<std::unique_ptr<RedPacket>> red_packets =
ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType,
seq_num);
EXPECT_FALSE(ulpfec_generator_.FecAvailable());
ASSERT_EQ(1u, red_packets.size());
VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType,
red_packets.front().get(), false);
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
ulpfec_generator_.GetFecPackets();
EXPECT_EQ(fec_packets.size(), 1u);
uint16_t seq_num = packet_generator_.NextPacketSeqNum();
fec_packets[0]->SetSequenceNumber(seq_num);
EXPECT_TRUE(ulpfec_generator_.GetFecPackets().empty());
EXPECT_EQ(fec_packets[0]->headers_size(), kRtpHeaderSize);
VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, false,
fec_packets[0]->Buffer());
}
TEST_F(UlpfecGeneratorTest, TwoFrameFec) {
@ -145,27 +148,27 @@ TEST_F(UlpfecGeneratorTest, TwoFrameFec) {
constexpr size_t kNumFrames = 2;
FecProtectionParams params = {15, 3, kFecMaskRandom};
ulpfec_generator_.SetFecParameters(params); // Expecting one FEC packet.
// Expecting one FEC packet.
ulpfec_generator_.SetProtectionParameters(params, params);
uint32_t last_timestamp = 0;
for (size_t i = 0; i < kNumFrames; ++i) {
packet_generator_.NewFrame(kNumPackets);
for (size_t j = 0; j < kNumPackets; ++j) {
std::unique_ptr<AugmentedPacket> packet =
packet_generator_.NextPacket(i * kNumPackets + j, 10);
EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(
packet->data, kRtpHeaderSize));
RtpPacketToSend rtp_packet(nullptr);
EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size()));
ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet);
last_timestamp = packet->header.timestamp;
}
}
EXPECT_TRUE(ulpfec_generator_.FecAvailable());
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
ulpfec_generator_.GetFecPackets();
EXPECT_EQ(fec_packets.size(), 1u);
const uint16_t seq_num = packet_generator_.NextPacketSeqNum();
std::vector<std::unique_ptr<RedPacket>> red_packets =
ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType,
seq_num);
EXPECT_FALSE(ulpfec_generator_.FecAvailable());
ASSERT_EQ(1u, red_packets.size());
VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType,
red_packets.front().get(), false);
fec_packets[0]->SetSequenceNumber(seq_num);
VerifyHeader(seq_num, last_timestamp, kRedPayloadType, kFecPayloadType, false,
fec_packets[0]->Buffer());
}
TEST_F(UlpfecGeneratorTest, MixedMediaRtpHeaderLengths) {
@ -174,34 +177,43 @@ TEST_F(UlpfecGeneratorTest, MixedMediaRtpHeaderLengths) {
// Only one frame required to generate FEC.
FecProtectionParams params = {127, 1, kFecMaskRandom};
ulpfec_generator_.SetFecParameters(params);
ulpfec_generator_.SetProtectionParameters(params, params);
// Fill up internal buffer with media packets with short RTP header length.
packet_generator_.NewFrame(kUlpfecMaxMediaPackets + 1);
for (size_t i = 0; i < kUlpfecMaxMediaPackets; ++i) {
std::unique_ptr<AugmentedPacket> packet =
packet_generator_.NextPacket(i, 10);
EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(
packet->data, kShortRtpHeaderLength));
EXPECT_FALSE(ulpfec_generator_.FecAvailable());
RtpPacketToSend rtp_packet(nullptr);
EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size()));
EXPECT_EQ(rtp_packet.headers_size(), kShortRtpHeaderLength);
ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet);
EXPECT_TRUE(ulpfec_generator_.GetFecPackets().empty());
}
// Kick off FEC generation with media packet with long RTP header length.
// Since the internal buffer is full, this packet will not be protected.
std::unique_ptr<AugmentedPacket> packet =
packet_generator_.NextPacket(kUlpfecMaxMediaPackets, 10);
EXPECT_EQ(0, ulpfec_generator_.AddRtpPacketAndGenerateFec(
packet->data, kLongRtpHeaderLength));
EXPECT_TRUE(ulpfec_generator_.FecAvailable());
RtpPacketToSend rtp_packet(nullptr);
EXPECT_TRUE(rtp_packet.Parse(packet->data.data(), packet->data.size()));
EXPECT_TRUE(rtp_packet.SetPayloadSize(0) != nullptr);
const uint32_t csrcs[]{1};
rtp_packet.SetCsrcs(csrcs);
EXPECT_EQ(rtp_packet.headers_size(), kLongRtpHeaderLength);
ulpfec_generator_.AddPacketAndGenerateFec(rtp_packet);
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
ulpfec_generator_.GetFecPackets();
EXPECT_FALSE(fec_packets.empty());
// Ensure that the RED header is placed correctly, i.e. the correct
// RTP header length was used in the RED packet creation.
const uint16_t seq_num = packet_generator_.NextPacketSeqNum();
std::vector<std::unique_ptr<RedPacket>> red_packets =
ulpfec_generator_.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType,
seq_num);
for (const auto& red_packet : red_packets) {
EXPECT_EQ(kFecPayloadType, red_packet->data()[kShortRtpHeaderLength]);
uint16_t seq_num = packet_generator_.NextPacketSeqNum();
for (const auto& fec_packet : fec_packets) {
fec_packet->SetSequenceNumber(seq_num++);
EXPECT_EQ(kFecPayloadType, fec_packet->data()[kShortRtpHeaderLength]);
}
}

51
modules/rtp_rtcp/source/video_fec_generator.h Normal file
View File

@ -0,0 +1,51 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_
#define MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_
#include <memory>
#include <vector>
#include "api/units/data_rate.h"
#include "modules/include/module_fec_types.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
namespace webrtc {
class VideoFecGenerator {
public:
VideoFecGenerator() = default;
virtual ~VideoFecGenerator() = default;
enum class FecType { kFlexFec, kUlpFec };
virtual FecType GetFecType() const = 0;
// Returns the SSRC used for FEC packets (i.e. FlexFec SSRC).
virtual absl::optional<uint32_t> FecSsrc() = 0;
// Returns the overhead, in bytes per packet, for FEC (and possibly RED).
virtual size_t MaxPacketOverhead() const = 0;
// Current rate of FEC packets generated, including all RTP-level headers.
virtual DataRate CurrentFecRate() const = 0;
// Set FEC rates, max frames before FEC is sent, and type of FEC masks.
virtual void SetProtectionParameters(
const FecProtectionParams& delta_params,
const FecProtectionParams& key_params) = 0;
// Called on new media packet to be protected. The generator may choose
// to generate FEC packets at this time, if so they will be stored in an
// internal buffer.
virtual void AddPacketAndGenerateFec(const RtpPacketToSend& packet) = 0;
// Get (and remove) and FEC packets pending in the generator. These packets
// will lack sequence numbers, that needs to be set externally.
// TODO(bugs.webrtc.org/11340): Actually FlexFec sets seq#, fix that!
virtual std::vector<std::unique_ptr<RtpPacketToSend>> GetFecPackets() = 0;
};
} // namespace webrtc
#endif // MODULES_RTP_RTCP_SOURCE_VIDEO_FEC_GENERATOR_H_

1
test/fuzzers/BUILD.gn
View File

@ -149,6 +149,7 @@ webrtc_fuzzer_test("ulpfec_generator_fuzzer") {
"../../modules/rtp_rtcp:rtp_rtcp_format",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../system_wrappers",
]
}

9
test/fuzzers/flexfec_sender_fuzzer.cc
View File

@ -41,7 +41,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
FecProtectionParams params = {
data[i++], static_cast<int>(data[i++] % 100),
data[i++] <= 127 ? kFecMaskRandom : kFecMaskBursty};
sender.SetFecParameters(params);
sender.SetProtectionParameters(params, params);
uint16_t seq_num = data[i++];
while (i + 1 < size) {
@ -59,11 +59,8 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
RtpPacketToSend rtp_packet(nullptr);
if (!rtp_packet.Parse(packet.get(), kRtpHeaderSize + payload_size))
break;
sender.AddRtpPacketAndGenerateFec(rtp_packet);
if (sender.FecAvailable()) {
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
sender.GetFecPackets();
}
sender.AddPacketAndGenerateFec(rtp_packet);
sender.GetFecPackets();
}
}

24
test/fuzzers/ulpfec_generator_fuzzer.cc
View File

@ -16,6 +16,7 @@
#include "modules/rtp_rtcp/source/ulpfec_generator.h"
#include "rtc_base/checks.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
@ -25,13 +26,14 @@ constexpr uint8_t kRedPayloadType = 97;
} // namespace
void FuzzOneInput(const uint8_t* data, size_t size) {
UlpfecGenerator generator;
SimulatedClock clock(1);
UlpfecGenerator generator(kRedPayloadType, kFecPayloadType, &clock);
size_t i = 0;
if (size < 4)
return;
FecProtectionParams params = {
data[i++] % 128, static_cast<int>(data[i++] % 10), kFecMaskBursty};
generator.SetFecParameters(params);
generator.SetProtectionParameters(params, params);
uint16_t seq_num = data[i++];
uint16_t prev_seq_num = 0;
while (i + 3 < size) {
@ -41,6 +43,9 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
break;
rtc::CopyOnWriteBuffer packet(&data[i], payload_size + rtp_header_length);
packet.EnsureCapacity(IP_PACKET_SIZE);
// Write a valid parsable header (version = 2, no padding, no extensions,
// no CSRCs).
ByteWriter<uint8_t>::WriteBigEndian(&packet[0], 2 << 6);
// Make sure sequence numbers are increasing.
ByteWriter<uint16_t>::WriteBigEndian(&packet[2], seq_num++);
i += payload_size + rtp_header_length;
@ -51,16 +56,15 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
// number became out of order.
if (protect && IsNewerSequenceNumber(seq_num, prev_seq_num) &&
seq_num < prev_seq_num + kUlpfecMaxMediaPackets) {
generator.AddRtpPacketAndGenerateFec(packet, rtp_header_length);
RtpPacketToSend rtp_packet(nullptr);
// Check that we actually have a parsable packet, we want to fuzz FEC
// logic, not RTP header parsing.
RTC_CHECK(rtp_packet.Parse(packet));
generator.AddPacketAndGenerateFec(rtp_packet);
prev_seq_num = seq_num;
}
const size_t num_fec_packets = generator.NumAvailableFecPackets();
if (num_fec_packets > 0) {
std::vector<std::unique_ptr<RedPacket>> fec_packets =
generator.GetUlpfecPacketsAsRed(kRedPayloadType, kFecPayloadType,
100);
RTC_CHECK_EQ(num_fec_packets, fec_packets.size());
}
generator.GetFecPackets();
}
}
} // namespace webrtc