9dfff29bc4
Prior to this change, FlexFEC packets that were paced would be lost in the RTPSender, since they were not stored in a packet history. This CL introduces such a packet history, as well as the needed wireup for higher layers to be aware that the particular RTPSender is able to send FlexFEC packets with a particular SSRC. Updated RTPSender unit test to reflect the fact that paced packets are now actually sent. BUG=webrtc:5654 Review-Url: https://codereview.webrtc.org/2491293002 Cr-Commit-Position: refs/heads/master@{#15066}
427 lines
16 KiB
C++
427 lines
16 KiB
C++
/*
|
|
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h"
|
|
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <memory>
|
|
#include <vector>
|
|
#include <utility>
|
|
|
|
#include "webrtc/base/checks.h"
|
|
#include "webrtc/base/logging.h"
|
|
#include "webrtc/base/trace_event.h"
|
|
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
|
|
|
|
namespace webrtc {
|
|
|
|
namespace {
|
|
constexpr size_t kRedForFecHeaderLength = 1;
|
|
|
|
void BuildRedPayload(const RtpPacketToSend& media_packet,
|
|
RtpPacketToSend* red_packet) {
|
|
uint8_t* red_payload = red_packet->AllocatePayload(
|
|
kRedForFecHeaderLength + media_packet.payload_size());
|
|
RTC_DCHECK(red_payload);
|
|
red_payload[0] = media_packet.PayloadType();
|
|
memcpy(&red_payload[kRedForFecHeaderLength], media_packet.payload(),
|
|
media_packet.payload_size());
|
|
}
|
|
} // namespace
|
|
|
|
RTPSenderVideo::RTPSenderVideo(Clock* clock,
|
|
RTPSender* rtp_sender,
|
|
FlexfecSender* flexfec_sender)
|
|
: rtp_sender_(rtp_sender),
|
|
clock_(clock),
|
|
video_type_(kRtpVideoGeneric),
|
|
retransmission_settings_(kRetransmitBaseLayer),
|
|
last_rotation_(kVideoRotation_0),
|
|
red_payload_type_(-1),
|
|
ulpfec_payload_type_(-1),
|
|
flexfec_sender_(flexfec_sender),
|
|
delta_fec_params_{0, 1, kFecMaskRandom},
|
|
key_fec_params_{0, 1, kFecMaskRandom},
|
|
fec_bitrate_(1000, RateStatistics::kBpsScale),
|
|
video_bitrate_(1000, RateStatistics::kBpsScale) {}
|
|
|
|
RTPSenderVideo::~RTPSenderVideo() {}
|
|
|
|
void RTPSenderVideo::SetVideoCodecType(RtpVideoCodecTypes video_type) {
|
|
video_type_ = video_type;
|
|
}
|
|
|
|
RtpVideoCodecTypes RTPSenderVideo::VideoCodecType() const {
|
|
return video_type_;
|
|
}
|
|
|
|
// Static.
|
|
RtpUtility::Payload* RTPSenderVideo::CreateVideoPayload(
|
|
const char payload_name[RTP_PAYLOAD_NAME_SIZE],
|
|
int8_t payload_type) {
|
|
RtpVideoCodecTypes video_type = kRtpVideoGeneric;
|
|
if (RtpUtility::StringCompare(payload_name, "VP8", 3)) {
|
|
video_type = kRtpVideoVp8;
|
|
} else if (RtpUtility::StringCompare(payload_name, "VP9", 3)) {
|
|
video_type = kRtpVideoVp9;
|
|
} else if (RtpUtility::StringCompare(payload_name, "H264", 4)) {
|
|
video_type = kRtpVideoH264;
|
|
} else if (RtpUtility::StringCompare(payload_name, "I420", 4)) {
|
|
video_type = kRtpVideoGeneric;
|
|
} else {
|
|
video_type = kRtpVideoGeneric;
|
|
}
|
|
RtpUtility::Payload* payload = new RtpUtility::Payload();
|
|
payload->name[RTP_PAYLOAD_NAME_SIZE - 1] = 0;
|
|
strncpy(payload->name, payload_name, RTP_PAYLOAD_NAME_SIZE - 1);
|
|
payload->typeSpecific.Video.videoCodecType = video_type;
|
|
payload->audio = false;
|
|
return payload;
|
|
}
|
|
|
|
void RTPSenderVideo::SendVideoPacket(std::unique_ptr<RtpPacketToSend> packet,
|
|
StorageType storage) {
|
|
// Remember some values about the packet before sending it away.
|
|
size_t packet_size = packet->size();
|
|
uint16_t seq_num = packet->SequenceNumber();
|
|
uint32_t rtp_timestamp = packet->Timestamp();
|
|
if (!rtp_sender_->SendToNetwork(std::move(packet), storage,
|
|
RtpPacketSender::kLowPriority)) {
|
|
LOG(LS_WARNING) << "Failed to send video packet " << seq_num;
|
|
return;
|
|
}
|
|
rtc::CritScope cs(&stats_crit_);
|
|
video_bitrate_.Update(packet_size, clock_->TimeInMilliseconds());
|
|
TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
|
|
"Video::PacketNormal", "timestamp", rtp_timestamp,
|
|
"seqnum", seq_num);
|
|
}
|
|
|
|
void RTPSenderVideo::SendVideoPacketAsRedMaybeWithUlpfec(
|
|
std::unique_ptr<RtpPacketToSend> media_packet,
|
|
StorageType media_packet_storage,
|
|
bool protect_media_packet) {
|
|
uint32_t rtp_timestamp = media_packet->Timestamp();
|
|
uint16_t media_seq_num = media_packet->SequenceNumber();
|
|
|
|
std::unique_ptr<RtpPacketToSend> red_packet(
|
|
new RtpPacketToSend(*media_packet));
|
|
BuildRedPayload(*media_packet, red_packet.get());
|
|
|
|
std::vector<std::unique_ptr<RedPacket>> fec_packets;
|
|
StorageType fec_storage = kDontRetransmit;
|
|
{
|
|
// Only protect while creating RED and FEC packets, not when sending.
|
|
rtc::CritScope cs(&crit_);
|
|
red_packet->SetPayloadType(red_payload_type_);
|
|
if (ulpfec_enabled()) {
|
|
if (protect_media_packet) {
|
|
ulpfec_generator_.AddRtpPacketAndGenerateFec(
|
|
media_packet->data(), media_packet->payload_size(),
|
|
media_packet->headers_size());
|
|
}
|
|
uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
|
|
if (num_fec_packets > 0) {
|
|
uint16_t first_fec_sequence_number =
|
|
rtp_sender_->AllocateSequenceNumber(num_fec_packets);
|
|
fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed(
|
|
red_payload_type_, ulpfec_payload_type_, first_fec_sequence_number,
|
|
media_packet->headers_size());
|
|
RTC_DCHECK_EQ(num_fec_packets, fec_packets.size());
|
|
if (retransmission_settings_ & kRetransmitFECPackets)
|
|
fec_storage = kAllowRetransmission;
|
|
}
|
|
}
|
|
}
|
|
// Send |red_packet| instead of |packet| for allocated sequence number.
|
|
size_t red_packet_size = red_packet->size();
|
|
if (rtp_sender_->SendToNetwork(std::move(red_packet), media_packet_storage,
|
|
RtpPacketSender::kLowPriority)) {
|
|
rtc::CritScope cs(&stats_crit_);
|
|
video_bitrate_.Update(red_packet_size, clock_->TimeInMilliseconds());
|
|
TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
|
|
"Video::PacketRed", "timestamp", rtp_timestamp,
|
|
"seqnum", media_seq_num);
|
|
} else {
|
|
LOG(LS_WARNING) << "Failed to send RED packet " << media_seq_num;
|
|
}
|
|
for (const auto& fec_packet : fec_packets) {
|
|
// TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid
|
|
// reparsing them.
|
|
std::unique_ptr<RtpPacketToSend> rtp_packet(
|
|
new RtpPacketToSend(*media_packet));
|
|
RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length()));
|
|
rtp_packet->set_capture_time_ms(media_packet->capture_time_ms());
|
|
uint16_t fec_sequence_number = rtp_packet->SequenceNumber();
|
|
if (rtp_sender_->SendToNetwork(std::move(rtp_packet), fec_storage,
|
|
RtpPacketSender::kLowPriority)) {
|
|
rtc::CritScope cs(&stats_crit_);
|
|
fec_bitrate_.Update(fec_packet->length(), clock_->TimeInMilliseconds());
|
|
TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
|
|
"Video::PacketUlpfec", "timestamp", rtp_timestamp,
|
|
"seqnum", fec_sequence_number);
|
|
} else {
|
|
LOG(LS_WARNING) << "Failed to send ULPFEC packet " << fec_sequence_number;
|
|
}
|
|
}
|
|
}
|
|
|
|
void RTPSenderVideo::SendVideoPacketWithFlexfec(
|
|
std::unique_ptr<RtpPacketToSend> media_packet,
|
|
StorageType media_packet_storage,
|
|
bool protect_media_packet) {
|
|
RTC_DCHECK(flexfec_sender_);
|
|
|
|
if (protect_media_packet)
|
|
flexfec_sender_->AddRtpPacketAndGenerateFec(*media_packet);
|
|
|
|
SendVideoPacket(std::move(media_packet), media_packet_storage);
|
|
|
|
if (flexfec_sender_->FecAvailable()) {
|
|
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
|
|
flexfec_sender_->GetFecPackets();
|
|
for (auto& fec_packet : fec_packets) {
|
|
uint32_t timestamp = fec_packet->Timestamp();
|
|
uint16_t seq_num = fec_packet->SequenceNumber();
|
|
if (rtp_sender_->SendToNetwork(std::move(fec_packet), kDontRetransmit,
|
|
RtpPacketSender::kLowPriority)) {
|
|
// TODO(brandtr): Wire up stats here.
|
|
TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
|
|
"Video::PacketFlexfec", "timestamp", timestamp,
|
|
"seqnum", seq_num);
|
|
} else {
|
|
LOG(LS_WARNING) << "Failed to send FlexFEC packet " << seq_num;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void RTPSenderVideo::SetUlpfecConfig(int red_payload_type,
|
|
int ulpfec_payload_type) {
|
|
// Sanity check. Per the definition of UlpfecConfig (see config.h),
|
|
// a payload type of -1 means that the corresponding feature is
|
|
// turned off.
|
|
RTC_DCHECK_GE(red_payload_type, -1);
|
|
RTC_DCHECK_LE(red_payload_type, 127);
|
|
RTC_DCHECK_GE(ulpfec_payload_type, -1);
|
|
RTC_DCHECK_LE(ulpfec_payload_type, 127);
|
|
|
|
rtc::CritScope cs(&crit_);
|
|
red_payload_type_ = red_payload_type;
|
|
ulpfec_payload_type_ = ulpfec_payload_type;
|
|
|
|
// Must not enable ULPFEC without RED.
|
|
// TODO(brandtr): We currently support enabling RED without ULPFEC. Change
|
|
// this when we have removed the RED/RTX send-side workaround, so that we
|
|
// ensure that RED and ULPFEC are only enabled together.
|
|
RTC_DCHECK(red_enabled() || !ulpfec_enabled());
|
|
|
|
// Reset FEC parameters.
|
|
delta_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom};
|
|
key_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom};
|
|
}
|
|
|
|
void RTPSenderVideo::GetUlpfecConfig(int* red_payload_type,
|
|
int* ulpfec_payload_type) const {
|
|
rtc::CritScope cs(&crit_);
|
|
*red_payload_type = red_payload_type_;
|
|
*ulpfec_payload_type = ulpfec_payload_type_;
|
|
}
|
|
|
|
size_t RTPSenderVideo::CalculateFecPacketOverhead() const {
|
|
if (flexfec_enabled())
|
|
return flexfec_sender_->MaxPacketOverhead();
|
|
|
|
size_t overhead = 0;
|
|
if (red_enabled()) {
|
|
// The RED overhead is due to a small header.
|
|
overhead += kRedForFecHeaderLength;
|
|
}
|
|
if (ulpfec_enabled()) {
|
|
// For ULPFEC, the overhead is the FEC headers plus RED for FEC header
|
|
// (see above) plus anything in RTP header beyond the 12 bytes base header
|
|
// (CSRC list, extensions...)
|
|
// This reason for the header extensions to be included here is that
|
|
// from an FEC viewpoint, they are part of the payload to be protected.
|
|
// (The base RTP header is already protected by the FEC header.)
|
|
overhead += ulpfec_generator_.MaxPacketOverhead() +
|
|
(rtp_sender_->RtpHeaderLength() - kRtpHeaderSize);
|
|
}
|
|
return overhead;
|
|
}
|
|
|
|
void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params,
|
|
const FecProtectionParams& key_params) {
|
|
rtc::CritScope cs(&crit_);
|
|
delta_fec_params_ = delta_params;
|
|
key_fec_params_ = key_params;
|
|
}
|
|
|
|
rtc::Optional<uint32_t> RTPSenderVideo::FlexfecSsrc() const {
|
|
if (flexfec_sender_) {
|
|
return rtc::Optional<uint32_t>(flexfec_sender_->ssrc());
|
|
}
|
|
return rtc::Optional<uint32_t>();
|
|
}
|
|
|
|
bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
|
|
FrameType frame_type,
|
|
int8_t payload_type,
|
|
uint32_t rtp_timestamp,
|
|
int64_t capture_time_ms,
|
|
const uint8_t* payload_data,
|
|
size_t payload_size,
|
|
const RTPFragmentationHeader* fragmentation,
|
|
const RTPVideoHeader* video_header) {
|
|
if (payload_size == 0)
|
|
return false;
|
|
|
|
// Create header that will be reused in all packets.
|
|
std::unique_ptr<RtpPacketToSend> rtp_header = rtp_sender_->AllocatePacket();
|
|
rtp_header->SetPayloadType(payload_type);
|
|
rtp_header->SetTimestamp(rtp_timestamp);
|
|
rtp_header->set_capture_time_ms(capture_time_ms);
|
|
|
|
size_t fec_packet_overhead;
|
|
bool red_enabled;
|
|
int32_t retransmission_settings;
|
|
{
|
|
rtc::CritScope cs(&crit_);
|
|
// According to
|
|
// http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
|
|
// ts_126114v120700p.pdf Section 7.4.5:
|
|
// The MTSI client shall add the payload bytes as defined in this clause
|
|
// onto the last RTP packet in each group of packets which make up a key
|
|
// frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
|
|
// (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
|
|
// packet in each group of packets which make up another type of frame
|
|
// (e.g. a P-Frame) only if the current value is different from the previous
|
|
// value sent.
|
|
if (video_header) {
|
|
// Set rotation when key frame or when changed (to follow standard).
|
|
// Or when different from 0 (to follow current receiver implementation).
|
|
VideoRotation current_rotation = video_header->rotation;
|
|
if (frame_type == kVideoFrameKey || current_rotation != last_rotation_ ||
|
|
current_rotation != kVideoRotation_0)
|
|
rtp_header->SetExtension<VideoOrientation>(current_rotation);
|
|
last_rotation_ = current_rotation;
|
|
}
|
|
|
|
// FEC settings.
|
|
const FecProtectionParams& fec_params =
|
|
frame_type == kVideoFrameKey ? key_fec_params_ : delta_fec_params_;
|
|
if (flexfec_enabled())
|
|
flexfec_sender_->SetFecParameters(fec_params);
|
|
if (ulpfec_enabled())
|
|
ulpfec_generator_.SetFecParameters(fec_params);
|
|
|
|
fec_packet_overhead = CalculateFecPacketOverhead();
|
|
red_enabled = this->red_enabled();
|
|
retransmission_settings = retransmission_settings_;
|
|
}
|
|
|
|
size_t packet_capacity = rtp_sender_->MaxPayloadLength() -
|
|
fec_packet_overhead -
|
|
(rtp_sender_->RtxStatus() ? kRtxHeaderSize : 0);
|
|
RTC_DCHECK_LE(packet_capacity, rtp_header->capacity());
|
|
RTC_DCHECK_GT(packet_capacity, rtp_header->headers_size());
|
|
size_t max_data_payload_length = packet_capacity - rtp_header->headers_size();
|
|
|
|
std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create(
|
|
video_type, max_data_payload_length,
|
|
video_header ? &(video_header->codecHeader) : nullptr, frame_type));
|
|
// Media packet storage.
|
|
StorageType storage = packetizer->GetStorageType(retransmission_settings);
|
|
|
|
// TODO(changbin): we currently don't support to configure the codec to
|
|
// output multiple partitions for VP8. Should remove below check after the
|
|
// issue is fixed.
|
|
const RTPFragmentationHeader* frag =
|
|
(video_type == kRtpVideoVp8) ? nullptr : fragmentation;
|
|
packetizer->SetPayloadData(payload_data, payload_size, frag);
|
|
|
|
bool first_frame = first_frame_sent_();
|
|
bool first = true;
|
|
bool last = false;
|
|
while (!last) {
|
|
std::unique_ptr<RtpPacketToSend> packet(new RtpPacketToSend(*rtp_header));
|
|
uint8_t* payload = packet->AllocatePayload(max_data_payload_length);
|
|
RTC_DCHECK(payload);
|
|
|
|
size_t payload_bytes_in_packet = 0;
|
|
if (!packetizer->NextPacket(payload, &payload_bytes_in_packet, &last))
|
|
return false;
|
|
|
|
packet->SetPayloadSize(payload_bytes_in_packet);
|
|
packet->SetMarker(last);
|
|
if (!rtp_sender_->AssignSequenceNumber(packet.get()))
|
|
return false;
|
|
|
|
const bool protect_packet =
|
|
(packetizer->GetProtectionType() == kProtectedPacket);
|
|
if (flexfec_enabled()) {
|
|
// TODO(brandtr): Remove the FlexFEC code path when FlexfecSender
|
|
// is wired up to PacedSender instead.
|
|
SendVideoPacketWithFlexfec(std::move(packet), storage, protect_packet);
|
|
} else if (red_enabled) {
|
|
SendVideoPacketAsRedMaybeWithUlpfec(std::move(packet), storage,
|
|
protect_packet);
|
|
} else {
|
|
SendVideoPacket(std::move(packet), storage);
|
|
}
|
|
|
|
if (first_frame) {
|
|
if (first) {
|
|
LOG(LS_INFO)
|
|
<< "Sent first RTP packet of the first video frame (pre-pacer)";
|
|
}
|
|
if (last) {
|
|
LOG(LS_INFO)
|
|
<< "Sent last RTP packet of the first video frame (pre-pacer)";
|
|
}
|
|
}
|
|
first = false;
|
|
}
|
|
|
|
TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
|
|
rtp_timestamp);
|
|
return true;
|
|
}
|
|
|
|
uint32_t RTPSenderVideo::VideoBitrateSent() const {
|
|
rtc::CritScope cs(&stats_crit_);
|
|
return video_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
|
|
}
|
|
|
|
uint32_t RTPSenderVideo::FecOverheadRate() const {
|
|
rtc::CritScope cs(&stats_crit_);
|
|
return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
|
|
}
|
|
|
|
int RTPSenderVideo::SelectiveRetransmissions() const {
|
|
rtc::CritScope cs(&crit_);
|
|
return retransmission_settings_;
|
|
}
|
|
|
|
void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
|
|
rtc::CritScope cs(&crit_);
|
|
retransmission_settings_ = settings;
|
|
}
|
|
|
|
} // namespace webrtc
|