284542b882
This lets the RTP code be unaware of lower layers, and the SetTransportOverhead method is deleted from RTPSender and RtpRtcp. Instead, that method is added to CongestionController and TransportFeedbackAdapter, where it is more appropriate. BUG=wertc:6847 Review-Url: https://codereview.webrtc.org/2589743002 Cr-Commit-Position: refs/heads/master@{#15995}
425 lines
16 KiB
C++
425 lines
16 KiB
C++
/*
|
|
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h"
|
|
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <memory>
|
|
#include <vector>
|
|
#include <utility>
|
|
|
|
#include "webrtc/base/checks.h"
|
|
#include "webrtc/base/logging.h"
|
|
#include "webrtc/base/trace_event.h"
|
|
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h"
|
|
#include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
|
|
|
|
namespace webrtc {
|
|
|
|
namespace {
|
|
constexpr size_t kRedForFecHeaderLength = 1;
|
|
|
|
void BuildRedPayload(const RtpPacketToSend& media_packet,
|
|
RtpPacketToSend* red_packet) {
|
|
uint8_t* red_payload = red_packet->AllocatePayload(
|
|
kRedForFecHeaderLength + media_packet.payload_size());
|
|
RTC_DCHECK(red_payload);
|
|
red_payload[0] = media_packet.PayloadType();
|
|
|
|
auto media_payload = media_packet.payload();
|
|
memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
|
|
media_payload.size());
|
|
}
|
|
} // namespace
|
|
|
|
RTPSenderVideo::RTPSenderVideo(Clock* clock,
|
|
RTPSender* rtp_sender,
|
|
FlexfecSender* flexfec_sender)
|
|
: rtp_sender_(rtp_sender),
|
|
clock_(clock),
|
|
video_type_(kRtpVideoGeneric),
|
|
retransmission_settings_(kRetransmitBaseLayer),
|
|
last_rotation_(kVideoRotation_0),
|
|
red_payload_type_(-1),
|
|
ulpfec_payload_type_(-1),
|
|
flexfec_sender_(flexfec_sender),
|
|
delta_fec_params_{0, 1, kFecMaskRandom},
|
|
key_fec_params_{0, 1, kFecMaskRandom},
|
|
fec_bitrate_(1000, RateStatistics::kBpsScale),
|
|
video_bitrate_(1000, RateStatistics::kBpsScale) {}
|
|
|
|
RTPSenderVideo::~RTPSenderVideo() {}
|
|
|
|
void RTPSenderVideo::SetVideoCodecType(RtpVideoCodecTypes video_type) {
|
|
video_type_ = video_type;
|
|
}
|
|
|
|
RtpVideoCodecTypes RTPSenderVideo::VideoCodecType() const {
|
|
return video_type_;
|
|
}
|
|
|
|
// Static.
|
|
RtpUtility::Payload* RTPSenderVideo::CreateVideoPayload(
|
|
const char payload_name[RTP_PAYLOAD_NAME_SIZE],
|
|
int8_t payload_type) {
|
|
RtpVideoCodecTypes video_type = kRtpVideoGeneric;
|
|
if (RtpUtility::StringCompare(payload_name, "VP8", 3)) {
|
|
video_type = kRtpVideoVp8;
|
|
} else if (RtpUtility::StringCompare(payload_name, "VP9", 3)) {
|
|
video_type = kRtpVideoVp9;
|
|
} else if (RtpUtility::StringCompare(payload_name, "H264", 4)) {
|
|
video_type = kRtpVideoH264;
|
|
} else if (RtpUtility::StringCompare(payload_name, "I420", 4)) {
|
|
video_type = kRtpVideoGeneric;
|
|
} else {
|
|
video_type = kRtpVideoGeneric;
|
|
}
|
|
RtpUtility::Payload* payload = new RtpUtility::Payload();
|
|
payload->name[RTP_PAYLOAD_NAME_SIZE - 1] = 0;
|
|
strncpy(payload->name, payload_name, RTP_PAYLOAD_NAME_SIZE - 1);
|
|
payload->typeSpecific.Video.videoCodecType = video_type;
|
|
payload->audio = false;
|
|
return payload;
|
|
}
|
|
|
|
void RTPSenderVideo::SendVideoPacket(std::unique_ptr<RtpPacketToSend> packet,
|
|
StorageType storage) {
|
|
// Remember some values about the packet before sending it away.
|
|
size_t packet_size = packet->size();
|
|
uint16_t seq_num = packet->SequenceNumber();
|
|
uint32_t rtp_timestamp = packet->Timestamp();
|
|
if (!rtp_sender_->SendToNetwork(std::move(packet), storage,
|
|
RtpPacketSender::kLowPriority)) {
|
|
LOG(LS_WARNING) << "Failed to send video packet " << seq_num;
|
|
return;
|
|
}
|
|
rtc::CritScope cs(&stats_crit_);
|
|
video_bitrate_.Update(packet_size, clock_->TimeInMilliseconds());
|
|
TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
|
|
"Video::PacketNormal", "timestamp", rtp_timestamp,
|
|
"seqnum", seq_num);
|
|
}
|
|
|
|
void RTPSenderVideo::SendVideoPacketAsRedMaybeWithUlpfec(
|
|
std::unique_ptr<RtpPacketToSend> media_packet,
|
|
StorageType media_packet_storage,
|
|
bool protect_media_packet) {
|
|
uint32_t rtp_timestamp = media_packet->Timestamp();
|
|
uint16_t media_seq_num = media_packet->SequenceNumber();
|
|
|
|
std::unique_ptr<RtpPacketToSend> red_packet(
|
|
new RtpPacketToSend(*media_packet));
|
|
BuildRedPayload(*media_packet, red_packet.get());
|
|
|
|
std::vector<std::unique_ptr<RedPacket>> fec_packets;
|
|
StorageType fec_storage = kDontRetransmit;
|
|
{
|
|
// Only protect while creating RED and FEC packets, not when sending.
|
|
rtc::CritScope cs(&crit_);
|
|
red_packet->SetPayloadType(red_payload_type_);
|
|
if (ulpfec_enabled()) {
|
|
if (protect_media_packet) {
|
|
ulpfec_generator_.AddRtpPacketAndGenerateFec(
|
|
media_packet->data(), media_packet->payload_size(),
|
|
media_packet->headers_size());
|
|
}
|
|
uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
|
|
if (num_fec_packets > 0) {
|
|
uint16_t first_fec_sequence_number =
|
|
rtp_sender_->AllocateSequenceNumber(num_fec_packets);
|
|
fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed(
|
|
red_payload_type_, ulpfec_payload_type_, first_fec_sequence_number,
|
|
media_packet->headers_size());
|
|
RTC_DCHECK_EQ(num_fec_packets, fec_packets.size());
|
|
if (retransmission_settings_ & kRetransmitFECPackets)
|
|
fec_storage = kAllowRetransmission;
|
|
}
|
|
}
|
|
}
|
|
// Send |red_packet| instead of |packet| for allocated sequence number.
|
|
size_t red_packet_size = red_packet->size();
|
|
if (rtp_sender_->SendToNetwork(std::move(red_packet), media_packet_storage,
|
|
RtpPacketSender::kLowPriority)) {
|
|
rtc::CritScope cs(&stats_crit_);
|
|
video_bitrate_.Update(red_packet_size, clock_->TimeInMilliseconds());
|
|
TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
|
|
"Video::PacketRed", "timestamp", rtp_timestamp,
|
|
"seqnum", media_seq_num);
|
|
} else {
|
|
LOG(LS_WARNING) << "Failed to send RED packet " << media_seq_num;
|
|
}
|
|
for (const auto& fec_packet : fec_packets) {
|
|
// TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid
|
|
// reparsing them.
|
|
std::unique_ptr<RtpPacketToSend> rtp_packet(
|
|
new RtpPacketToSend(*media_packet));
|
|
RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length()));
|
|
rtp_packet->set_capture_time_ms(media_packet->capture_time_ms());
|
|
uint16_t fec_sequence_number = rtp_packet->SequenceNumber();
|
|
if (rtp_sender_->SendToNetwork(std::move(rtp_packet), fec_storage,
|
|
RtpPacketSender::kLowPriority)) {
|
|
rtc::CritScope cs(&stats_crit_);
|
|
fec_bitrate_.Update(fec_packet->length(), clock_->TimeInMilliseconds());
|
|
TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
|
|
"Video::PacketUlpfec", "timestamp", rtp_timestamp,
|
|
"seqnum", fec_sequence_number);
|
|
} else {
|
|
LOG(LS_WARNING) << "Failed to send ULPFEC packet " << fec_sequence_number;
|
|
}
|
|
}
|
|
}
|
|
|
|
void RTPSenderVideo::SendVideoPacketWithFlexfec(
|
|
std::unique_ptr<RtpPacketToSend> media_packet,
|
|
StorageType media_packet_storage,
|
|
bool protect_media_packet) {
|
|
RTC_DCHECK(flexfec_sender_);
|
|
|
|
if (protect_media_packet)
|
|
flexfec_sender_->AddRtpPacketAndGenerateFec(*media_packet);
|
|
|
|
SendVideoPacket(std::move(media_packet), media_packet_storage);
|
|
|
|
if (flexfec_sender_->FecAvailable()) {
|
|
std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
|
|
flexfec_sender_->GetFecPackets();
|
|
for (auto& fec_packet : fec_packets) {
|
|
uint32_t timestamp = fec_packet->Timestamp();
|
|
uint16_t seq_num = fec_packet->SequenceNumber();
|
|
if (rtp_sender_->SendToNetwork(std::move(fec_packet), kDontRetransmit,
|
|
RtpPacketSender::kLowPriority)) {
|
|
// TODO(brandtr): Wire up stats here.
|
|
TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
|
|
"Video::PacketFlexfec", "timestamp", timestamp,
|
|
"seqnum", seq_num);
|
|
} else {
|
|
LOG(LS_WARNING) << "Failed to send FlexFEC packet " << seq_num;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void RTPSenderVideo::SetUlpfecConfig(int red_payload_type,
|
|
int ulpfec_payload_type) {
|
|
// Sanity check. Per the definition of UlpfecConfig (see config.h),
|
|
// a payload type of -1 means that the corresponding feature is
|
|
// turned off.
|
|
RTC_DCHECK_GE(red_payload_type, -1);
|
|
RTC_DCHECK_LE(red_payload_type, 127);
|
|
RTC_DCHECK_GE(ulpfec_payload_type, -1);
|
|
RTC_DCHECK_LE(ulpfec_payload_type, 127);
|
|
|
|
rtc::CritScope cs(&crit_);
|
|
red_payload_type_ = red_payload_type;
|
|
ulpfec_payload_type_ = ulpfec_payload_type;
|
|
|
|
// Must not enable ULPFEC without RED.
|
|
// TODO(brandtr): We currently support enabling RED without ULPFEC. Change
|
|
// this when we have removed the RED/RTX send-side workaround, so that we
|
|
// ensure that RED and ULPFEC are only enabled together.
|
|
RTC_DCHECK(red_enabled() || !ulpfec_enabled());
|
|
|
|
// Reset FEC parameters.
|
|
delta_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom};
|
|
key_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom};
|
|
}
|
|
|
|
void RTPSenderVideo::GetUlpfecConfig(int* red_payload_type,
|
|
int* ulpfec_payload_type) const {
|
|
rtc::CritScope cs(&crit_);
|
|
*red_payload_type = red_payload_type_;
|
|
*ulpfec_payload_type = ulpfec_payload_type_;
|
|
}
|
|
|
|
size_t RTPSenderVideo::CalculateFecPacketOverhead() const {
|
|
if (flexfec_enabled())
|
|
return flexfec_sender_->MaxPacketOverhead();
|
|
|
|
size_t overhead = 0;
|
|
if (red_enabled()) {
|
|
// The RED overhead is due to a small header.
|
|
overhead += kRedForFecHeaderLength;
|
|
}
|
|
if (ulpfec_enabled()) {
|
|
// For ULPFEC, the overhead is the FEC headers plus RED for FEC header
|
|
// (see above) plus anything in RTP header beyond the 12 bytes base header
|
|
// (CSRC list, extensions...)
|
|
// This reason for the header extensions to be included here is that
|
|
// from an FEC viewpoint, they are part of the payload to be protected.
|
|
// (The base RTP header is already protected by the FEC header.)
|
|
overhead += ulpfec_generator_.MaxPacketOverhead() +
|
|
(rtp_sender_->RtpHeaderLength() - kRtpHeaderSize);
|
|
}
|
|
return overhead;
|
|
}
|
|
|
|
void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params,
|
|
const FecProtectionParams& key_params) {
|
|
rtc::CritScope cs(&crit_);
|
|
delta_fec_params_ = delta_params;
|
|
key_fec_params_ = key_params;
|
|
}
|
|
|
|
rtc::Optional<uint32_t> RTPSenderVideo::FlexfecSsrc() const {
|
|
if (flexfec_sender_) {
|
|
return rtc::Optional<uint32_t>(flexfec_sender_->ssrc());
|
|
}
|
|
return rtc::Optional<uint32_t>();
|
|
}
|
|
|
|
bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
|
|
FrameType frame_type,
|
|
int8_t payload_type,
|
|
uint32_t rtp_timestamp,
|
|
int64_t capture_time_ms,
|
|
const uint8_t* payload_data,
|
|
size_t payload_size,
|
|
const RTPFragmentationHeader* fragmentation,
|
|
const RTPVideoHeader* video_header) {
|
|
if (payload_size == 0)
|
|
return false;
|
|
|
|
// Create header that will be reused in all packets.
|
|
std::unique_ptr<RtpPacketToSend> rtp_header = rtp_sender_->AllocatePacket();
|
|
rtp_header->SetPayloadType(payload_type);
|
|
rtp_header->SetTimestamp(rtp_timestamp);
|
|
rtp_header->set_capture_time_ms(capture_time_ms);
|
|
|
|
size_t fec_packet_overhead;
|
|
bool red_enabled;
|
|
int32_t retransmission_settings;
|
|
{
|
|
rtc::CritScope cs(&crit_);
|
|
// According to
|
|
// http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
|
|
// ts_126114v120700p.pdf Section 7.4.5:
|
|
// The MTSI client shall add the payload bytes as defined in this clause
|
|
// onto the last RTP packet in each group of packets which make up a key
|
|
// frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
|
|
// (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
|
|
// packet in each group of packets which make up another type of frame
|
|
// (e.g. a P-Frame) only if the current value is different from the previous
|
|
// value sent.
|
|
if (video_header) {
|
|
// Set rotation when key frame or when changed (to follow standard).
|
|
// Or when different from 0 (to follow current receiver implementation).
|
|
VideoRotation current_rotation = video_header->rotation;
|
|
if (frame_type == kVideoFrameKey || current_rotation != last_rotation_ ||
|
|
current_rotation != kVideoRotation_0)
|
|
rtp_header->SetExtension<VideoOrientation>(current_rotation);
|
|
last_rotation_ = current_rotation;
|
|
}
|
|
|
|
// FEC settings.
|
|
const FecProtectionParams& fec_params =
|
|
frame_type == kVideoFrameKey ? key_fec_params_ : delta_fec_params_;
|
|
if (flexfec_enabled())
|
|
flexfec_sender_->SetFecParameters(fec_params);
|
|
if (ulpfec_enabled())
|
|
ulpfec_generator_.SetFecParameters(fec_params);
|
|
|
|
fec_packet_overhead = CalculateFecPacketOverhead();
|
|
red_enabled = this->red_enabled();
|
|
retransmission_settings = retransmission_settings_;
|
|
}
|
|
|
|
size_t packet_capacity = rtp_sender_->MaxRtpPacketSize() -
|
|
fec_packet_overhead -
|
|
(rtp_sender_->RtxStatus() ? kRtxHeaderSize : 0);
|
|
RTC_DCHECK_LE(packet_capacity, rtp_header->capacity());
|
|
RTC_DCHECK_GT(packet_capacity, rtp_header->headers_size());
|
|
size_t max_data_payload_length = packet_capacity - rtp_header->headers_size();
|
|
|
|
std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create(
|
|
video_type, max_data_payload_length,
|
|
video_header ? &(video_header->codecHeader) : nullptr, frame_type));
|
|
// Media packet storage.
|
|
StorageType storage = packetizer->GetStorageType(retransmission_settings);
|
|
|
|
// TODO(changbin): we currently don't support to configure the codec to
|
|
// output multiple partitions for VP8. Should remove below check after the
|
|
// issue is fixed.
|
|
const RTPFragmentationHeader* frag =
|
|
(video_type == kRtpVideoVp8) ? nullptr : fragmentation;
|
|
packetizer->SetPayloadData(payload_data, payload_size, frag);
|
|
|
|
bool first_frame = first_frame_sent_();
|
|
bool first = true;
|
|
bool last = false;
|
|
while (!last) {
|
|
std::unique_ptr<RtpPacketToSend> packet(new RtpPacketToSend(*rtp_header));
|
|
|
|
if (!packetizer->NextPacket(packet.get(), &last))
|
|
return false;
|
|
RTC_DCHECK_LE(packet->payload_size(), max_data_payload_length);
|
|
|
|
if (!rtp_sender_->AssignSequenceNumber(packet.get()))
|
|
return false;
|
|
|
|
const bool protect_packet =
|
|
(packetizer->GetProtectionType() == kProtectedPacket);
|
|
if (flexfec_enabled()) {
|
|
// TODO(brandtr): Remove the FlexFEC code path when FlexfecSender
|
|
// is wired up to PacedSender instead.
|
|
SendVideoPacketWithFlexfec(std::move(packet), storage, protect_packet);
|
|
} else if (red_enabled) {
|
|
SendVideoPacketAsRedMaybeWithUlpfec(std::move(packet), storage,
|
|
protect_packet);
|
|
} else {
|
|
SendVideoPacket(std::move(packet), storage);
|
|
}
|
|
|
|
if (first_frame) {
|
|
if (first) {
|
|
LOG(LS_INFO)
|
|
<< "Sent first RTP packet of the first video frame (pre-pacer)";
|
|
}
|
|
if (last) {
|
|
LOG(LS_INFO)
|
|
<< "Sent last RTP packet of the first video frame (pre-pacer)";
|
|
}
|
|
}
|
|
first = false;
|
|
}
|
|
|
|
TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
|
|
rtp_timestamp);
|
|
return true;
|
|
}
|
|
|
|
uint32_t RTPSenderVideo::VideoBitrateSent() const {
|
|
rtc::CritScope cs(&stats_crit_);
|
|
return video_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
|
|
}
|
|
|
|
uint32_t RTPSenderVideo::FecOverheadRate() const {
|
|
rtc::CritScope cs(&stats_crit_);
|
|
return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
|
|
}
|
|
|
|
int RTPSenderVideo::SelectiveRetransmissions() const {
|
|
rtc::CritScope cs(&crit_);
|
|
return retransmission_settings_;
|
|
}
|
|
|
|
void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
|
|
rtc::CritScope cs(&crit_);
|
|
retransmission_settings_ = settings;
|
|
}
|
|
|
|
} // namespace webrtc
|