admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
webrtc_m130/modules/rtp_rtcp/source/rtp_sender_video.cc
Danil Chapovalov c146b5f77b Represent unset VideoPlayoutDelay with nullopt rather than special value 
Remove support for setting one limit without another limit
because related rtp header extension doesn't support such values.

Start morphing VideoPlayouDelay into a class and stricter type: add accessors returning TimeDelta

Bug: webrtc:13756
Change-Id: If0dd02620528dc870b015beeff3a8103e04022ee
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/315921
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#40570}
2023-08-18 13:17:50 +00:00

875 lines
35 KiB
C++
Raw Blame History

/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/rtp_sender_video.h"
#include <stdlib.h>
#include <string.h>
#include <algorithm>
#include <limits>
#include <memory>
#include <string>
#include <utility>
#include "absl/algorithm/container.h"
#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "api/transport/rtp/dependency_descriptor.h"
#include "api/units/frequency.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "modules/rtp_rtcp/source/rtp_format.h"
#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h"
#include "modules/rtp_rtcp/source/time_util.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/field_trial_parser.h"
#include "rtc_base/logging.h"
#include "rtc_base/trace_event.h"
namespace webrtc {
namespace {
constexpr size_t kRedForFecHeaderLength = 1;
constexpr TimeDelta kMaxUnretransmittableFrameInterval =
TimeDelta::Millis(33 * 4);
void BuildRedPayload(const RtpPacketToSend& media_packet,
RtpPacketToSend* red_packet) {
uint8_t* red_payload = red_packet->AllocatePayload(
kRedForFecHeaderLength + media_packet.payload_size());
RTC_DCHECK(red_payload);
red_payload[0] = media_packet.PayloadType();
auto media_payload = media_packet.payload();
memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
media_payload.size());
}
bool MinimizeDescriptor(RTPVideoHeader* video_header) {
if (auto* vp8 =
absl::get_if<RTPVideoHeaderVP8>(&video_header->video_type_header)) {
// Set minimum fields the RtpPacketizer is using to create vp8 packets.
// nonReference is the only field that doesn't require extra space.
bool non_reference = vp8->nonReference;
vp8->InitRTPVideoHeaderVP8();
vp8->nonReference = non_reference;
return true;
}
return false;
}
bool IsBaseLayer(const RTPVideoHeader& video_header) {
switch (video_header.codec) {
case kVideoCodecVP8: {
const auto& vp8 =
absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
return (vp8.temporalIdx == 0 || vp8.temporalIdx == kNoTemporalIdx);
}
case kVideoCodecVP9: {
const auto& vp9 =
absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
return (vp9.temporal_idx == 0 || vp9.temporal_idx == kNoTemporalIdx);
}
case kVideoCodecH264:
// TODO(kron): Implement logic for H264 once WebRTC supports temporal
// layers for H264.
break;
default:
break;
}
return true;
}
absl::optional<VideoPlayoutDelay> LoadVideoPlayoutDelayOverride(
const FieldTrialsView* key_value_config) {
RTC_DCHECK(key_value_config);
FieldTrialOptional<int> playout_delay_min_ms("min_ms", absl::nullopt);
FieldTrialOptional<int> playout_delay_max_ms("max_ms", absl::nullopt);
ParseFieldTrial({&playout_delay_max_ms, &playout_delay_min_ms},
key_value_config->Lookup("WebRTC-ForceSendPlayoutDelay"));
return playout_delay_max_ms && playout_delay_min_ms
? absl::make_optional<VideoPlayoutDelay>(*playout_delay_min_ms,
*playout_delay_max_ms)
: absl::nullopt;
}
// Some packets can be skipped and the stream can still be decoded. Those
// packets are less likely to be retransmitted if they are lost.
bool PacketWillLikelyBeRequestedForRestransmitionIfLost(
const RTPVideoHeader& video_header) {
return IsBaseLayer(video_header) &&
!(video_header.generic.has_value()
? absl::c_linear_search(
video_header.generic->decode_target_indications,
DecodeTargetIndication::kDiscardable)
: false);
}
} // namespace
RTPSenderVideo::RTPSenderVideo(const Config& config)
: rtp_sender_(config.rtp_sender),
clock_(config.clock),
retransmission_settings_(
config.enable_retransmit_all_layers
? kRetransmitAllLayers
: (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)),
last_rotation_(kVideoRotation_0),
transmit_color_space_next_frame_(false),
send_allocation_(SendVideoLayersAllocation::kDontSend),
playout_delay_pending_(false),
forced_playout_delay_(LoadVideoPlayoutDelayOverride(config.field_trials)),
red_payload_type_(config.red_payload_type),
fec_type_(config.fec_type),
fec_overhead_bytes_(config.fec_overhead_bytes),
post_encode_overhead_bitrate_(/*max_window_size=*/TimeDelta::Seconds(1)),
frame_encryptor_(config.frame_encryptor),
require_frame_encryption_(config.require_frame_encryption),
generic_descriptor_auth_experiment_(!absl::StartsWith(
config.field_trials->Lookup("WebRTC-GenericDescriptorAuth"),
"Disabled")),
absolute_capture_time_sender_(config.clock),
frame_transformer_delegate_(
config.frame_transformer
? rtc::make_ref_counted<RTPSenderVideoFrameTransformerDelegate>(
this,
config.frame_transformer,
rtp_sender_->SSRC(),
config.task_queue_factory)
: nullptr) {
if (frame_transformer_delegate_)
frame_transformer_delegate_->Init();
}
RTPSenderVideo::~RTPSenderVideo() {
if (frame_transformer_delegate_)
frame_transformer_delegate_->Reset();
}
void RTPSenderVideo::LogAndSendToNetwork(
std::vector<std::unique_ptr<RtpPacketToSend>> packets,
size_t encoder_output_size) {
{
MutexLock lock(&stats_mutex_);
size_t packetized_payload_size = 0;
for (const auto& packet : packets) {
if (*packet->packet_type() == RtpPacketMediaType::kVideo) {
packetized_payload_size += packet->payload_size();
}
}
// AV1 and H264 packetizers may produce less packetized bytes than
// unpacketized.
if (packetized_payload_size >= encoder_output_size) {
post_encode_overhead_bitrate_.Update(
packetized_payload_size - encoder_output_size, clock_->CurrentTime());
}
}
rtp_sender_->EnqueuePackets(std::move(packets));
}
size_t RTPSenderVideo::FecPacketOverhead() const {
size_t overhead = fec_overhead_bytes_;
if (red_enabled()) {
// The RED overhead is due to a small header.
overhead += kRedForFecHeaderLength;
if (fec_type_ == VideoFecGenerator::FecType::kUlpFec) {
// For ULPFEC, the overhead is the FEC headers plus RED for FEC header
// (see above) plus anything in RTP header beyond the 12 bytes base header
// (CSRC list, extensions...)
// This reason for the header extensions to be included here is that
// from an FEC viewpoint, they are part of the payload to be protected.
// (The base RTP header is already protected by the FEC header.)
overhead +=
rtp_sender_->FecOrPaddingPacketMaxRtpHeaderLength() - kRtpHeaderSize;
}
}
return overhead;
}
void RTPSenderVideo::SetRetransmissionSetting(int32_t retransmission_settings) {
RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
retransmission_settings_ = retransmission_settings;
}
void RTPSenderVideo::SetVideoStructure(
const FrameDependencyStructure* video_structure) {
if (frame_transformer_delegate_) {
frame_transformer_delegate_->SetVideoStructureUnderLock(video_structure);
return;
}
SetVideoStructureInternal(video_structure);
}
void RTPSenderVideo::SetVideoStructureAfterTransformation(
const FrameDependencyStructure* video_structure) {
SetVideoStructureInternal(video_structure);
}
void RTPSenderVideo::SetVideoStructureInternal(
const FrameDependencyStructure* video_structure) {
RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
if (video_structure == nullptr) {
video_structure_ = nullptr;
return;
}
// Simple sanity checks video structure is set up.
RTC_DCHECK_GT(video_structure->num_decode_targets, 0);
RTC_DCHECK_GT(video_structure->templates.size(), 0);
int structure_id = 0;
if (video_structure_) {
if (*video_structure_ == *video_structure) {
// Same structure (just a new key frame), no update required.
return;
}
// When setting different video structure make sure structure_id is updated
// so that templates from different structures do not collide.
static constexpr int kMaxTemplates = 64;
structure_id =
(video_structure_->structure_id + video_structure_->templates.size()) %
kMaxTemplates;
}
video_structure_ =
std::make_unique<FrameDependencyStructure>(*video_structure);
video_structure_->structure_id = structure_id;
}
void RTPSenderVideo::SetVideoLayersAllocation(
VideoLayersAllocation allocation) {
if (frame_transformer_delegate_) {
frame_transformer_delegate_->SetVideoLayersAllocationUnderLock(
std::move(allocation));
return;
}
SetVideoLayersAllocationInternal(std::move(allocation));
}
void RTPSenderVideo::SetVideoLayersAllocationAfterTransformation(
VideoLayersAllocation allocation) {
SetVideoLayersAllocationInternal(std::move(allocation));
}
void RTPSenderVideo::SetVideoLayersAllocationInternal(
VideoLayersAllocation allocation) {
RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
if (!allocation_ || allocation.active_spatial_layers.size() !=
allocation_->active_spatial_layers.size()) {
send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
} else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) {
send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution;
}
if (send_allocation_ == SendVideoLayersAllocation::kSendWithoutResolution) {
// Check if frame rate changed more than 5fps since the last time the
// extension was sent with frame rate and resolution.
for (size_t i = 0; i < allocation.active_spatial_layers.size(); ++i) {
if (abs(static_cast<int>(
allocation.active_spatial_layers[i].frame_rate_fps) -
static_cast<int>(
last_full_sent_allocation_->active_spatial_layers[i]
.frame_rate_fps)) > 5) {
send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
break;
}
}
}
allocation_ = std::move(allocation);
}
void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
bool first_packet,
bool last_packet,
RtpPacketToSend* packet) const {
// Send color space when changed or if the frame is a key frame. Keep
// sending color space information until the first base layer frame to
// guarantee that the information is retrieved by the receiver.
bool set_color_space =
video_header.color_space != last_color_space_ ||
video_header.frame_type == VideoFrameType::kVideoFrameKey ||
transmit_color_space_next_frame_;
// Color space requires two-byte header extensions if HDR metadata is
// included. Therefore, it's best to add this extension first so that the
// other extensions in the same packet are written as two-byte headers at
// once.
if (last_packet && set_color_space && video_header.color_space)
packet->SetExtension<ColorSpaceExtension>(video_header.color_space.value());
// According to
// http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
// ts_126114v120700p.pdf Section 7.4.5:
// The MTSI client shall add the payload bytes as defined in this clause
// onto the last RTP packet in each group of packets which make up a key
// frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
// (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
// packet in each group of packets which make up another type of frame
// (e.g. a P-Frame) only if the current value is different from the previous
// value sent.
// Set rotation when key frame or when changed (to follow standard).
// Or when different from 0 (to follow current receiver implementation).
bool set_video_rotation =
video_header.frame_type == VideoFrameType::kVideoFrameKey ||
video_header.rotation != last_rotation_ ||
video_header.rotation != kVideoRotation_0;
if (last_packet && set_video_rotation)
packet->SetExtension<VideoOrientation>(video_header.rotation);
// Report content type only for key frames.
if (last_packet &&
video_header.frame_type == VideoFrameType::kVideoFrameKey &&
video_header.content_type != VideoContentType::UNSPECIFIED)
packet->SetExtension<VideoContentTypeExtension>(video_header.content_type);
if (last_packet &&
video_header.video_timing.flags != VideoSendTiming::kInvalid)
packet->SetExtension<VideoTimingExtension>(video_header.video_timing);
// If transmitted, add to all packets; ack logic depends on this.
if (playout_delay_pending_ && current_playout_delay_.has_value()) {
packet->SetExtension<PlayoutDelayLimits>(*current_playout_delay_);
}
if (first_packet && video_header.absolute_capture_time.has_value()) {
packet->SetExtension<AbsoluteCaptureTimeExtension>(
*video_header.absolute_capture_time);
}
if (video_header.generic) {
bool extension_is_set = false;
if (packet->IsRegistered<RtpDependencyDescriptorExtension>() &&
video_structure_ != nullptr) {
DependencyDescriptor descriptor;
descriptor.first_packet_in_frame = first_packet;
descriptor.last_packet_in_frame = last_packet;
descriptor.frame_number = video_header.generic->frame_id & 0xFFFF;
descriptor.frame_dependencies.spatial_id =
video_header.generic->spatial_index;
descriptor.frame_dependencies.temporal_id =
video_header.generic->temporal_index;
for (int64_t dep : video_header.generic->dependencies) {
descriptor.frame_dependencies.frame_diffs.push_back(
video_header.generic->frame_id - dep);
}
descriptor.frame_dependencies.chain_diffs =
video_header.generic->chain_diffs;
descriptor.frame_dependencies.decode_target_indications =
video_header.generic->decode_target_indications;
RTC_DCHECK_EQ(
descriptor.frame_dependencies.decode_target_indications.size(),
video_structure_->num_decode_targets);
if (first_packet) {
descriptor.active_decode_targets_bitmask =
active_decode_targets_tracker_.ActiveDecodeTargetsBitmask();
}
// VP9 mark all layer frames of the first picture as kVideoFrameKey,
// Structure should be attached to the descriptor to lowest spatial layer
// when inter layer dependency is used, i.e. L structures; or to all
// layers when inter layer dependency is not used, i.e. S structures.
// Distinguish these two cases by checking if there are any dependencies.
if (video_header.frame_type == VideoFrameType::kVideoFrameKey &&
video_header.generic->dependencies.empty() && first_packet) {
// To avoid extra structure copy, temporary share ownership of the
// video_structure with the dependency descriptor.
descriptor.attached_structure =
absl::WrapUnique(video_structure_.get());
}
extension_is_set = packet->SetExtension<RtpDependencyDescriptorExtension>(
*video_structure_,
active_decode_targets_tracker_.ActiveChainsBitmask(), descriptor);
// Remove the temporary shared ownership.
descriptor.attached_structure.release();
}
// Do not use generic frame descriptor when dependency descriptor is stored.
if (packet->IsRegistered<RtpGenericFrameDescriptorExtension00>() &&
!extension_is_set) {
RtpGenericFrameDescriptor generic_descriptor;
generic_descriptor.SetFirstPacketInSubFrame(first_packet);
generic_descriptor.SetLastPacketInSubFrame(last_packet);
if (first_packet) {
generic_descriptor.SetFrameId(
static_cast<uint16_t>(video_header.generic->frame_id));
for (int64_t dep : video_header.generic->dependencies) {
generic_descriptor.AddFrameDependencyDiff(
video_header.generic->frame_id - dep);
}
uint8_t spatial_bitmask = 1 << video_header.generic->spatial_index;
generic_descriptor.SetSpatialLayersBitmask(spatial_bitmask);
generic_descriptor.SetTemporalLayer(
video_header.generic->temporal_index);
if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
generic_descriptor.SetResolution(video_header.width,
video_header.height);
}
}
packet->SetExtension<RtpGenericFrameDescriptorExtension00>(
generic_descriptor);
}
}
if (packet->IsRegistered<RtpVideoLayersAllocationExtension>() &&
first_packet &&
send_allocation_ != SendVideoLayersAllocation::kDontSend &&
(video_header.frame_type == VideoFrameType::kVideoFrameKey ||
PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header))) {
VideoLayersAllocation allocation = allocation_.value();
allocation.resolution_and_frame_rate_is_valid =
send_allocation_ == SendVideoLayersAllocation::kSendWithResolution;
packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation);
}
if (first_packet && video_header.video_frame_tracking_id) {
packet->SetExtension<VideoFrameTrackingIdExtension>(
*video_header.video_frame_tracking_id);
}
}
bool RTPSenderVideo::SendVideo(int payload_type,
absl::optional<VideoCodecType> codec_type,
uint32_t rtp_timestamp,
Timestamp capture_time,
rtc::ArrayView<const uint8_t> payload,
size_t encoder_output_size,
RTPVideoHeader video_header,
TimeDelta expected_retransmission_time,
std::vector<uint32_t> csrcs) {
TRACE_EVENT_ASYNC_STEP1(
"webrtc", "Video", capture_time.ms_or(0), "Send", "type",
std::string(VideoFrameTypeToString(video_header.frame_type)));
RTC_CHECK_RUNS_SERIALIZED(&send_checker_);
if (video_header.frame_type == VideoFrameType::kEmptyFrame)
return true;
if (payload.empty())
return false;
if (!rtp_sender_->SendingMedia()) {
return false;
}
int32_t retransmission_settings = retransmission_settings_;
if (codec_type == VideoCodecType::kVideoCodecH264) {
// Backward compatibility for older receivers without temporal layer logic.
retransmission_settings = kRetransmitBaseLayer | kRetransmitHigherLayers;
}
const uint8_t temporal_id = GetTemporalId(video_header);
// TODO(bugs.webrtc.org/10714): retransmission_settings_ should generally be
// replaced by expected_retransmission_time.IsFinite().
const bool allow_retransmission =
expected_retransmission_time.IsFinite() &&
AllowRetransmission(temporal_id, retransmission_settings,
expected_retransmission_time);
MaybeUpdateCurrentPlayoutDelay(video_header);
if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
if (current_playout_delay_.has_value()) {
// Force playout delay on key-frames, if set.
playout_delay_pending_ = true;
}
if (allocation_) {
// Send the bitrate allocation on every key frame.
send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
}
}
if (video_structure_ != nullptr && video_header.generic) {
active_decode_targets_tracker_.OnFrame(
video_structure_->decode_target_protected_by_chain,
video_header.generic->active_decode_targets,
video_header.frame_type == VideoFrameType::kVideoFrameKey,
video_header.generic->frame_id, video_header.generic->chain_diffs);
}
// No FEC protection for upper temporal layers, if used.
const bool use_fec = fec_type_.has_value() &&
(temporal_id == 0 || temporal_id == kNoTemporalIdx);
// Maximum size of packet including rtp headers.
// Extra space left in case packet will be resent using fec or rtx.
int packet_capacity = rtp_sender_->MaxRtpPacketSize();
if (use_fec) {
packet_capacity -= FecPacketOverhead();
}
if (allow_retransmission) {
packet_capacity -= rtp_sender_->RtxPacketOverhead();
}
std::unique_ptr<RtpPacketToSend> single_packet =
rtp_sender_->AllocatePacket(csrcs);
RTC_DCHECK_LE(packet_capacity, single_packet->capacity());
single_packet->SetPayloadType(payload_type);
single_packet->SetTimestamp(rtp_timestamp);
if (capture_time.IsFinite())
single_packet->set_capture_time(capture_time);
// Construct the absolute capture time extension if not provided.
if (!video_header.absolute_capture_time.has_value() &&
capture_time.IsFinite()) {
video_header.absolute_capture_time.emplace();
video_header.absolute_capture_time->absolute_capture_timestamp =
Int64MsToUQ32x32(
clock_->ConvertTimestampToNtpTime(capture_time).ToMs());
video_header.absolute_capture_time->estimated_capture_clock_offset = 0;
}
// Let `absolute_capture_time_sender_` decide if the extension should be sent.
if (video_header.absolute_capture_time.has_value()) {
video_header.absolute_capture_time =
absolute_capture_time_sender_.OnSendPacket(
AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
single_packet->Csrcs()),
single_packet->Timestamp(), kVideoPayloadTypeFrequency,
video_header.absolute_capture_time->absolute_capture_timestamp,
video_header.absolute_capture_time->estimated_capture_clock_offset);
}
auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
// Simplest way to estimate how much extensions would occupy is to set them.
AddRtpHeaderExtensions(video_header,
/*first_packet=*/true, /*last_packet=*/true,
single_packet.get());
if (video_structure_ != nullptr &&
single_packet->IsRegistered<RtpDependencyDescriptorExtension>() &&
!single_packet->HasExtension<RtpDependencyDescriptorExtension>()) {
RTC_DCHECK_EQ(video_header.frame_type, VideoFrameType::kVideoFrameKey);
// Disable attaching dependency descriptor to delta packets (including
// non-first packet of a key frame) when it wasn't attached to a key frame,
// as dependency descriptor can't be usable in such case.
RTC_LOG(LS_WARNING) << "Disable dependency descriptor because failed to "
"attach it to a key frame.";
video_structure_ = nullptr;
}
AddRtpHeaderExtensions(video_header,
/*first_packet=*/true, /*last_packet=*/false,
first_packet.get());
AddRtpHeaderExtensions(video_header,
/*first_packet=*/false, /*last_packet=*/false,
middle_packet.get());
AddRtpHeaderExtensions(video_header,
/*first_packet=*/false, /*last_packet=*/true,
last_packet.get());
RTC_DCHECK_GT(packet_capacity, single_packet->headers_size());
RTC_DCHECK_GT(packet_capacity, first_packet->headers_size());
RTC_DCHECK_GT(packet_capacity, middle_packet->headers_size());
RTC_DCHECK_GT(packet_capacity, last_packet->headers_size());
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = packet_capacity - middle_packet->headers_size();
RTC_DCHECK_GE(single_packet->headers_size(), middle_packet->headers_size());
limits.single_packet_reduction_len =
single_packet->headers_size() - middle_packet->headers_size();
RTC_DCHECK_GE(first_packet->headers_size(), middle_packet->headers_size());
limits.first_packet_reduction_len =
first_packet->headers_size() - middle_packet->headers_size();
RTC_DCHECK_GE(last_packet->headers_size(), middle_packet->headers_size());
limits.last_packet_reduction_len =
last_packet->headers_size() - middle_packet->headers_size();
bool has_generic_descriptor =
first_packet->HasExtension<RtpGenericFrameDescriptorExtension00>() ||
first_packet->HasExtension<RtpDependencyDescriptorExtension>();
// Minimization of the vp8 descriptor may erase temporal_id, so use
// `temporal_id` rather than reference `video_header` beyond this point.
if (has_generic_descriptor) {
MinimizeDescriptor(&video_header);
}
rtc::Buffer encrypted_video_payload;
if (frame_encryptor_ != nullptr) {
const size_t max_ciphertext_size =
frame_encryptor_->GetMaxCiphertextByteSize(cricket::MEDIA_TYPE_VIDEO,
payload.size());
encrypted_video_payload.SetSize(max_ciphertext_size);
size_t bytes_written = 0;
// Enable header authentication if the field trial isn't disabled.
std::vector<uint8_t> additional_data;
if (generic_descriptor_auth_experiment_) {
additional_data = RtpDescriptorAuthentication(video_header);
}
if (frame_encryptor_->Encrypt(
cricket::MEDIA_TYPE_VIDEO, first_packet->Ssrc(), additional_data,
payload, encrypted_video_payload, &bytes_written) != 0) {
return false;
}
encrypted_video_payload.SetSize(bytes_written);
payload = encrypted_video_payload;
} else if (require_frame_encryption_) {
RTC_LOG(LS_WARNING)
<< "No FrameEncryptor is attached to this video sending stream but "
"one is required since require_frame_encryptor is set";
}
std::unique_ptr<RtpPacketizer> packetizer =
RtpPacketizer::Create(codec_type, payload, limits, video_header);
const size_t num_packets = packetizer->NumPackets();
if (num_packets == 0)
return false;
bool first_frame = first_frame_sent_();
std::vector<std::unique_ptr<RtpPacketToSend>> rtp_packets;
for (size_t i = 0; i < num_packets; ++i) {
std::unique_ptr<RtpPacketToSend> packet;
int expected_payload_capacity;
// Choose right packet template:
if (num_packets == 1) {
packet = std::move(single_packet);
expected_payload_capacity =
limits.max_payload_len - limits.single_packet_reduction_len;
} else if (i == 0) {
packet = std::move(first_packet);
expected_payload_capacity =
limits.max_payload_len - limits.first_packet_reduction_len;
} else if (i == num_packets - 1) {
packet = std::move(last_packet);
expected_payload_capacity =
limits.max_payload_len - limits.last_packet_reduction_len;
} else {
packet = std::make_unique<RtpPacketToSend>(*middle_packet);
expected_payload_capacity = limits.max_payload_len;
}
packet->set_first_packet_of_frame(i == 0);
if (!packetizer->NextPacket(packet.get()))
return false;
RTC_DCHECK_LE(packet->payload_size(), expected_payload_capacity);
packet->set_allow_retransmission(allow_retransmission);
packet->set_is_key_frame(video_header.frame_type ==
VideoFrameType::kVideoFrameKey);
// Put packetization finish timestamp into extension.
if (packet->HasExtension<VideoTimingExtension>()) {
packet->set_packetization_finish_time(clock_->CurrentTime());
}
packet->set_fec_protect_packet(use_fec);
if (red_enabled()) {
// TODO(sprang): Consider packetizing directly into packets with the RED
// header already in place, to avoid this copy.
std::unique_ptr<RtpPacketToSend> red_packet(new RtpPacketToSend(*packet));
BuildRedPayload(*packet, red_packet.get());
red_packet->SetPayloadType(*red_payload_type_);
red_packet->set_is_red(true);
// Append `red_packet` instead of `packet` to output.
red_packet->set_packet_type(RtpPacketMediaType::kVideo);
red_packet->set_allow_retransmission(packet->allow_retransmission());
rtp_packets.emplace_back(std::move(red_packet));
} else {
packet->set_packet_type(RtpPacketMediaType::kVideo);
rtp_packets.emplace_back(std::move(packet));
}
if (first_frame) {
if (i == 0) {
RTC_LOG(LS_INFO)
<< "Sent first RTP packet of the first video frame (pre-pacer)";
}
if (i == num_packets - 1) {
RTC_LOG(LS_INFO)
<< "Sent last RTP packet of the first video frame (pre-pacer)";
}
}
}
LogAndSendToNetwork(std::move(rtp_packets), encoder_output_size);
// Update details about the last sent frame.
last_rotation_ = video_header.rotation;
if (video_header.color_space != last_color_space_) {
last_color_space_ = video_header.color_space;
transmit_color_space_next_frame_ = !IsBaseLayer(video_header);
} else {
transmit_color_space_next_frame_ =
transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false;
}
if (video_header.frame_type == VideoFrameType::kVideoFrameKey ||
PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header)) {
// This frame will likely be delivered, no need to populate playout
// delay extensions until it changes again.
playout_delay_pending_ = false;
if (send_allocation_ == SendVideoLayersAllocation::kSendWithResolution) {
last_full_sent_allocation_ = allocation_;
}
send_allocation_ = SendVideoLayersAllocation::kDontSend;
}
TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time.ms_or(0), "timestamp",
rtp_timestamp);
return true;
}
bool RTPSenderVideo::SendEncodedImage(int payload_type,
absl::optional<VideoCodecType> codec_type,
uint32_t rtp_timestamp,
const EncodedImage& encoded_image,
RTPVideoHeader video_header,
TimeDelta expected_retransmission_time) {
if (frame_transformer_delegate_) {
// The frame will be sent async once transformed.
return frame_transformer_delegate_->TransformFrame(
payload_type, codec_type, rtp_timestamp, encoded_image, video_header,
expected_retransmission_time);
}
return SendVideo(payload_type, codec_type, rtp_timestamp,
encoded_image.CaptureTime(), encoded_image,
encoded_image.size(), video_header,
expected_retransmission_time, /*csrcs=*/{});
}
DataRate RTPSenderVideo::PostEncodeOverhead() const {
MutexLock lock(&stats_mutex_);
return post_encode_overhead_bitrate_.Rate(clock_->CurrentTime())
.value_or(DataRate::Zero());
}
bool RTPSenderVideo::AllowRetransmission(
uint8_t temporal_id,
int32_t retransmission_settings,
TimeDelta expected_retransmission_time) {
if (retransmission_settings == kRetransmitOff)
return false;
MutexLock lock(&stats_mutex_);
// Media packet storage.
if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
UpdateConditionalRetransmit(temporal_id, expected_retransmission_time)) {
retransmission_settings |= kRetransmitHigherLayers;
}
if (temporal_id == kNoTemporalIdx)
return true;
if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
return true;
if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
return true;
return false;
}
uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) {
struct TemporalIdGetter {
uint8_t operator()(const RTPVideoHeaderVP8& vp8) { return vp8.temporalIdx; }
uint8_t operator()(const RTPVideoHeaderVP9& vp9) {
return vp9.temporal_idx;
}
uint8_t operator()(const RTPVideoHeaderH264&) { return kNoTemporalIdx; }
uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) {
return kNoTemporalIdx;
}
uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; }
};
return absl::visit(TemporalIdGetter(), header.video_type_header);
}
bool RTPSenderVideo::UpdateConditionalRetransmit(
uint8_t temporal_id,
TimeDelta expected_retransmission_time) {
Timestamp now = clock_->CurrentTime();
// Update stats for any temporal layer.
TemporalLayerStats* current_layer_stats =
&frame_stats_by_temporal_layer_[temporal_id];
current_layer_stats->frame_rate.Update(now);
TimeDelta tl_frame_interval = now - current_layer_stats->last_frame_time;
current_layer_stats->last_frame_time = now;
// Conditional retransmit only applies to upper layers.
if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
if (tl_frame_interval >= kMaxUnretransmittableFrameInterval) {
// Too long since a retransmittable frame in this layer, enable NACK
// protection.
return true;
} else {
// Estimate when the next frame of any lower layer will be sent.
Timestamp expected_next_frame_time = Timestamp::PlusInfinity();
for (int i = temporal_id - 1; i >= 0; --i) {
TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
absl::optional<Frequency> rate = stats->frame_rate.Rate(now);
if (rate > Frequency::Zero()) {
Timestamp tl_next = stats->last_frame_time + 1 / *rate;
if (tl_next - now > -expected_retransmission_time &&
tl_next < expected_next_frame_time) {
expected_next_frame_time = tl_next;
}
}
}
if (expected_next_frame_time - now > expected_retransmission_time) {
// The next frame in a lower layer is expected at a later time (or
// unable to tell due to lack of data) than a retransmission is
// estimated to be able to arrive, so allow this packet to be nacked.
return true;
}
}
}
return false;
}
void RTPSenderVideo::MaybeUpdateCurrentPlayoutDelay(
const RTPVideoHeader& header) {
absl::optional<VideoPlayoutDelay> requested_delay =
forced_playout_delay_.has_value() ? forced_playout_delay_
: header.playout_delay;
if (!requested_delay.has_value()) {
return;
}
if (!requested_delay->Valid()) {
RTC_DLOG(LS_ERROR) << "Requested playout delay values out of order";
return;
}
current_playout_delay_ = requested_delay;
playout_delay_pending_ = true;
}
} // namespace webrtc
Reference in New Issue View Git Blame Copy Permalink