admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
webrtc_m130/modules/rtp_rtcp/source/rtp_sender_video.cc
Marina Ciocea dc69fd2b80 [InsertableStreams] Fix video sender simulcast. 
The transformer was previously moved into the config of the first stream
which resulted in incorrect behavior for simulcast. Use the transformer
in all the streams.

Pass the sender's ssrc on registring the transformed frame callback, to
associate separate transformer sinks for each sender.

Bug: chromium:1065838
Change-Id: I5c52dacb241c68268681b85f875257b24987849e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/173332
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Reviewed-by: Tommi <tommi@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31050}
2020-04-11 10:30:32 +00:00

882 lines
34 KiB
C++
Raw Blame History

/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/rtp_sender_video.h"
#include <stdlib.h>
#include <string.h>
#include <algorithm>
#include <limits>
#include <memory>
#include <string>
#include <utility>
#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "api/transport/rtp/dependency_descriptor.h"
#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "modules/rtp_rtcp/source/rtp_format.h"
#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "modules/rtp_rtcp/source/time_util.h"
#include "modules/rtp_rtcp/source/transformable_encoded_frame.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/trace_event.h"
namespace webrtc {
namespace {
constexpr size_t kRedForFecHeaderLength = 1;
constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
// This is experimental field trial to exclude transport sequence number from
// FEC packets and should only be used in conjunction with datagram transport.
// Datagram transport removes transport sequence numbers from RTP packets and
// uses datagram feedback loop to re-generate RTCP feedback packets, but FEC
// contorol packets are calculated before sequence number is removed and as a
// result recovered packets will be corrupt unless we also remove transport
// sequence number during FEC calculation.
//
// TODO(sukhanov): We need to find a better way to implement FEC with datagram
// transport, probably moving FEC to datagram integration layter. We should
// also remove special field trial once we switch datagram path from
// RTCConfiguration flags to field trial and use the same field trial for FEC
// workaround.
const char kExcludeTransportSequenceNumberFromFecFieldTrial[] =
"WebRTC-ExcludeTransportSequenceNumberFromFec";
void BuildRedPayload(const RtpPacketToSend& media_packet,
RtpPacketToSend* red_packet) {
uint8_t* red_payload = red_packet->AllocatePayload(
kRedForFecHeaderLength + media_packet.payload_size());
RTC_DCHECK(red_payload);
red_payload[0] = media_packet.PayloadType();
auto media_payload = media_packet.payload();
memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
media_payload.size());
}
bool MinimizeDescriptor(RTPVideoHeader* video_header) {
if (auto* vp8 =
absl::get_if<RTPVideoHeaderVP8>(&video_header->video_type_header)) {
// Set minimum fields the RtpPacketizer is using to create vp8 packets.
// nonReference is the only field that doesn't require extra space.
bool non_reference = vp8->nonReference;
vp8->InitRTPVideoHeaderVP8();
vp8->nonReference = non_reference;
return true;
}
// TODO(danilchap): Reduce vp9 codec specific descriptor too.
return false;
}
bool IsBaseLayer(const RTPVideoHeader& video_header) {
switch (video_header.codec) {
case kVideoCodecVP8: {
const auto& vp8 =
absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
return (vp8.temporalIdx == 0 || vp8.temporalIdx == kNoTemporalIdx);
}
case kVideoCodecVP9: {
const auto& vp9 =
absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
return (vp9.temporal_idx == 0 || vp9.temporal_idx == kNoTemporalIdx);
}
case kVideoCodecH264:
// TODO(kron): Implement logic for H264 once WebRTC supports temporal
// layers for H264.
break;
default:
break;
}
return true;
}
#if RTC_TRACE_EVENTS_ENABLED
const char* FrameTypeToString(VideoFrameType frame_type) {
switch (frame_type) {
case VideoFrameType::kEmptyFrame:
return "empty";
case VideoFrameType::kVideoFrameKey:
return "video_key";
case VideoFrameType::kVideoFrameDelta:
return "video_delta";
default:
RTC_NOTREACHED();
return "";
}
}
#endif
bool IsNoopDelay(const PlayoutDelay& delay) {
return delay.min_ms == -1 && delay.max_ms == -1;
}
} // namespace
RTPSenderVideo::RTPSenderVideo(const Config& config)
: rtp_sender_(config.rtp_sender),
clock_(config.clock),
retransmission_settings_(
config.enable_retransmit_all_layers
? kRetransmitAllLayers
: (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)),
last_rotation_(kVideoRotation_0),
transmit_color_space_next_frame_(false),
current_playout_delay_{-1, -1},
playout_delay_pending_(false),
red_payload_type_(config.red_payload_type),
fec_generator_(config.fec_generator),
video_bitrate_(1000, RateStatistics::kBpsScale),
packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale),
frame_encryptor_(config.frame_encryptor),
require_frame_encryption_(config.require_frame_encryption),
generic_descriptor_auth_experiment_(!absl::StartsWith(
config.field_trials->Lookup("WebRTC-GenericDescriptorAuth"),
"Disabled")),
exclude_transport_sequence_number_from_fec_experiment_(absl::StartsWith(
config.field_trials->Lookup(
kExcludeTransportSequenceNumberFromFecFieldTrial),
"Enabled")),
absolute_capture_time_sender_(config.clock),
frame_transformer_delegate_(
config.frame_transformer
? new rtc::RefCountedObject<
RTPSenderVideoFrameTransformerDelegate>(
this,
config.frame_transformer,
rtp_sender_->SSRC())
: nullptr) {
if (frame_transformer_delegate_)
frame_transformer_delegate_->Init();
}
RTPSenderVideo::~RTPSenderVideo() {
if (frame_transformer_delegate_)
frame_transformer_delegate_->Reset();
}
void RTPSenderVideo::LogAndSendToNetwork(
std::vector<std::unique_ptr<RtpPacketToSend>> packets,
size_t unpacketized_payload_size) {
int64_t now_ms = clock_->TimeInMilliseconds();
#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
for (const auto& packet : packets) {
if (packet->packet_type() == RtpPacketMediaType::kForwardErrorCorrection) {
const uint32_t ssrc = packet->Ssrc();
BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoFecBitrate_kbps", now_ms,
FecOverheadRate() / 1000, ssrc);
}
}
#endif
{
rtc::CritScope cs(&stats_crit_);
size_t packetized_payload_size = 0;
for (const auto& packet : packets) {
if (*packet->packet_type() == RtpPacketMediaType::kVideo) {
video_bitrate_.Update(packet->size(), now_ms);
packetized_payload_size += packet->payload_size();
}
}
// AV1 packetizer may produce less packetized bytes than unpacketized.
if (packetized_payload_size >= unpacketized_payload_size) {
packetization_overhead_bitrate_.Update(
packetized_payload_size - unpacketized_payload_size,
clock_->TimeInMilliseconds());
}
}
rtp_sender_->EnqueuePackets(std::move(packets));
}
size_t RTPSenderVideo::FecPacketOverhead() const {
size_t overhead = fec_generator_ ? fec_generator_->MaxPacketOverhead() : 0u;
if (red_enabled()) {
// The RED overhead is due to a small header.
overhead += kRedForFecHeaderLength;
// TODO(bugs.webrtc.org/11340): Move this into UlpfecGenerator.
if (fec_generator_ &&
fec_generator_->GetFecType() == VideoFecGenerator::FecType::kUlpFec) {
// For ULPFEC, the overhead is the FEC headers plus RED for FEC header
// (see above) plus anything in RTP header beyond the 12 bytes base header
// (CSRC list, extensions...)
// This reason for the header extensions to be included here is that
// from an FEC viewpoint, they are part of the payload to be protected.
// (The base RTP header is already protected by the FEC header.)
overhead += rtp_sender_->RtpHeaderLength() - kRtpHeaderSize;
}
}
return overhead;
}
void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params,
const FecProtectionParams& key_params) {
if (fec_generator_) {
fec_generator_->SetProtectionParameters(delta_params, key_params);
}
}
void RTPSenderVideo::SetVideoStructure(
const FrameDependencyStructure* video_structure) {
if (frame_transformer_delegate_) {
frame_transformer_delegate_->SetVideoStructureUnderLock(video_structure);
return;
}
// Lock is being held by SetVideoStructure() caller.
SetVideoStructureUnderLock(video_structure);
}
void RTPSenderVideo::SetVideoStructureUnderLock(
const FrameDependencyStructure* video_structure) {
RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
if (video_structure == nullptr) {
video_structure_ = nullptr;
return;
}
// Simple sanity checks video structure is set up.
RTC_DCHECK_GT(video_structure->num_decode_targets, 0);
RTC_DCHECK_GT(video_structure->templates.size(), 0);
int structure_id = 0;
if (video_structure_) {
if (*video_structure_ == *video_structure) {
// Same structure (just a new key frame), no update required.
return;
}
// When setting different video structure make sure structure_id is updated
// so that templates from different structures do not collide.
static constexpr int kMaxTemplates = 64;
structure_id =
(video_structure_->structure_id + video_structure_->templates.size()) %
kMaxTemplates;
}
video_structure_ =
std::make_unique<FrameDependencyStructure>(*video_structure);
video_structure_->structure_id = structure_id;
// TODO(bugs.webrtc.org/10342): Support chains.
video_structure_->num_chains = 0;
}
void RTPSenderVideo::AddRtpHeaderExtensions(
const RTPVideoHeader& video_header,
const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
bool first_packet,
bool last_packet,
RtpPacketToSend* packet) const {
// Send color space when changed or if the frame is a key frame. Keep
// sending color space information until the first base layer frame to
// guarantee that the information is retrieved by the receiver.
bool set_color_space =
video_header.color_space != last_color_space_ ||
video_header.frame_type == VideoFrameType::kVideoFrameKey ||
transmit_color_space_next_frame_;
// Color space requires two-byte header extensions if HDR metadata is
// included. Therefore, it's best to add this extension first so that the
// other extensions in the same packet are written as two-byte headers at
// once.
if (last_packet && set_color_space && video_header.color_space)
packet->SetExtension<ColorSpaceExtension>(video_header.color_space.value());
// According to
// http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
// ts_126114v120700p.pdf Section 7.4.5:
// The MTSI client shall add the payload bytes as defined in this clause
// onto the last RTP packet in each group of packets which make up a key
// frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
// (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
// packet in each group of packets which make up another type of frame
// (e.g. a P-Frame) only if the current value is different from the previous
// value sent.
// Set rotation when key frame or when changed (to follow standard).
// Or when different from 0 (to follow current receiver implementation).
bool set_video_rotation =
video_header.frame_type == VideoFrameType::kVideoFrameKey ||
video_header.rotation != last_rotation_ ||
video_header.rotation != kVideoRotation_0;
if (last_packet && set_video_rotation)
packet->SetExtension<VideoOrientation>(video_header.rotation);
// Report content type only for key frames.
if (last_packet &&
video_header.frame_type == VideoFrameType::kVideoFrameKey &&
video_header.content_type != VideoContentType::UNSPECIFIED)
packet->SetExtension<VideoContentTypeExtension>(video_header.content_type);
if (last_packet &&
video_header.video_timing.flags != VideoSendTiming::kInvalid)
packet->SetExtension<VideoTimingExtension>(video_header.video_timing);
// If transmitted, add to all packets; ack logic depends on this.
if (playout_delay_pending_) {
packet->SetExtension<PlayoutDelayLimits>(current_playout_delay_);
}
if (first_packet && absolute_capture_time) {
packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time);
}
if (video_header.codec == kVideoCodecH264 &&
video_header.frame_marking.temporal_id != kNoTemporalIdx) {
FrameMarking frame_marking = video_header.frame_marking;
frame_marking.start_of_frame = first_packet;
frame_marking.end_of_frame = last_packet;
packet->SetExtension<FrameMarkingExtension>(frame_marking);
}
if (video_header.generic) {
bool extension_is_set = false;
if (video_structure_ != nullptr) {
DependencyDescriptor descriptor;
descriptor.first_packet_in_frame = first_packet;
descriptor.last_packet_in_frame = last_packet;
descriptor.frame_number = video_header.generic->frame_id & 0xFFFF;
descriptor.frame_dependencies.spatial_id =
video_header.generic->spatial_index;
descriptor.frame_dependencies.temporal_id =
video_header.generic->temporal_index;
for (int64_t dep : video_header.generic->dependencies) {
descriptor.frame_dependencies.frame_diffs.push_back(
video_header.generic->frame_id - dep);
}
descriptor.frame_dependencies.decode_target_indications =
video_header.generic->decode_target_indications;
RTC_DCHECK_EQ(
descriptor.frame_dependencies.decode_target_indications.size(),
video_structure_->num_decode_targets);
// To avoid extra structure copy, temporary share ownership of the
// video_structure with the dependency descriptor.
if (video_header.frame_type == VideoFrameType::kVideoFrameKey &&
first_packet) {
descriptor.attached_structure =
absl::WrapUnique(video_structure_.get());
}
extension_is_set = packet->SetExtension<RtpDependencyDescriptorExtension>(
*video_structure_, descriptor);
// Remove the temporary shared ownership.
descriptor.attached_structure.release();
}
// Do not use v0/v1 generic frame descriptor when v2 is stored.
if (!extension_is_set) {
RtpGenericFrameDescriptor generic_descriptor;
generic_descriptor.SetFirstPacketInSubFrame(first_packet);
generic_descriptor.SetLastPacketInSubFrame(last_packet);
generic_descriptor.SetDiscardable(video_header.generic->discardable);
if (first_packet) {
generic_descriptor.SetFrameId(
static_cast<uint16_t>(video_header.generic->frame_id));
for (int64_t dep : video_header.generic->dependencies) {
generic_descriptor.AddFrameDependencyDiff(
video_header.generic->frame_id - dep);
}
uint8_t spatial_bimask = 1 << video_header.generic->spatial_index;
generic_descriptor.SetSpatialLayersBitmask(spatial_bimask);
generic_descriptor.SetTemporalLayer(
video_header.generic->temporal_index);
if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
generic_descriptor.SetResolution(video_header.width,
video_header.height);
}
}
packet->SetExtension<RtpGenericFrameDescriptorExtension00>(
generic_descriptor);
}
}
}
bool RTPSenderVideo::SendVideo(
int payload_type,
absl::optional<VideoCodecType> codec_type,
uint32_t rtp_timestamp,
int64_t capture_time_ms,
rtc::ArrayView<const uint8_t> payload,
const RTPFragmentationHeader* fragmentation,
RTPVideoHeader video_header,
absl::optional<int64_t> expected_retransmission_time_ms) {
#if RTC_TRACE_EVENTS_ENABLED
TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type",
FrameTypeToString(video_header.frame_type));
#endif
RTC_CHECK_RUNS_SERIALIZED(&send_checker_);
if (video_header.frame_type == VideoFrameType::kEmptyFrame)
return true;
if (payload.empty())
return false;
int32_t retransmission_settings = retransmission_settings_;
if (codec_type == VideoCodecType::kVideoCodecH264) {
// Backward compatibility for older receivers without temporal layer logic.
retransmission_settings = kRetransmitBaseLayer | kRetransmitHigherLayers;
}
MaybeUpdateCurrentPlayoutDelay(video_header);
if (video_header.frame_type == VideoFrameType::kVideoFrameKey &&
!IsNoopDelay(current_playout_delay_)) {
// Force playout delay on key-frames, if set.
playout_delay_pending_ = true;
}
// Maximum size of packet including rtp headers.
// Extra space left in case packet will be resent using fec or rtx.
int packet_capacity = rtp_sender_->MaxRtpPacketSize() - FecPacketOverhead() -
(rtp_sender_->RtxStatus() ? kRtxHeaderSize : 0);
std::unique_ptr<RtpPacketToSend> single_packet =
rtp_sender_->AllocatePacket();
RTC_DCHECK_LE(packet_capacity, single_packet->capacity());
single_packet->SetPayloadType(payload_type);
single_packet->SetTimestamp(rtp_timestamp);
single_packet->set_capture_time_ms(capture_time_ms);
const absl::optional<AbsoluteCaptureTime> absolute_capture_time =
absolute_capture_time_sender_.OnSendPacket(
AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
single_packet->Csrcs()),
single_packet->Timestamp(), kVideoPayloadTypeFrequency,
Int64MsToUQ32x32(single_packet->capture_time_ms() + NtpOffsetMs()),
/*estimated_capture_clock_offset=*/absl::nullopt);
auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
// Simplest way to estimate how much extensions would occupy is to set them.
AddRtpHeaderExtensions(video_header, absolute_capture_time,
/*first_packet=*/true, /*last_packet=*/true,
single_packet.get());
AddRtpHeaderExtensions(video_header, absolute_capture_time,
/*first_packet=*/true, /*last_packet=*/false,
first_packet.get());
AddRtpHeaderExtensions(video_header, absolute_capture_time,
/*first_packet=*/false, /*last_packet=*/false,
middle_packet.get());
AddRtpHeaderExtensions(video_header, absolute_capture_time,
/*first_packet=*/false, /*last_packet=*/true,
last_packet.get());
RTC_DCHECK_GT(packet_capacity, single_packet->headers_size());
RTC_DCHECK_GT(packet_capacity, first_packet->headers_size());
RTC_DCHECK_GT(packet_capacity, middle_packet->headers_size());
RTC_DCHECK_GT(packet_capacity, last_packet->headers_size());
RtpPacketizer::PayloadSizeLimits limits;
limits.max_payload_len = packet_capacity - middle_packet->headers_size();
RTC_DCHECK_GE(single_packet->headers_size(), middle_packet->headers_size());
limits.single_packet_reduction_len =
single_packet->headers_size() - middle_packet->headers_size();
RTC_DCHECK_GE(first_packet->headers_size(), middle_packet->headers_size());
limits.first_packet_reduction_len =
first_packet->headers_size() - middle_packet->headers_size();
RTC_DCHECK_GE(last_packet->headers_size(), middle_packet->headers_size());
limits.last_packet_reduction_len =
last_packet->headers_size() - middle_packet->headers_size();
bool has_generic_descriptor =
first_packet->HasExtension<RtpGenericFrameDescriptorExtension00>() ||
first_packet->HasExtension<RtpDependencyDescriptorExtension>();
// Minimization of the vp8 descriptor may erase temporal_id, so save it.
const uint8_t temporal_id = GetTemporalId(video_header);
if (has_generic_descriptor) {
MinimizeDescriptor(&video_header);
}
// TODO(benwright@webrtc.org) - Allocate enough to always encrypt inline.
rtc::Buffer encrypted_video_payload;
if (frame_encryptor_ != nullptr) {
if (!has_generic_descriptor) {
return false;
}
const size_t max_ciphertext_size =
frame_encryptor_->GetMaxCiphertextByteSize(cricket::MEDIA_TYPE_VIDEO,
payload.size());
encrypted_video_payload.SetSize(max_ciphertext_size);
size_t bytes_written = 0;
// Enable header authentication if the field trial isn't disabled.
std::vector<uint8_t> additional_data;
if (generic_descriptor_auth_experiment_) {
additional_data = RtpDescriptorAuthentication(video_header);
}
if (frame_encryptor_->Encrypt(
cricket::MEDIA_TYPE_VIDEO, first_packet->Ssrc(), additional_data,
payload, encrypted_video_payload, &bytes_written) != 0) {
return false;
}
encrypted_video_payload.SetSize(bytes_written);
payload = encrypted_video_payload;
} else if (require_frame_encryption_) {
RTC_LOG(LS_WARNING)
<< "No FrameEncryptor is attached to this video sending stream but "
"one is required since require_frame_encryptor is set";
}
std::unique_ptr<RtpPacketizer> packetizer = RtpPacketizer::Create(
codec_type, payload, limits, video_header, fragmentation);
// TODO(bugs.webrtc.org/10714): retransmission_settings_ should generally be
// replaced by expected_retransmission_time_ms.has_value(). For now, though,
// only VP8 with an injected frame buffer controller actually controls it.
const bool allow_retransmission =
expected_retransmission_time_ms.has_value()
? AllowRetransmission(temporal_id, retransmission_settings,
expected_retransmission_time_ms.value())
: false;
const size_t num_packets = packetizer->NumPackets();
size_t unpacketized_payload_size;
if (fragmentation && fragmentation->fragmentationVectorSize > 0) {
unpacketized_payload_size = 0;
for (uint16_t i = 0; i < fragmentation->fragmentationVectorSize; ++i) {
unpacketized_payload_size += fragmentation->fragmentationLength[i];
}
} else {
unpacketized_payload_size = payload.size();
}
if (num_packets == 0)
return false;
bool first_frame = first_frame_sent_();
std::vector<std::unique_ptr<RtpPacketToSend>> rtp_packets;
for (size_t i = 0; i < num_packets; ++i) {
std::unique_ptr<RtpPacketToSend> packet;
int expected_payload_capacity;
// Choose right packet template:
if (num_packets == 1) {
packet = std::move(single_packet);
expected_payload_capacity =
limits.max_payload_len - limits.single_packet_reduction_len;
} else if (i == 0) {
packet = std::move(first_packet);
expected_payload_capacity =
limits.max_payload_len - limits.first_packet_reduction_len;
} else if (i == num_packets - 1) {
packet = std::move(last_packet);
expected_payload_capacity =
limits.max_payload_len - limits.last_packet_reduction_len;
} else {
packet = std::make_unique<RtpPacketToSend>(*middle_packet);
expected_payload_capacity = limits.max_payload_len;
}
packet->set_first_packet_of_frame(i == 0);
if (!packetizer->NextPacket(packet.get()))
return false;
RTC_DCHECK_LE(packet->payload_size(), expected_payload_capacity);
if (!rtp_sender_->AssignSequenceNumber(packet.get()))
return false;
// No FEC protection for upper temporal layers, if used.
bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx;
packet->set_allow_retransmission(allow_retransmission);
// Put packetization finish timestamp into extension.
if (packet->HasExtension<VideoTimingExtension>()) {
packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
}
if (protect_packet && fec_generator_) {
if (red_enabled() &&
exclude_transport_sequence_number_from_fec_experiment_) {
// See comments at the top of the file why experiment
// "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in
// conjunction with datagram transport.
// TODO(sukhanov): We may also need to implement it for flexfec_sender
// if we decide to keep this approach in the future.
uint16_t transport_senquence_number;
if (packet->GetExtension<webrtc::TransportSequenceNumber>(
&transport_senquence_number)) {
if (!packet->RemoveExtension(webrtc::TransportSequenceNumber::kId)) {
RTC_NOTREACHED()
<< "Failed to remove transport sequence number, packet="
<< packet->ToString();
}
}
}
fec_generator_->AddPacketAndGenerateFec(*packet);
}
if (red_enabled()) {
std::unique_ptr<RtpPacketToSend> red_packet(new RtpPacketToSend(*packet));
BuildRedPayload(*packet, red_packet.get());
red_packet->SetPayloadType(*red_payload_type_);
// Send |red_packet| instead of |packet| for allocated sequence number.
red_packet->set_packet_type(RtpPacketMediaType::kVideo);
red_packet->set_allow_retransmission(packet->allow_retransmission());
rtp_packets.emplace_back(std::move(red_packet));
} else {
packet->set_packet_type(RtpPacketMediaType::kVideo);
rtp_packets.emplace_back(std::move(packet));
}
if (first_frame) {
if (i == 0) {
RTC_LOG(LS_INFO)
<< "Sent first RTP packet of the first video frame (pre-pacer)";
}
if (i == num_packets - 1) {
RTC_LOG(LS_INFO)
<< "Sent last RTP packet of the first video frame (pre-pacer)";
}
}
}
if (fec_generator_) {
// Fetch any FEC packets generated from the media frame and add them to
// the list of packets to send.
auto fec_packets = fec_generator_->GetFecPackets();
// TODO(bugs.webrtc.org/11340): Move sequence number assignment into
// UlpfecGenerator.
const bool generate_sequence_numbers = !fec_generator_->FecSsrc();
for (auto& fec_packet : fec_packets) {
if (generate_sequence_numbers) {
rtp_sender_->AssignSequenceNumber(fec_packet.get());
}
rtp_packets.emplace_back(std::move(fec_packet));
}
}
LogAndSendToNetwork(std::move(rtp_packets), unpacketized_payload_size);
// Update details about the last sent frame.
last_rotation_ = video_header.rotation;
if (video_header.color_space != last_color_space_) {
last_color_space_ = video_header.color_space;
transmit_color_space_next_frame_ = !IsBaseLayer(video_header);
} else {
transmit_color_space_next_frame_ =
transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false;
}
if (video_header.frame_type == VideoFrameType::kVideoFrameKey ||
(IsBaseLayer(video_header) &&
!(video_header.generic.has_value() ? video_header.generic->discardable
: false))) {
// This frame has guaranteed delivery, no need to populate playout
// delay extensions until it changes again.
playout_delay_pending_ = false;
}
TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
rtp_timestamp);
return true;
}
bool RTPSenderVideo::SendEncodedImage(
int payload_type,
absl::optional<VideoCodecType> codec_type,
uint32_t rtp_timestamp,
const EncodedImage& encoded_image,
const RTPFragmentationHeader* fragmentation,
RTPVideoHeader video_header,
absl::optional<int64_t> expected_retransmission_time_ms) {
if (frame_transformer_delegate_) {
// The frame will be sent async once transformed.
return frame_transformer_delegate_->TransformFrame(
payload_type, codec_type, rtp_timestamp, encoded_image, fragmentation,
video_header, expected_retransmission_time_ms);
}
return SendVideo(payload_type, codec_type, rtp_timestamp,
encoded_image.capture_time_ms_, encoded_image, fragmentation,
video_header, expected_retransmission_time_ms);
}
uint32_t RTPSenderVideo::VideoBitrateSent() const {
rtc::CritScope cs(&stats_crit_);
return video_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
}
uint32_t RTPSenderVideo::FecOverheadRate() const {
return fec_generator_ ? fec_generator_->CurrentFecRate().bps<uint32_t>() : 0u;
}
uint32_t RTPSenderVideo::PacketizationOverheadBps() const {
rtc::CritScope cs(&stats_crit_);
return packetization_overhead_bitrate_.Rate(clock_->TimeInMilliseconds())
.value_or(0);
}
bool RTPSenderVideo::AllowRetransmission(
uint8_t temporal_id,
int32_t retransmission_settings,
int64_t expected_retransmission_time_ms) {
if (retransmission_settings == kRetransmitOff)
return false;
rtc::CritScope cs(&stats_crit_);
// Media packet storage.
if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
UpdateConditionalRetransmit(temporal_id,
expected_retransmission_time_ms)) {
retransmission_settings |= kRetransmitHigherLayers;
}
if (temporal_id == kNoTemporalIdx)
return true;
if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
return true;
if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
return true;
return false;
}
uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) {
struct TemporalIdGetter {
uint8_t operator()(const RTPVideoHeaderVP8& vp8) { return vp8.temporalIdx; }
uint8_t operator()(const RTPVideoHeaderVP9& vp9) {
return vp9.temporal_idx;
}
uint8_t operator()(const RTPVideoHeaderH264&) { return kNoTemporalIdx; }
uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) {
return kNoTemporalIdx;
}
uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; }
};
switch (header.codec) {
case kVideoCodecH264:
return header.frame_marking.temporal_id;
default:
return absl::visit(TemporalIdGetter(), header.video_type_header);
}
}
bool RTPSenderVideo::UpdateConditionalRetransmit(
uint8_t temporal_id,
int64_t expected_retransmission_time_ms) {
int64_t now_ms = clock_->TimeInMilliseconds();
// Update stats for any temporal layer.
TemporalLayerStats* current_layer_stats =
&frame_stats_by_temporal_layer_[temporal_id];
current_layer_stats->frame_rate_fp1000s.Update(1, now_ms);
int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_ms;
current_layer_stats->last_frame_time_ms = now_ms;
// Conditional retransmit only applies to upper layers.
if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
// Too long since a retransmittable frame in this layer, enable NACK
// protection.
return true;
} else {
// Estimate when the next frame of any lower layer will be sent.
const int64_t kUndefined = std::numeric_limits<int64_t>::max();
int64_t expected_next_frame_time = kUndefined;
for (int i = temporal_id - 1; i >= 0; --i) {
TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
absl::optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms);
if (rate) {
int64_t tl_next = stats->last_frame_time_ms + 1000000 / *rate;
if (tl_next - now_ms > -expected_retransmission_time_ms &&
tl_next < expected_next_frame_time) {
expected_next_frame_time = tl_next;
}
}
}
if (expected_next_frame_time == kUndefined ||
expected_next_frame_time - now_ms > expected_retransmission_time_ms) {
// The next frame in a lower layer is expected at a later time (or
// unable to tell due to lack of data) than a retransmission is
// estimated to be able to arrive, so allow this packet to be nacked.
return true;
}
}
}
return false;
}
void RTPSenderVideo::MaybeUpdateCurrentPlayoutDelay(
const RTPVideoHeader& header) {
if (IsNoopDelay(header.playout_delay)) {
return;
}
PlayoutDelay requested_delay = header.playout_delay;
if (requested_delay.min_ms > PlayoutDelayLimits::kMaxMs ||
requested_delay.max_ms > PlayoutDelayLimits::kMaxMs) {
RTC_DLOG(LS_ERROR)
<< "Requested playout delay values out of range, ignored";
return;
}
if (requested_delay.max_ms != -1 &&
requested_delay.min_ms > requested_delay.max_ms) {
RTC_DLOG(LS_ERROR) << "Requested playout delay values out of order";
return;
}
if (!playout_delay_pending_) {
current_playout_delay_ = requested_delay;
playout_delay_pending_ = true;
return;
}
if ((requested_delay.min_ms == -1 ||
requested_delay.min_ms == current_playout_delay_.min_ms) &&
(requested_delay.max_ms == -1 ||
requested_delay.max_ms == current_playout_delay_.max_ms)) {
// No change, ignore.
return;
}
if (requested_delay.min_ms == -1) {
RTC_DCHECK_GE(requested_delay.max_ms, 0);
requested_delay.min_ms =
std::min(current_playout_delay_.min_ms, requested_delay.max_ms);
}
if (requested_delay.max_ms == -1) {
requested_delay.max_ms =
std::max(current_playout_delay_.max_ms, requested_delay.min_ms);
}
current_playout_delay_ = requested_delay;
playout_delay_pending_ = true;
}
} // namespace webrtc
Reference in New Issue View Git Blame Copy Permalink