webrtc_m130/modules/rtp_rtcp/source/rtp_sender_video.cc

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/rtp_rtcp/source/rtp_sender_video.h"

#include <stdlib.h>
#include <string.h>

#include <limits>
#include <memory>
#include <string>
#include <utility>

#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/byte_io.h"
#include "modules/rtp_rtcp/source/rtp_format.h"
#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/trace_event.h"

namespace webrtc {

namespace {
constexpr size_t kRedForFecHeaderLength = 1;
constexpr size_t kRtpSequenceNumberMapMaxEntries = 1 << 13;
constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;

// This is experimental field trial to exclude transport sequence number from
// FEC packets and should only be used in conjunction with datagram transport.
// Datagram transport removes transport sequence numbers from RTP packets and
// uses datagram feedback loop to re-generate RTCP feedback packets, but FEC
// contorol packets are calculated before sequence number is removed and as a
// result recovered packets will be corrupt unless we also remove transport
// sequence number during FEC calculation.
//
// TODO(sukhanov): We need to find find better way to implement FEC with
// datagram transport, probably moving FEC to datagram integration layter. We
// should also remove special field trial once we switch datagram path from
// RTCConfiguration flags to field trial and use the same field trial for FEC
// workaround.
const char kExcludeTransportSequenceNumberFromFecFieldTrial[] =
    "WebRTC-ExcludeTransportSequenceNumberFromFec";

void BuildRedPayload(const RtpPacketToSend& media_packet,
                     RtpPacketToSend* red_packet) {
  uint8_t* red_payload = red_packet->AllocatePayload(
      kRedForFecHeaderLength + media_packet.payload_size());
  RTC_DCHECK(red_payload);
  red_payload[0] = media_packet.PayloadType();

  auto media_payload = media_packet.payload();
  memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
         media_payload.size());
}

void AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
                            const absl::optional<PlayoutDelay>& playout_delay,
                            VideoFrameType frame_type,
                            bool set_video_rotation,
                            bool set_color_space,
                            bool set_frame_marking,
                            bool first_packet,
                            bool last_packet,
                            RtpPacketToSend* packet) {
  // Color space requires two-byte header extensions if HDR metadata is
  // included. Therefore, it's best to add this extension first so that the
  // other extensions in the same packet are written as two-byte headers at
  // once.
  if (last_packet && set_color_space && video_header.color_space)
    packet->SetExtension<ColorSpaceExtension>(video_header.color_space.value());

  if (last_packet && set_video_rotation)
    packet->SetExtension<VideoOrientation>(video_header.rotation);

  // Report content type only for key frames.
  if (last_packet && frame_type == VideoFrameType::kVideoFrameKey &&
      video_header.content_type != VideoContentType::UNSPECIFIED)
    packet->SetExtension<VideoContentTypeExtension>(video_header.content_type);

  if (last_packet &&
      video_header.video_timing.flags != VideoSendTiming::kInvalid)
    packet->SetExtension<VideoTimingExtension>(video_header.video_timing);

  // If transmitted, add to all packets; ack logic depends on this.
  if (playout_delay) {
    packet->SetExtension<PlayoutDelayLimits>(*playout_delay);
  }

  if (set_frame_marking) {
    FrameMarking frame_marking = video_header.frame_marking;
    frame_marking.start_of_frame = first_packet;
    frame_marking.end_of_frame = last_packet;
    packet->SetExtension<FrameMarkingExtension>(frame_marking);
  }

  if (video_header.generic) {
    RtpGenericFrameDescriptor generic_descriptor;
    generic_descriptor.SetFirstPacketInSubFrame(first_packet);
    generic_descriptor.SetLastPacketInSubFrame(last_packet);
    generic_descriptor.SetDiscardable(video_header.generic->discardable);

    if (first_packet) {
      generic_descriptor.SetFrameId(
          static_cast<uint16_t>(video_header.generic->frame_id));
      for (int64_t dep : video_header.generic->dependencies) {
        generic_descriptor.AddFrameDependencyDiff(
            video_header.generic->frame_id - dep);
      }

      uint8_t spatial_bimask = 1 << video_header.generic->spatial_index;
      for (int layer : video_header.generic->higher_spatial_layers) {
        RTC_DCHECK_GT(layer, video_header.generic->spatial_index);
        RTC_DCHECK_LT(layer, 8);
        spatial_bimask |= 1 << layer;
      }
      generic_descriptor.SetSpatialLayersBitmask(spatial_bimask);

      generic_descriptor.SetTemporalLayer(video_header.generic->temporal_index);

      if (frame_type == VideoFrameType::kVideoFrameKey) {
        generic_descriptor.SetResolution(video_header.width,
                                         video_header.height);
      }
    }

    if (!packet->SetExtension<RtpGenericFrameDescriptorExtension01>(
            generic_descriptor)) {
      packet->SetExtension<RtpGenericFrameDescriptorExtension00>(
          generic_descriptor);
    }
  }
}

bool MinimizeDescriptor(const RTPVideoHeader& full, RTPVideoHeader* minimized) {
  if (full.codec == VideoCodecType::kVideoCodecVP8) {
    minimized->codec = VideoCodecType::kVideoCodecVP8;
    const auto& vp8 = absl::get<RTPVideoHeaderVP8>(full.video_type_header);
    // Set minimum fields the RtpPacketizer is using to create vp8 packets.
    auto& min_vp8 = minimized->video_type_header.emplace<RTPVideoHeaderVP8>();
    min_vp8.InitRTPVideoHeaderVP8();
    min_vp8.nonReference = vp8.nonReference;
    return true;
  }
  // TODO(danilchap): Reduce vp9 codec specific descriptor too.
  return false;
}

bool IsBaseLayer(const RTPVideoHeader& video_header) {
  switch (video_header.codec) {
    case kVideoCodecVP8: {
      const auto& vp8 =
          absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
      return (vp8.temporalIdx == 0 || vp8.temporalIdx == kNoTemporalIdx);
    }
    case kVideoCodecVP9: {
      const auto& vp9 =
          absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
      return (vp9.temporal_idx == 0 || vp9.temporal_idx == kNoTemporalIdx);
    }
    case kVideoCodecH264:
      // TODO(kron): Implement logic for H264 once WebRTC supports temporal
      // layers for H264.
      break;
    default:
      break;
  }
  return true;
}

const char* FrameTypeToString(VideoFrameType frame_type) {
  switch (frame_type) {
    case VideoFrameType::kEmptyFrame:
      return "empty";
    case VideoFrameType::kVideoFrameKey:
      return "video_key";
    case VideoFrameType::kVideoFrameDelta:
      return "video_delta";
    default:
      RTC_NOTREACHED();
      return "";
  }
}

}  // namespace

RTPSenderVideo::RTPSenderVideo(Clock* clock,
                               RTPSender* rtp_sender,
                               FlexfecSender* flexfec_sender,
                               PlayoutDelayOracle* playout_delay_oracle,
                               FrameEncryptorInterface* frame_encryptor,
                               bool require_frame_encryption,
                               bool need_rtp_packet_infos,
                               const WebRtcKeyValueConfig& field_trials)
    : rtp_sender_(rtp_sender),
      clock_(clock),
      retransmission_settings_(kRetransmitBaseLayer |
                               kConditionallyRetransmitHigherLayers),
      last_rotation_(kVideoRotation_0),
      transmit_color_space_next_frame_(false),
      playout_delay_oracle_(playout_delay_oracle),
      rtp_sequence_number_map_(need_rtp_packet_infos
                                   ? absl::make_unique<RtpSequenceNumberMap>(
                                         kRtpSequenceNumberMapMaxEntries)
                                   : nullptr),
      red_payload_type_(-1),
      ulpfec_payload_type_(-1),
      flexfec_sender_(flexfec_sender),
      delta_fec_params_{0, 1, kFecMaskRandom},
      key_fec_params_{0, 1, kFecMaskRandom},
      fec_bitrate_(1000, RateStatistics::kBpsScale),
      video_bitrate_(1000, RateStatistics::kBpsScale),
      packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale),
      frame_encryptor_(frame_encryptor),
      require_frame_encryption_(require_frame_encryption),
      generic_descriptor_auth_experiment_(
          field_trials.Lookup("WebRTC-GenericDescriptorAuth").find("Enabled") ==
          0),
      exclude_transport_sequence_number_from_fec_experiment_(
          field_trials.Lookup(kExcludeTransportSequenceNumberFromFecFieldTrial)
              .find("Enabled") == 0) {
  RTC_DCHECK(playout_delay_oracle_);
}

RTPSenderVideo::~RTPSenderVideo() {}

void RTPSenderVideo::RegisterPayloadType(int8_t payload_type,
                                         absl::string_view payload_name,
                                         bool raw_payload) {
  absl::optional<VideoCodecType> video_type;
  if (!raw_payload) {
    if (absl::EqualsIgnoreCase(payload_name, "VP8")) {
      video_type = kVideoCodecVP8;
    } else if (absl::EqualsIgnoreCase(payload_name, "VP9")) {
      video_type = kVideoCodecVP9;
    } else if (absl::EqualsIgnoreCase(payload_name, "H264")) {
      video_type = kVideoCodecH264;
    } else {
      video_type = kVideoCodecGeneric;
    }
  }

  {
    rtc::CritScope cs(&payload_type_crit_);
    payload_type_map_[payload_type] = video_type;
  }

  // Backward compatibility for older receivers without temporal layer logic
  if (absl::EqualsIgnoreCase(payload_name, "H264")) {
    rtc::CritScope cs(&crit_);
    retransmission_settings_ = kRetransmitBaseLayer | kRetransmitHigherLayers;
  }
}

void RTPSenderVideo::SendVideoPacket(std::unique_ptr<RtpPacketToSend> packet,
                                     StorageType storage) {
  // Remember some values about the packet before sending it away.
  size_t packet_size = packet->size();
  uint16_t seq_num = packet->SequenceNumber();
  packet->set_packet_type(RtpPacketToSend::Type::kVideo);
  if (!LogAndSendToNetwork(std::move(packet), storage)) {
    RTC_LOG(LS_WARNING) << "Failed to send video packet " << seq_num;
    return;
  }
  rtc::CritScope cs(&stats_crit_);
  video_bitrate_.Update(packet_size, clock_->TimeInMilliseconds());
}

void RTPSenderVideo::SendVideoPacketAsRedMaybeWithUlpfec(
    std::unique_ptr<RtpPacketToSend> media_packet,
    StorageType media_packet_storage,
    bool protect_media_packet) {
  uint16_t media_seq_num = media_packet->SequenceNumber();

  std::unique_ptr<RtpPacketToSend> red_packet(
      new RtpPacketToSend(*media_packet));
  BuildRedPayload(*media_packet, red_packet.get());

  std::vector<std::unique_ptr<RedPacket>> fec_packets;
  {
    // Only protect while creating RED and FEC packets, not when sending.
    rtc::CritScope cs(&crit_);
    red_packet->SetPayloadType(red_payload_type_);
    if (ulpfec_enabled()) {
      if (protect_media_packet) {
        if (exclude_transport_sequence_number_from_fec_experiment_) {
          // See comments at the top of the file why experiment
          // "WebRTC-kExcludeTransportSequenceNumberFromFec" is needed in
          // conjunction with datagram transport.
          // TODO(sukhanov): We may also need to implement it for flexfec_sender
          // if we decide to keep this approach in the future.
          uint16_t transport_senquence_number;
          if (media_packet->GetExtension<webrtc::TransportSequenceNumber>(
                  &transport_senquence_number)) {
            if (!media_packet->RemoveExtension(
                    webrtc::TransportSequenceNumber::kId)) {
              RTC_NOTREACHED()
                  << "Failed to remove transport sequence number, packet="
                  << media_packet->ToString();
            }
          }
        }

        ulpfec_generator_.AddRtpPacketAndGenerateFec(
            media_packet->data(), media_packet->payload_size(),
            media_packet->headers_size());
      }
      uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
      if (num_fec_packets > 0) {
        uint16_t first_fec_sequence_number =
            rtp_sender_->AllocateSequenceNumber(num_fec_packets);
        fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed(
            red_payload_type_, ulpfec_payload_type_, first_fec_sequence_number);
        RTC_DCHECK_EQ(num_fec_packets, fec_packets.size());
      }
    }
  }
  // Send |red_packet| instead of |packet| for allocated sequence number.
  size_t red_packet_size = red_packet->size();
  red_packet->set_packet_type(RtpPacketToSend::Type::kVideo);
  if (LogAndSendToNetwork(std::move(red_packet), media_packet_storage)) {
    rtc::CritScope cs(&stats_crit_);
    video_bitrate_.Update(red_packet_size, clock_->TimeInMilliseconds());
  } else {
    RTC_LOG(LS_WARNING) << "Failed to send RED packet " << media_seq_num;
  }
  for (const auto& fec_packet : fec_packets) {
    // TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid
    // reparsing them.
    std::unique_ptr<RtpPacketToSend> rtp_packet(
        new RtpPacketToSend(*media_packet));
    RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length()));
    rtp_packet->set_capture_time_ms(media_packet->capture_time_ms());
    rtp_packet->set_packet_type(RtpPacketToSend::Type::kForwardErrorCorrection);
    uint16_t fec_sequence_number = rtp_packet->SequenceNumber();
    if (LogAndSendToNetwork(std::move(rtp_packet), kDontRetransmit)) {
      rtc::CritScope cs(&stats_crit_);
      fec_bitrate_.Update(fec_packet->length(), clock_->TimeInMilliseconds());
    } else {
      RTC_LOG(LS_WARNING) << "Failed to send ULPFEC packet "
                          << fec_sequence_number;
    }
  }
}

void RTPSenderVideo::SendVideoPacketWithFlexfec(
    std::unique_ptr<RtpPacketToSend> media_packet,
    StorageType media_packet_storage,
    bool protect_media_packet) {
  RTC_DCHECK(flexfec_sender_);

  if (protect_media_packet)
    flexfec_sender_->AddRtpPacketAndGenerateFec(*media_packet);

  SendVideoPacket(std::move(media_packet), media_packet_storage);

  if (flexfec_sender_->FecAvailable()) {
    std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
        flexfec_sender_->GetFecPackets();
    for (auto& fec_packet : fec_packets) {
      size_t packet_length = fec_packet->size();
      uint16_t seq_num = fec_packet->SequenceNumber();
      fec_packet->set_packet_type(
          RtpPacketToSend::Type::kForwardErrorCorrection);
      if (LogAndSendToNetwork(std::move(fec_packet), kDontRetransmit)) {
        rtc::CritScope cs(&stats_crit_);
        fec_bitrate_.Update(packet_length, clock_->TimeInMilliseconds());
      } else {
        RTC_LOG(LS_WARNING) << "Failed to send FlexFEC packet " << seq_num;
      }
    }
  }
}

bool RTPSenderVideo::LogAndSendToNetwork(
    std::unique_ptr<RtpPacketToSend> packet,
    StorageType storage) {
#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
  int64_t now_ms = clock_->TimeInMilliseconds();
  BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoTotBitrate_kbps", now_ms,
                                  rtp_sender_->ActualSendBitrateKbit(),
                                  packet->Ssrc());
  BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoFecBitrate_kbps", now_ms,
                                  FecOverheadRate() / 1000, packet->Ssrc());
  BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoNackBitrate_kbps", now_ms,
                                  rtp_sender_->NackOverheadRate() / 1000,
                                  packet->Ssrc());
#endif
  return rtp_sender_->SendToNetwork(std::move(packet), storage);
}

void RTPSenderVideo::SetUlpfecConfig(int red_payload_type,
                                     int ulpfec_payload_type) {
  // Sanity check. Per the definition of UlpfecConfig (see config.h),
  // a payload type of -1 means that the corresponding feature is
  // turned off.
  RTC_DCHECK_GE(red_payload_type, -1);
  RTC_DCHECK_LE(red_payload_type, 127);
  RTC_DCHECK_GE(ulpfec_payload_type, -1);
  RTC_DCHECK_LE(ulpfec_payload_type, 127);

  rtc::CritScope cs(&crit_);
  red_payload_type_ = red_payload_type;
  ulpfec_payload_type_ = ulpfec_payload_type;

  // Must not enable ULPFEC without RED.
  RTC_DCHECK(!(red_enabled() ^ ulpfec_enabled()));

  // Reset FEC parameters.
  delta_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom};
  key_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom};
}

size_t RTPSenderVideo::CalculateFecPacketOverhead() const {
  if (flexfec_enabled())
    return flexfec_sender_->MaxPacketOverhead();

  size_t overhead = 0;
  if (red_enabled()) {
    // The RED overhead is due to a small header.
    overhead += kRedForFecHeaderLength;
  }
  if (ulpfec_enabled()) {
    // For ULPFEC, the overhead is the FEC headers plus RED for FEC header
    // (see above) plus anything in RTP header beyond the 12 bytes base header
    // (CSRC list, extensions...)
    // This reason for the header extensions to be included here is that
    // from an FEC viewpoint, they are part of the payload to be protected.
    // (The base RTP header is already protected by the FEC header.)
    overhead += ulpfec_generator_.MaxPacketOverhead() +
                (rtp_sender_->RtpHeaderLength() - kRtpHeaderSize);
  }
  return overhead;
}

void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params,
                                      const FecProtectionParams& key_params) {
  rtc::CritScope cs(&crit_);
  delta_fec_params_ = delta_params;
  key_fec_params_ = key_params;
}

absl::optional<uint32_t> RTPSenderVideo::FlexfecSsrc() const {
  if (flexfec_sender_) {
    return flexfec_sender_->ssrc();
  }
  return absl::nullopt;
}

bool RTPSenderVideo::SendVideo(
    VideoFrameType frame_type,
    int8_t payload_type,
    uint32_t rtp_timestamp,
    int64_t capture_time_ms,
    const uint8_t* payload_data,
    size_t payload_size,
    const RTPFragmentationHeader* fragmentation,
    const RTPVideoHeader* video_header,
    absl::optional<int64_t> expected_retransmission_time_ms) {
  TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type",
                          FrameTypeToString(frame_type));

  if (frame_type == VideoFrameType::kEmptyFrame)
    return true;

  if (payload_size == 0)
    return false;
  RTC_CHECK(video_header);

  size_t fec_packet_overhead;
  bool red_enabled;
  int32_t retransmission_settings;
  bool set_video_rotation;
  bool set_color_space = false;
  bool set_frame_marking =
      video_header->codec == kVideoCodecH264 &&
      video_header->frame_marking.temporal_id != kNoTemporalIdx;

  const absl::optional<PlayoutDelay> playout_delay =
      playout_delay_oracle_->PlayoutDelayToSend(video_header->playout_delay);
  {
    rtc::CritScope cs(&crit_);
    // According to
    // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
    // ts_126114v120700p.pdf Section 7.4.5:
    // The MTSI client shall add the payload bytes as defined in this clause
    // onto the last RTP packet in each group of packets which make up a key
    // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
    // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
    // packet in each group of packets which make up another type of frame
    // (e.g. a P-Frame) only if the current value is different from the previous
    // value sent.
    // Set rotation when key frame or when changed (to follow standard).
    // Or when different from 0 (to follow current receiver implementation).
    set_video_rotation = frame_type == VideoFrameType::kVideoFrameKey ||
                         video_header->rotation != last_rotation_ ||
                         video_header->rotation != kVideoRotation_0;
    last_rotation_ = video_header->rotation;

    // Send color space when changed or if the frame is a key frame. Keep
    // sending color space information until the first base layer frame to
    // guarantee that the information is retrieved by the receiver.
    if (video_header->color_space != last_color_space_) {
      last_color_space_ = video_header->color_space;
      set_color_space = true;
      transmit_color_space_next_frame_ = !IsBaseLayer(*video_header);
    } else {
      set_color_space = frame_type == VideoFrameType::kVideoFrameKey ||
                        transmit_color_space_next_frame_;
      transmit_color_space_next_frame_ = transmit_color_space_next_frame_
                                             ? !IsBaseLayer(*video_header)
                                             : false;
    }

    // FEC settings.
    const FecProtectionParams& fec_params =
        frame_type == VideoFrameType::kVideoFrameKey ? key_fec_params_
                                                     : delta_fec_params_;
    if (flexfec_enabled())
      flexfec_sender_->SetFecParameters(fec_params);
    if (ulpfec_enabled())
      ulpfec_generator_.SetFecParameters(fec_params);

    fec_packet_overhead = CalculateFecPacketOverhead();
    red_enabled = this->red_enabled();
    retransmission_settings = retransmission_settings_;
  }

  // Maximum size of packet including rtp headers.
  // Extra space left in case packet will be resent using fec or rtx.
  int packet_capacity = rtp_sender_->MaxRtpPacketSize() - fec_packet_overhead -
                        (rtp_sender_->RtxStatus() ? kRtxHeaderSize : 0);

  std::unique_ptr<RtpPacketToSend> single_packet =
      rtp_sender_->AllocatePacket();
  RTC_DCHECK_LE(packet_capacity, single_packet->capacity());
  single_packet->SetPayloadType(payload_type);
  single_packet->SetTimestamp(rtp_timestamp);
  single_packet->set_capture_time_ms(capture_time_ms);

  auto first_packet = absl::make_unique<RtpPacketToSend>(*single_packet);
  auto middle_packet = absl::make_unique<RtpPacketToSend>(*single_packet);
  auto last_packet = absl::make_unique<RtpPacketToSend>(*single_packet);
  // Simplest way to estimate how much extensions would occupy is to set them.
  AddRtpHeaderExtensions(*video_header, playout_delay, frame_type,
                         set_video_rotation, set_color_space, set_frame_marking,
                         /*first=*/true, /*last=*/true, single_packet.get());
  AddRtpHeaderExtensions(*video_header, playout_delay, frame_type,
                         set_video_rotation, set_color_space, set_frame_marking,
                         /*first=*/true, /*last=*/false, first_packet.get());
  AddRtpHeaderExtensions(*video_header, playout_delay, frame_type,
                         set_video_rotation, set_color_space, set_frame_marking,
                         /*first=*/false, /*last=*/false, middle_packet.get());
  AddRtpHeaderExtensions(*video_header, playout_delay, frame_type,
                         set_video_rotation, set_color_space, set_frame_marking,
                         /*first=*/false, /*last=*/true, last_packet.get());

  RTC_DCHECK_GT(packet_capacity, single_packet->headers_size());
  RTC_DCHECK_GT(packet_capacity, first_packet->headers_size());
  RTC_DCHECK_GT(packet_capacity, middle_packet->headers_size());
  RTC_DCHECK_GT(packet_capacity, last_packet->headers_size());
  RtpPacketizer::PayloadSizeLimits limits;
  limits.max_payload_len = packet_capacity - middle_packet->headers_size();

  RTC_DCHECK_GE(single_packet->headers_size(), middle_packet->headers_size());
  limits.single_packet_reduction_len =
      single_packet->headers_size() - middle_packet->headers_size();

  RTC_DCHECK_GE(first_packet->headers_size(), middle_packet->headers_size());
  limits.first_packet_reduction_len =
      first_packet->headers_size() - middle_packet->headers_size();

  RTC_DCHECK_GE(last_packet->headers_size(), middle_packet->headers_size());
  limits.last_packet_reduction_len =
      last_packet->headers_size() - middle_packet->headers_size();

  RTPVideoHeader minimized_video_header;
  const RTPVideoHeader* packetize_video_header = video_header;

  rtc::ArrayView<const uint8_t> generic_descriptor_raw_00 =
      first_packet->GetRawExtension<RtpGenericFrameDescriptorExtension00>();
  rtc::ArrayView<const uint8_t> generic_descriptor_raw_01 =
      first_packet->GetRawExtension<RtpGenericFrameDescriptorExtension01>();

  if (!generic_descriptor_raw_00.empty() &&
      !generic_descriptor_raw_01.empty()) {
    RTC_LOG(LS_WARNING) << "Two versions of GFD extension used.";
    return false;
  }

  rtc::ArrayView<const uint8_t> generic_descriptor_raw =
      !generic_descriptor_raw_01.empty() ? generic_descriptor_raw_01
                                         : generic_descriptor_raw_00;
  if (!generic_descriptor_raw.empty()) {
    if (MinimizeDescriptor(*video_header, &minimized_video_header)) {
      packetize_video_header = &minimized_video_header;
    }
  }

  // TODO(benwright@webrtc.org) - Allocate enough to always encrypt inline.
  rtc::Buffer encrypted_video_payload;
  if (frame_encryptor_ != nullptr) {
    if (generic_descriptor_raw.empty()) {
      return false;
    }

    const size_t max_ciphertext_size =
        frame_encryptor_->GetMaxCiphertextByteSize(cricket::MEDIA_TYPE_VIDEO,
                                                   payload_size);
    encrypted_video_payload.SetSize(max_ciphertext_size);

    size_t bytes_written = 0;

    // Only enable header authentication if the field trial is enabled.
    rtc::ArrayView<const uint8_t> additional_data;
    if (generic_descriptor_auth_experiment_) {
      additional_data = generic_descriptor_raw;
    }

    if (frame_encryptor_->Encrypt(
            cricket::MEDIA_TYPE_VIDEO, first_packet->Ssrc(), additional_data,
            rtc::MakeArrayView(payload_data, payload_size),
            encrypted_video_payload, &bytes_written) != 0) {
      return false;
    }

    encrypted_video_payload.SetSize(bytes_written);
    payload_data = encrypted_video_payload.data();
    payload_size = encrypted_video_payload.size();
  } else if (require_frame_encryption_) {
    RTC_LOG(LS_WARNING)
        << "No FrameEncryptor is attached to this video sending stream but "
        << "one is required since require_frame_encryptor is set";
  }

  absl::optional<VideoCodecType> type;
  {
    rtc::CritScope cs(&payload_type_crit_);
    const auto it = payload_type_map_.find(payload_type);
    if (it == payload_type_map_.end()) {
      RTC_LOG(LS_ERROR) << "Payload type " << static_cast<int>(payload_type)
                        << " not registered.";
      return false;
    }
    type = it->second;
  }
  std::unique_ptr<RtpPacketizer> packetizer = RtpPacketizer::Create(
      type, rtc::MakeArrayView(payload_data, payload_size), limits,
      *packetize_video_header, frame_type, fragmentation);

  const uint8_t temporal_id = GetTemporalId(*video_header);
  // TODO(bugs.webrtc.org/10714): retransmission_settings_ should generally be
  // replaced by expected_retransmission_time_ms.has_value(). For now, though,
  // only VP8 with an injected frame buffer controller actually controls it.
  const StorageType storage =
      expected_retransmission_time_ms.has_value()
          ? GetStorageType(temporal_id, retransmission_settings,
                           expected_retransmission_time_ms.value())
          : StorageType::kDontRetransmit;
  const size_t num_packets = packetizer->NumPackets();

  size_t unpacketized_payload_size;
  if (fragmentation && fragmentation->fragmentationVectorSize > 0) {
    unpacketized_payload_size = 0;
    for (uint16_t i = 0; i < fragmentation->fragmentationVectorSize; ++i) {
      unpacketized_payload_size += fragmentation->fragmentationLength[i];
    }
  } else {
    unpacketized_payload_size = payload_size;
  }
  size_t packetized_payload_size = 0;

  if (num_packets == 0)
    return false;

  uint16_t first_sequence_number;
  bool first_frame = first_frame_sent_();
  for (size_t i = 0; i < num_packets; ++i) {
    std::unique_ptr<RtpPacketToSend> packet;
    int expected_payload_capacity;
    // Choose right packet template:
    if (num_packets == 1) {
      packet = std::move(single_packet);
      expected_payload_capacity =
          limits.max_payload_len - limits.single_packet_reduction_len;
    } else if (i == 0) {
      packet = std::move(first_packet);
      expected_payload_capacity =
          limits.max_payload_len - limits.first_packet_reduction_len;
    } else if (i == num_packets - 1) {
      packet = std::move(last_packet);
      expected_payload_capacity =
          limits.max_payload_len - limits.last_packet_reduction_len;
    } else {
      packet = absl::make_unique<RtpPacketToSend>(*middle_packet);
      expected_payload_capacity = limits.max_payload_len;
    }

    if (!packetizer->NextPacket(packet.get()))
      return false;
    RTC_DCHECK_LE(packet->payload_size(), expected_payload_capacity);
    if (!rtp_sender_->AssignSequenceNumber(packet.get()))
      return false;
    packetized_payload_size += packet->payload_size();

    if (rtp_sequence_number_map_ && i == 0) {
      first_sequence_number = packet->SequenceNumber();
    }

    if (i == 0) {
      playout_delay_oracle_->OnSentPacket(packet->SequenceNumber(),
                                          playout_delay);
    }
    // No FEC protection for upper temporal layers, if used.
    bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx;

    // Put packetization finish timestamp into extension.
    if (packet->HasExtension<VideoTimingExtension>()) {
      packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
      // TODO(webrtc:10750): wait a couple of months and remove the statement
      // below. For now we can't use packets with VideoTimingFrame extensions in
      // Fec because the extension is modified after FEC is calculated by pacer
      // and network. This may cause corruptions in video payload and header.
      // The fix in receive code is implemented, but until all the receivers
      // are updated, senders can't send potentially breaking packets.
      protect_packet = false;
    }

    if (flexfec_enabled()) {
      // TODO(brandtr): Remove the FlexFEC code path when FlexfecSender
      // is wired up to PacedSender instead.
      SendVideoPacketWithFlexfec(std::move(packet), storage, protect_packet);
    } else if (red_enabled) {
      SendVideoPacketAsRedMaybeWithUlpfec(std::move(packet), storage,
                                          protect_packet);
    } else {
      SendVideoPacket(std::move(packet), storage);
    }

    if (first_frame) {
      if (i == 0) {
        RTC_LOG(LS_INFO)
            << "Sent first RTP packet of the first video frame (pre-pacer)";
      }
      if (i == num_packets - 1) {
        RTC_LOG(LS_INFO)
            << "Sent last RTP packet of the first video frame (pre-pacer)";
      }
    }
  }

  if (rtp_sequence_number_map_) {
    const uint32_t timestamp = rtp_timestamp - rtp_sender_->TimestampOffset();
    rtc::CritScope cs(&crit_);
    rtp_sequence_number_map_->InsertFrame(first_sequence_number, num_packets,
                                          timestamp);
  }

  rtc::CritScope cs(&stats_crit_);
  RTC_DCHECK_GE(packetized_payload_size, unpacketized_payload_size);
  packetization_overhead_bitrate_.Update(
      packetized_payload_size - unpacketized_payload_size,
      clock_->TimeInMilliseconds());

  TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
                         rtp_timestamp);
  return true;
}

uint32_t RTPSenderVideo::VideoBitrateSent() const {
  rtc::CritScope cs(&stats_crit_);
  return video_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
}

uint32_t RTPSenderVideo::FecOverheadRate() const {
  rtc::CritScope cs(&stats_crit_);
  return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
}

uint32_t RTPSenderVideo::PacketizationOverheadBps() const {
  rtc::CritScope cs(&stats_crit_);
  return packetization_overhead_bitrate_.Rate(clock_->TimeInMilliseconds())
      .value_or(0);
}

std::vector<RtpSequenceNumberMap::Info> RTPSenderVideo::GetSentRtpPacketInfos(
    rtc::ArrayView<const uint16_t> sequence_numbers) const {
  RTC_DCHECK(!sequence_numbers.empty());

  std::vector<RtpSequenceNumberMap::Info> results;
  if (!rtp_sequence_number_map_) {
    return results;
  }
  results.reserve(sequence_numbers.size());

  {
    rtc::CritScope cs(&crit_);
    for (uint16_t sequence_number : sequence_numbers) {
      const absl::optional<RtpSequenceNumberMap::Info> info =
          rtp_sequence_number_map_->Get(sequence_number);
      if (!info) {
        // The empty vector will be returned. We can delay the clearing
        // of the vector until after we exit the critical section.
        break;
      }
      results.push_back(*info);
    }
  }

  if (results.size() != sequence_numbers.size()) {
    results.clear();  // Some sequence number was not found.
  }

  return results;
}

StorageType RTPSenderVideo::GetStorageType(
    uint8_t temporal_id,
    int32_t retransmission_settings,
    int64_t expected_retransmission_time_ms) {
  if (retransmission_settings == kRetransmitOff)
    return StorageType::kDontRetransmit;

  rtc::CritScope cs(&stats_crit_);
  // Media packet storage.
  if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
      UpdateConditionalRetransmit(temporal_id,
                                  expected_retransmission_time_ms)) {
    retransmission_settings |= kRetransmitHigherLayers;
  }

  if (temporal_id == kNoTemporalIdx)
    return kAllowRetransmission;

  if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
    return kAllowRetransmission;

  if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
    return kAllowRetransmission;

  return kDontRetransmit;
}

uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) {
  struct TemporalIdGetter {
    uint8_t operator()(const RTPVideoHeaderVP8& vp8) { return vp8.temporalIdx; }
    uint8_t operator()(const RTPVideoHeaderVP9& vp9) {
      return vp9.temporal_idx;
    }
    uint8_t operator()(const RTPVideoHeaderH264&) { return kNoTemporalIdx; }
    uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; }
  };
  switch (header.codec) {
    case kVideoCodecH264:
      return header.frame_marking.temporal_id;
    default:
      return absl::visit(TemporalIdGetter(), header.video_type_header);
  }
}

bool RTPSenderVideo::UpdateConditionalRetransmit(
    uint8_t temporal_id,
    int64_t expected_retransmission_time_ms) {
  int64_t now_ms = clock_->TimeInMilliseconds();
  // Update stats for any temporal layer.
  TemporalLayerStats* current_layer_stats =
      &frame_stats_by_temporal_layer_[temporal_id];
  current_layer_stats->frame_rate_fp1000s.Update(1, now_ms);
  int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_ms;
  current_layer_stats->last_frame_time_ms = now_ms;

  // Conditional retransmit only applies to upper layers.
  if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
    if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
      // Too long since a retransmittable frame in this layer, enable NACK
      // protection.
      return true;
    } else {
      // Estimate when the next frame of any lower layer will be sent.
      const int64_t kUndefined = std::numeric_limits<int64_t>::max();
      int64_t expected_next_frame_time = kUndefined;
      for (int i = temporal_id - 1; i >= 0; --i) {
        TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
        absl::optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms);
        if (rate) {
          int64_t tl_next = stats->last_frame_time_ms + 1000000 / *rate;
          if (tl_next - now_ms > -expected_retransmission_time_ms &&
              tl_next < expected_next_frame_time) {
            expected_next_frame_time = tl_next;
          }
        }
      }

      if (expected_next_frame_time == kUndefined ||
          expected_next_frame_time - now_ms > expected_retransmission_time_ms) {
        // The next frame in a lower layer is expected at a later time (or
        // unable to tell due to lack of data) than a retransmission is
        // estimated to be able to arrive, so allow this packet to be nacked.
        return true;
      }
    }
  }

  return false;
}

}  // namespace webrtc