From 768de78d10506ae192e5f12af48fce0c9b9e31ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Terelius?= Date: Fri, 10 Jun 2022 18:22:31 +0000 Subject: [PATCH] Revert "Remove legacy RtpVideoStreamReceiver." MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit da12e10aba4d12e7a6fb3882dc667901c9e17aa2. Reason for revert: Speculative revert due to failing downstream tests Original change's description: > Remove legacy RtpVideoStreamReceiver. > > Bug: none > Change-Id: I434a56980f4d6c68381abae973cd846c71441b08 > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/236846 > Reviewed-by: Erik Språng > Commit-Queue: Philip Eliasson > Cr-Commit-Position: refs/heads/main@{#37177} Bug: none Change-Id: I960c7693955c061ad95f921905d35c87733a3283 No-Presubmit: true No-Tree-Checks: true No-Try: true Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/265397 Commit-Queue: Björn Terelius Owners-Override: Björn Terelius Bot-Commit: rubber-stamper@appspot.gserviceaccount.com Auto-Submit: Björn Terelius Cr-Commit-Position: refs/heads/main@{#37183} --- video/BUILD.gn | 3 + video/rtp_video_stream_receiver.cc | 1278 +++++++++++++++++++ video/rtp_video_stream_receiver.h | 436 +++++++ video/rtp_video_stream_receiver_unittest.cc | 1238 ++++++++++++++++++ 4 files changed, 2955 insertions(+) create mode 100644 video/rtp_video_stream_receiver.cc create mode 100644 video/rtp_video_stream_receiver.h create mode 100644 video/rtp_video_stream_receiver_unittest.cc diff --git a/video/BUILD.gn b/video/BUILD.gn index 737e02853b..74eb3bbe2d 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -172,6 +172,8 @@ rtc_source_set("video_legacy") { "call_stats.h", "receive_statistics_proxy.cc", "receive_statistics_proxy.h", + "rtp_video_stream_receiver.cc", + "rtp_video_stream_receiver.h", "video_quality_observer.cc", "video_quality_observer.h", ] @@ -1005,6 +1007,7 @@ if (rtc_include_tests) { sources = [ "call_stats_unittest.cc", "receive_statistics_proxy_unittest.cc", + "rtp_video_stream_receiver_unittest.cc", ] deps = [ ":video_legacy", diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc new file mode 100644 index 0000000000..b09e5fc5a3 --- /dev/null +++ b/video/rtp_video_stream_receiver.cc @@ -0,0 +1,1278 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/rtp_video_stream_receiver.h" + +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/memory/memory.h" +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "media/base/media_constants.h" +#include "modules/pacing/packet_router.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/include/rtp_cvo.h" +#include "modules/rtp_rtcp/include/ulpfec_receiver.h" +#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h" +#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_format.h" +#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" +#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_header_extensions.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_config.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h" +#include "modules/utility/include/process_thread.h" +#include "modules/video_coding/deprecated/nack_module.h" +#include "modules/video_coding/frame_object.h" +#include "modules/video_coding/h264_sprop_parameter_sets.h" +#include "modules/video_coding/h264_sps_pps_tracker.h" +#include "modules/video_coding/packet_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/location.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/ntp_time.h" +#include "video/receive_statistics_proxy.h" + +namespace webrtc { + +namespace { +// TODO(philipel): Change kPacketBufferStartSize back to 32 in M63 see: +// crbug.com/752886 +constexpr int kPacketBufferStartSize = 512; +constexpr int kPacketBufferMaxSize = 2048; + +int PacketBufferMaxSize(const FieldTrialsView& field_trials) { + // The group here must be a positive power of 2, in which case that is used as + // size. All other values shall result in the default value being used. + const std::string group_name = + field_trials.Lookup("WebRTC-PacketBufferMaxSize"); + int packet_buffer_max_size = kPacketBufferMaxSize; + if (!group_name.empty() && + (sscanf(group_name.c_str(), "%d", &packet_buffer_max_size) != 1 || + packet_buffer_max_size <= 0 || + // Verify that the number is a positive power of 2. + (packet_buffer_max_size & (packet_buffer_max_size - 1)) != 0)) { + RTC_LOG(LS_WARNING) << "Invalid packet buffer max size: " << group_name; + packet_buffer_max_size = kPacketBufferMaxSize; + } + return packet_buffer_max_size; +} + +std::unique_ptr CreateRtpRtcpModule( + Clock* clock, + ReceiveStatistics* receive_statistics, + Transport* outgoing_transport, + RtcpRttStats* rtt_stats, + RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, + RtcpCnameCallback* rtcp_cname_callback, + bool non_sender_rtt_measurement, + uint32_t local_ssrc) { + RtpRtcpInterface::Configuration configuration; + configuration.clock = clock; + configuration.audio = false; + configuration.receiver_only = true; + configuration.receive_statistics = receive_statistics; + configuration.outgoing_transport = outgoing_transport; + configuration.rtt_stats = rtt_stats; + configuration.rtcp_packet_type_counter_observer = + rtcp_packet_type_counter_observer; + configuration.rtcp_cname_callback = rtcp_cname_callback; + configuration.local_media_ssrc = local_ssrc; + configuration.non_sender_rtt_measurement = non_sender_rtt_measurement; + + std::unique_ptr rtp_rtcp = RtpRtcp::DEPRECATED_Create(configuration); + rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); + + return rtp_rtcp; +} + +static const int kPacketLogIntervalMs = 10000; + +} // namespace + +RtpVideoStreamReceiver::RtcpFeedbackBuffer::RtcpFeedbackBuffer( + KeyFrameRequestSender* key_frame_request_sender, + NackSender* nack_sender, + LossNotificationSender* loss_notification_sender) + : key_frame_request_sender_(key_frame_request_sender), + nack_sender_(nack_sender), + loss_notification_sender_(loss_notification_sender), + request_key_frame_(false) { + RTC_DCHECK(key_frame_request_sender_); + RTC_DCHECK(nack_sender_); + RTC_DCHECK(loss_notification_sender_); +} + +void RtpVideoStreamReceiver::RtcpFeedbackBuffer::RequestKeyFrame() { + MutexLock lock(&mutex_); + request_key_frame_ = true; +} + +void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendNack( + const std::vector& sequence_numbers, + bool buffering_allowed) { + RTC_DCHECK(!sequence_numbers.empty()); + MutexLock lock(&mutex_); + nack_sequence_numbers_.insert(nack_sequence_numbers_.end(), + sequence_numbers.cbegin(), + sequence_numbers.cend()); + if (!buffering_allowed) { + // Note that while *buffering* is not allowed, *batching* is, meaning that + // previously buffered messages may be sent along with the current message. + SendRtcpFeedback(ConsumeRtcpFeedbackLocked()); + } +} + +void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendLossNotification( + uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) { + RTC_DCHECK(buffering_allowed); + MutexLock lock(&mutex_); + RTC_DCHECK(!lntf_state_) + << "SendLossNotification() called twice in a row with no call to " + "SendBufferedRtcpFeedback() in between."; + lntf_state_ = absl::make_optional( + last_decoded_seq_num, last_received_seq_num, decodability_flag); +} + +void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendBufferedRtcpFeedback() { + SendRtcpFeedback(ConsumeRtcpFeedback()); +} + +RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumedRtcpFeedback +RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumeRtcpFeedback() { + MutexLock lock(&mutex_); + return ConsumeRtcpFeedbackLocked(); +} + +RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumedRtcpFeedback +RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumeRtcpFeedbackLocked() { + ConsumedRtcpFeedback feedback; + std::swap(feedback.request_key_frame, request_key_frame_); + std::swap(feedback.nack_sequence_numbers, nack_sequence_numbers_); + std::swap(feedback.lntf_state, lntf_state_); + return feedback; +} + +void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendRtcpFeedback( + ConsumedRtcpFeedback feedback) { + if (feedback.lntf_state) { + // If either a NACK or a key frame request is sent, we should buffer + // the LNTF and wait for them (NACK or key frame request) to trigger + // the compound feedback message. + // Otherwise, the LNTF should be sent out immediately. + const bool buffering_allowed = + feedback.request_key_frame || !feedback.nack_sequence_numbers.empty(); + + loss_notification_sender_->SendLossNotification( + feedback.lntf_state->last_decoded_seq_num, + feedback.lntf_state->last_received_seq_num, + feedback.lntf_state->decodability_flag, buffering_allowed); + } + + if (feedback.request_key_frame) { + key_frame_request_sender_->RequestKeyFrame(); + } else if (!feedback.nack_sequence_numbers.empty()) { + nack_sender_->SendNack(feedback.nack_sequence_numbers, true); + } +} + +// DEPRECATED +RtpVideoStreamReceiver::RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + PacketRouter* packet_router, + const VideoReceiveStreamInterface::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer, + const FieldTrialsView* field_trials) + : RtpVideoStreamReceiver(clock, + transport, + rtt_stats, + packet_router, + config, + rtp_receive_statistics, + receive_stats_proxy, + receive_stats_proxy, + process_thread, + nack_sender, + keyframe_request_sender, + complete_frame_callback, + frame_decryptor, + frame_transformer, + field_trials) {} + +RtpVideoStreamReceiver::RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + PacketRouter* packet_router, + const VideoReceiveStreamInterface::Config* config, + ReceiveStatistics* rtp_receive_statistics, + RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, + RtcpCnameCallback* rtcp_cname_callback, + ProcessThread* process_thread, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer, + const FieldTrialsView* field_trials) + : field_trials_(field_trials ? *field_trials : owned_field_trials_), + clock_(clock), + config_(*config), + packet_router_(packet_router), + process_thread_(process_thread), + ntp_estimator_(clock), + rtp_header_extensions_(config_.rtp.extensions), + forced_playout_delay_max_ms_("max_ms", absl::nullopt), + forced_playout_delay_min_ms_("min_ms", absl::nullopt), + rtp_receive_statistics_(rtp_receive_statistics), + ulpfec_receiver_(UlpfecReceiver::Create(config->rtp.remote_ssrc, + this, + config->rtp.extensions)), + receiving_(false), + last_packet_log_ms_(-1), + rtp_rtcp_(CreateRtpRtcpModule( + clock, + rtp_receive_statistics_, + transport, + rtt_stats, + rtcp_packet_type_counter_observer, + rtcp_cname_callback, + config_.rtp.rtcp_xr.receiver_reference_time_report, + config_.rtp.local_ssrc)), + complete_frame_callback_(complete_frame_callback), + keyframe_request_sender_(keyframe_request_sender), + // TODO(bugs.webrtc.org/10336): Let `rtcp_feedback_buffer_` communicate + // directly with `rtp_rtcp_`. + rtcp_feedback_buffer_(this, nack_sender, this), + packet_buffer_(kPacketBufferStartSize, + PacketBufferMaxSize(field_trials_)), + reference_finder_(std::make_unique()), + has_received_frame_(false), + frames_decryptable_(false), + absolute_capture_time_interpolator_(clock) { + constexpr bool remb_candidate = true; + if (packet_router_) + packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); + + RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff) + << "A stream should not be configured with RTCP disabled. This value is " + "reserved for internal usage."; + // TODO(pbos): What's an appropriate local_ssrc for receive-only streams? + RTC_DCHECK(config_.rtp.local_ssrc != 0); + RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc); + + rtp_rtcp_->SetRTCPStatus(config_.rtp.rtcp_mode); + rtp_rtcp_->SetRemoteSSRC(config_.rtp.remote_ssrc); + + static const int kMaxPacketAgeToNack = 450; + const int max_reordering_threshold = (config_.rtp.nack.rtp_history_ms > 0) + ? kMaxPacketAgeToNack + : kDefaultMaxReorderingThreshold; + rtp_receive_statistics_->SetMaxReorderingThreshold(config_.rtp.remote_ssrc, + max_reordering_threshold); + // TODO(nisse): For historic reasons, we applied the above + // max_reordering_threshold also for RTX stats, which makes little sense since + // we don't NACK rtx packets. Consider deleting the below block, and rely on + // the default threshold. + if (config_.rtp.rtx_ssrc) { + rtp_receive_statistics_->SetMaxReorderingThreshold( + config_.rtp.rtx_ssrc, max_reordering_threshold); + } + ParseFieldTrial( + {&forced_playout_delay_max_ms_, &forced_playout_delay_min_ms_}, + field_trials_.Lookup("WebRTC-ForcePlayoutDelay")); + + process_thread_->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE); + + if (config_.rtp.lntf.enabled) { + loss_notification_controller_ = + std::make_unique(&rtcp_feedback_buffer_, + &rtcp_feedback_buffer_); + } + + if (config_.rtp.nack.rtp_history_ms != 0) { + nack_module_ = std::make_unique( + clock_, &rtcp_feedback_buffer_, &rtcp_feedback_buffer_, field_trials_); + process_thread_->RegisterModule(nack_module_.get(), RTC_FROM_HERE); + } + + // Only construct the encrypted receiver if frame encryption is enabled. + if (config_.crypto_options.sframe.require_frame_encryption) { + buffered_frame_decryptor_ = + std::make_unique(this, this, field_trials_); + if (frame_decryptor != nullptr) { + buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor)); + } + } + + if (frame_transformer) { + frame_transformer_delegate_ = + rtc::make_ref_counted( + this, std::move(frame_transformer), rtc::Thread::Current(), + config_.rtp.remote_ssrc); + frame_transformer_delegate_->Init(); + } +} + +RtpVideoStreamReceiver::~RtpVideoStreamReceiver() { + RTC_DCHECK(secondary_sinks_.empty()); + + if (nack_module_) { + process_thread_->DeRegisterModule(nack_module_.get()); + } + + process_thread_->DeRegisterModule(rtp_rtcp_.get()); + + if (packet_router_) + packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get()); + UpdateHistograms(); + if (frame_transformer_delegate_) + frame_transformer_delegate_->Reset(); +} + +void RtpVideoStreamReceiver::AddReceiveCodec( + uint8_t payload_type, + VideoCodecType codec_type, + const std::map& codec_params, + bool raw_payload) { + if (codec_params.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) || + field_trials_.IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe")) { + MutexLock lock(&packet_buffer_lock_); + packet_buffer_.ForceSpsPpsIdrIsH264Keyframe(); + } + payload_type_map_.emplace( + payload_type, raw_payload ? std::make_unique() + : CreateVideoRtpDepacketizer(codec_type)); + pt_codec_params_.emplace(payload_type, codec_params); +} + +absl::optional RtpVideoStreamReceiver::GetSyncInfo() const { + Syncable::Info info; + if (rtp_rtcp_->RemoteNTP(&info.capture_time_ntp_secs, + &info.capture_time_ntp_frac, + /*rtcp_arrival_time_secs=*/nullptr, + /*rtcp_arrival_time_frac=*/nullptr, + &info.capture_time_source_clock) != 0) { + return absl::nullopt; + } + { + MutexLock lock(&sync_info_lock_); + if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_) { + return absl::nullopt; + } + info.latest_received_capture_timestamp = *last_received_rtp_timestamp_; + info.latest_receive_time_ms = last_received_rtp_system_time_->ms(); + } + + // Leaves info.current_delay_ms uninitialized. + return info; +} + +RtpVideoStreamReceiver::ParseGenericDependenciesResult +RtpVideoStreamReceiver::ParseGenericDependenciesExtension( + const RtpPacketReceived& rtp_packet, + RTPVideoHeader* video_header) { + if (rtp_packet.HasExtension()) { + webrtc::DependencyDescriptor dependency_descriptor; + if (!rtp_packet.GetExtension( + video_structure_.get(), &dependency_descriptor)) { + // Descriptor is there, but failed to parse. Either it is invalid, + // or too old packet (after relevant video_structure_ changed), + // or too new packet (before relevant video_structure_ arrived). + // Drop such packet to be on the safe side. + // TODO(bugs.webrtc.org/10342): Stash too new packet. + RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc() + << " Failed to parse dependency descriptor."; + return kDropPacket; + } + if (dependency_descriptor.attached_structure != nullptr && + !dependency_descriptor.first_packet_in_frame) { + RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc() + << "Invalid dependency descriptor: structure " + "attached to non first packet of a frame."; + return kDropPacket; + } + video_header->is_first_packet_in_frame = + dependency_descriptor.first_packet_in_frame; + video_header->is_last_packet_in_frame = + dependency_descriptor.last_packet_in_frame; + + int64_t frame_id = + frame_id_unwrapper_.Unwrap(dependency_descriptor.frame_number); + auto& generic_descriptor_info = video_header->generic.emplace(); + generic_descriptor_info.frame_id = frame_id; + generic_descriptor_info.spatial_index = + dependency_descriptor.frame_dependencies.spatial_id; + generic_descriptor_info.temporal_index = + dependency_descriptor.frame_dependencies.temporal_id; + for (int fdiff : dependency_descriptor.frame_dependencies.frame_diffs) { + generic_descriptor_info.dependencies.push_back(frame_id - fdiff); + } + generic_descriptor_info.decode_target_indications = + dependency_descriptor.frame_dependencies.decode_target_indications; + if (dependency_descriptor.resolution) { + video_header->width = dependency_descriptor.resolution->Width(); + video_header->height = dependency_descriptor.resolution->Height(); + } + + // FrameDependencyStructure is sent in dependency descriptor of the first + // packet of a key frame and required for parsed dependency descriptor in + // all the following packets until next key frame. + // Save it if there is a (potentially) new structure. + if (dependency_descriptor.attached_structure) { + RTC_DCHECK(dependency_descriptor.first_packet_in_frame); + if (video_structure_frame_id_ > frame_id) { + RTC_LOG(LS_WARNING) + << "Arrived key frame with id " << frame_id << " and structure id " + << dependency_descriptor.attached_structure->structure_id + << " is older than the latest received key frame with id " + << *video_structure_frame_id_ << " and structure id " + << video_structure_->structure_id; + return kDropPacket; + } + video_structure_ = std::move(dependency_descriptor.attached_structure); + video_structure_frame_id_ = frame_id; + video_header->frame_type = VideoFrameType::kVideoFrameKey; + } else { + video_header->frame_type = VideoFrameType::kVideoFrameDelta; + } + return kHasGenericDescriptor; + } + + RtpGenericFrameDescriptor generic_frame_descriptor; + if (!rtp_packet.GetExtension( + &generic_frame_descriptor)) { + return kNoGenericDescriptor; + } + + video_header->is_first_packet_in_frame = + generic_frame_descriptor.FirstPacketInSubFrame(); + video_header->is_last_packet_in_frame = + generic_frame_descriptor.LastPacketInSubFrame(); + + if (generic_frame_descriptor.FirstPacketInSubFrame()) { + video_header->frame_type = + generic_frame_descriptor.FrameDependenciesDiffs().empty() + ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta; + + auto& generic_descriptor_info = video_header->generic.emplace(); + int64_t frame_id = + frame_id_unwrapper_.Unwrap(generic_frame_descriptor.FrameId()); + generic_descriptor_info.frame_id = frame_id; + generic_descriptor_info.spatial_index = + generic_frame_descriptor.SpatialLayer(); + generic_descriptor_info.temporal_index = + generic_frame_descriptor.TemporalLayer(); + for (uint16_t fdiff : generic_frame_descriptor.FrameDependenciesDiffs()) { + generic_descriptor_info.dependencies.push_back(frame_id - fdiff); + } + } + video_header->width = generic_frame_descriptor.Width(); + video_header->height = generic_frame_descriptor.Height(); + return kHasGenericDescriptor; +} + +void RtpVideoStreamReceiver::OnReceivedPayloadData( + rtc::CopyOnWriteBuffer codec_payload, + const RtpPacketReceived& rtp_packet, + const RTPVideoHeader& video) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + + auto packet = + std::make_unique(rtp_packet, video); + + RTPVideoHeader& video_header = packet->video_header; + video_header.rotation = kVideoRotation_0; + video_header.content_type = VideoContentType::UNSPECIFIED; + video_header.video_timing.flags = VideoSendTiming::kInvalid; + video_header.is_last_packet_in_frame |= rtp_packet.Marker(); + + if (const auto* vp9_header = + absl::get_if(&video_header.video_type_header)) { + video_header.is_last_packet_in_frame |= vp9_header->end_of_frame; + video_header.is_first_packet_in_frame |= vp9_header->beginning_of_frame; + } + + rtp_packet.GetExtension(&video_header.rotation); + rtp_packet.GetExtension( + &video_header.content_type); + rtp_packet.GetExtension(&video_header.video_timing); + if (forced_playout_delay_max_ms_ && forced_playout_delay_min_ms_) { + video_header.playout_delay.max_ms = *forced_playout_delay_max_ms_; + video_header.playout_delay.min_ms = *forced_playout_delay_min_ms_; + } else { + rtp_packet.GetExtension(&video_header.playout_delay); + } + + ParseGenericDependenciesResult generic_descriptor_state = + ParseGenericDependenciesExtension(rtp_packet, &video_header); + + if (!rtp_packet.recovered()) { + UpdatePacketReceiveTimestamps( + rtp_packet, video_header.frame_type == VideoFrameType::kVideoFrameKey); + } + + if (generic_descriptor_state == kDropPacket) + return; + + // Color space should only be transmitted in the last packet of a frame, + // therefore, neglect it otherwise so that last_color_space_ is not reset by + // mistake. + if (video_header.is_last_packet_in_frame) { + video_header.color_space = rtp_packet.GetExtension(); + if (video_header.color_space || + video_header.frame_type == VideoFrameType::kVideoFrameKey) { + // Store color space since it's only transmitted when changed or for key + // frames. Color space will be cleared if a key frame is transmitted + // without color space information. + last_color_space_ = video_header.color_space; + } else if (last_color_space_) { + video_header.color_space = last_color_space_; + } + } + video_header.video_frame_tracking_id = + rtp_packet.GetExtension(); + + if (loss_notification_controller_) { + if (rtp_packet.recovered()) { + // TODO(bugs.webrtc.org/10336): Implement support for reordering. + RTC_LOG(LS_INFO) + << "LossNotificationController does not support reordering."; + } else if (generic_descriptor_state == kNoGenericDescriptor) { + RTC_LOG(LS_WARNING) << "LossNotificationController requires generic " + "frame descriptor, but it is missing."; + } else { + if (video_header.is_first_packet_in_frame) { + RTC_DCHECK(video_header.generic); + LossNotificationController::FrameDetails frame; + frame.is_keyframe = + video_header.frame_type == VideoFrameType::kVideoFrameKey; + frame.frame_id = video_header.generic->frame_id; + frame.frame_dependencies = video_header.generic->dependencies; + loss_notification_controller_->OnReceivedPacket( + rtp_packet.SequenceNumber(), &frame); + } else { + loss_notification_controller_->OnReceivedPacket( + rtp_packet.SequenceNumber(), nullptr); + } + } + } + + if (nack_module_) { + const bool is_keyframe = + video_header.is_first_packet_in_frame && + video_header.frame_type == VideoFrameType::kVideoFrameKey; + + packet->times_nacked = nack_module_->OnReceivedPacket( + rtp_packet.SequenceNumber(), is_keyframe, rtp_packet.recovered()); + } else { + packet->times_nacked = -1; + } + + if (codec_payload.size() == 0) { + NotifyReceiverOfEmptyPacket(packet->seq_num); + rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); + return; + } + + if (packet->codec() == kVideoCodecH264) { + // Only when we start to receive packets will we know what payload type + // that will be used. When we know the payload type insert the correct + // sps/pps into the tracker. + if (packet->payload_type != last_payload_type_) { + last_payload_type_ = packet->payload_type; + InsertSpsPpsIntoTracker(packet->payload_type); + } + + video_coding::H264SpsPpsTracker::FixedBitstream fixed = + tracker_.CopyAndFixBitstream( + rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()), + &packet->video_header); + + switch (fixed.action) { + case video_coding::H264SpsPpsTracker::kRequestKeyframe: + rtcp_feedback_buffer_.RequestKeyFrame(); + rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); + [[fallthrough]]; + case video_coding::H264SpsPpsTracker::kDrop: + return; + case video_coding::H264SpsPpsTracker::kInsert: + packet->video_payload = std::move(fixed.bitstream); + break; + } + + } else { + packet->video_payload = std::move(codec_payload); + } + + rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); + frame_counter_.Add(packet->timestamp); + video_coding::PacketBuffer::InsertResult insert_result; + { + MutexLock lock(&packet_buffer_lock_); + int64_t unwrapped_rtp_seq_num = + rtp_seq_num_unwrapper_.Unwrap(rtp_packet.SequenceNumber()); + auto& packet_info = + packet_infos_ + .emplace( + unwrapped_rtp_seq_num, + RtpPacketInfo( + rtp_packet.Ssrc(), rtp_packet.Csrcs(), + rtp_packet.Timestamp(), + /*audio_level=*/absl::nullopt, + rtp_packet.GetExtension(), + /*receive_time_ms=*/clock_->TimeInMilliseconds())) + .first->second; + + // Try to extrapolate absolute capture time if it is missing. + absl::optional absolute_capture_time = + absolute_capture_time_interpolator_.OnReceivePacket( + AbsoluteCaptureTimeInterpolator::GetSource(packet_info.ssrc(), + packet_info.csrcs()), + packet_info.rtp_timestamp(), + // Assume frequency is the same one for all video frames. + kVideoPayloadTypeFrequency, packet_info.absolute_capture_time()); + packet_info.set_absolute_capture_time(absolute_capture_time); + + if (absolute_capture_time.has_value()) { + packet_info.set_local_capture_clock_offset( + capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset( + absolute_capture_time->estimated_capture_clock_offset)); + } + + insert_result = packet_buffer_.InsertPacket(std::move(packet)); + } + OnInsertedPacket(std::move(insert_result)); +} + +void RtpVideoStreamReceiver::OnRecoveredPacket(const uint8_t* rtp_packet, + size_t rtp_packet_length) { + RtpPacketReceived packet; + if (!packet.Parse(rtp_packet, rtp_packet_length)) + return; + if (packet.PayloadType() == config_.rtp.red_payload_type) { + RTC_LOG(LS_WARNING) << "Discarding recovered packet with RED encapsulation"; + return; + } + + packet.IdentifyExtensions(rtp_header_extensions_); + packet.set_payload_type_frequency(kVideoPayloadTypeFrequency); + // TODO(nisse): UlpfecReceiverImpl::ProcessReceivedFec passes both + // original (decapsulated) media packets and recovered packets to + // this callback. We need a way to distinguish, for setting + // packet.recovered() correctly. Ideally, move RED decapsulation out + // of the Ulpfec implementation. + + ReceivePacket(packet); +} + +// This method handles both regular RTP packets and packets recovered +// via FlexFEC. +void RtpVideoStreamReceiver::OnRtpPacket(const RtpPacketReceived& packet) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + + if (!receiving_) { + return; + } + + ReceivePacket(packet); + + // Update receive statistics after ReceivePacket. + // Receive statistics will be reset if the payload type changes (make sure + // that the first packet is included in the stats). + if (!packet.recovered()) { + rtp_receive_statistics_->OnRtpPacket(packet); + } + + for (RtpPacketSinkInterface* secondary_sink : secondary_sinks_) { + secondary_sink->OnRtpPacket(packet); + } +} + +void RtpVideoStreamReceiver::RequestKeyFrame() { + // TODO(bugs.webrtc.org/10336): Allow the sender to ignore key frame requests + // issued by anything other than the LossNotificationController if it (the + // sender) is relying on LNTF alone. + if (keyframe_request_sender_) { + keyframe_request_sender_->RequestKeyFrame(); + } else { + rtp_rtcp_->SendPictureLossIndication(); + } +} + +void RtpVideoStreamReceiver::SendLossNotification( + uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) { + RTC_DCHECK(config_.rtp.lntf.enabled); + rtp_rtcp_->SendLossNotification(last_decoded_seq_num, last_received_seq_num, + decodability_flag, buffering_allowed); +} + +bool RtpVideoStreamReceiver::IsUlpfecEnabled() const { + return config_.rtp.ulpfec_payload_type != -1; +} + +bool RtpVideoStreamReceiver::IsRetransmissionsEnabled() const { + return config_.rtp.nack.rtp_history_ms > 0; +} + +void RtpVideoStreamReceiver::RequestPacketRetransmit( + const std::vector& sequence_numbers) { + rtp_rtcp_->SendNack(sequence_numbers); +} + +bool RtpVideoStreamReceiver::IsDecryptable() const { + return frames_decryptable_.load(); +} + +void RtpVideoStreamReceiver::OnInsertedPacket( + video_coding::PacketBuffer::InsertResult result) { + std::vector> assembled_frames; + { + MutexLock lock(&packet_buffer_lock_); + video_coding::PacketBuffer::Packet* first_packet = nullptr; + int max_nack_count; + int64_t min_recv_time; + int64_t max_recv_time; + std::vector> payloads; + RtpPacketInfos::vector_type packet_infos; + + bool frame_boundary = true; + for (auto& packet : result.packets) { + // PacketBuffer promisses frame boundaries are correctly set on each + // packet. Document that assumption with the DCHECKs. + RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame()); + int64_t unwrapped_rtp_seq_num = + rtp_seq_num_unwrapper_.Unwrap(packet->seq_num); + RTC_DCHECK(packet_infos_.count(unwrapped_rtp_seq_num) > 0); + RtpPacketInfo& packet_info = packet_infos_[unwrapped_rtp_seq_num]; + if (packet->is_first_packet_in_frame()) { + first_packet = packet.get(); + max_nack_count = packet->times_nacked; + min_recv_time = packet_info.receive_time().ms(); + max_recv_time = packet_info.receive_time().ms(); + } else { + max_nack_count = std::max(max_nack_count, packet->times_nacked); + min_recv_time = + std::min(min_recv_time, packet_info.receive_time().ms()); + max_recv_time = + std::max(max_recv_time, packet_info.receive_time().ms()); + } + payloads.emplace_back(packet->video_payload); + packet_infos.push_back(packet_info); + + frame_boundary = packet->is_last_packet_in_frame(); + if (packet->is_last_packet_in_frame()) { + auto depacketizer_it = + payload_type_map_.find(first_packet->payload_type); + RTC_CHECK(depacketizer_it != payload_type_map_.end()); + + rtc::scoped_refptr bitstream = + depacketizer_it->second->AssembleFrame(payloads); + if (!bitstream) { + // Failed to assemble a frame. Discard and continue. + continue; + } + + const video_coding::PacketBuffer::Packet& last_packet = *packet; + assembled_frames.push_back(std::make_unique( + first_packet->seq_num, // + last_packet.seq_num, // + last_packet.marker_bit, // + max_nack_count, // + min_recv_time, // + max_recv_time, // + first_packet->timestamp, // + ntp_estimator_.Estimate(first_packet->timestamp), // + last_packet.video_header.video_timing, // + first_packet->payload_type, // + first_packet->codec(), // + last_packet.video_header.rotation, // + last_packet.video_header.content_type, // + first_packet->video_header, // + last_packet.video_header.color_space, // + RtpPacketInfos(std::move(packet_infos)), // + std::move(bitstream))); + payloads.clear(); + packet_infos.clear(); + } + } + RTC_DCHECK(frame_boundary); + + if (result.buffer_cleared) { + packet_infos_.clear(); + } + } // packet_buffer_lock_ + + if (result.buffer_cleared) { + { + MutexLock lock(&sync_info_lock_); + last_received_rtp_system_time_.reset(); + last_received_keyframe_rtp_system_time_.reset(); + last_received_keyframe_rtp_timestamp_.reset(); + } + RequestKeyFrame(); + } + + for (auto& frame : assembled_frames) { + OnAssembledFrame(std::move(frame)); + } +} + +void RtpVideoStreamReceiver::OnAssembledFrame( + std::unique_ptr frame) { + RTC_DCHECK_RUN_ON(&network_tc_); + RTC_DCHECK(frame); + + const absl::optional& descriptor = + frame->GetRtpVideoHeader().generic; + + if (loss_notification_controller_ && descriptor) { + loss_notification_controller_->OnAssembledFrame( + frame->first_seq_num(), descriptor->frame_id, + absl::c_linear_search(descriptor->decode_target_indications, + DecodeTargetIndication::kDiscardable), + descriptor->dependencies); + } + + // If frames arrive before a key frame, they would not be decodable. + // In that case, request a key frame ASAP. + if (!has_received_frame_) { + if (frame->FrameType() != VideoFrameType::kVideoFrameKey) { + // `loss_notification_controller_`, if present, would have already + // requested a key frame when the first packet for the non-key frame + // had arrived, so no need to replicate the request. + if (!loss_notification_controller_) { + RequestKeyFrame(); + } + } + has_received_frame_ = true; + } + + MutexLock lock(&reference_finder_lock_); + // Reset `reference_finder_` if `frame` is new and the codec have changed. + if (current_codec_) { + bool frame_is_newer = + AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_); + + if (frame->codec_type() != current_codec_) { + if (frame_is_newer) { + // When we reset the `reference_finder_` we don't want new picture ids + // to overlap with old picture ids. To ensure that doesn't happen we + // start from the `last_completed_picture_id_` and add an offset in + // case of reordering. + reference_finder_ = std::make_unique( + last_completed_picture_id_ + std::numeric_limits::max()); + current_codec_ = frame->codec_type(); + } else { + // Old frame from before the codec switch, discard it. + return; + } + } + + if (frame_is_newer) { + last_assembled_frame_rtp_timestamp_ = frame->Timestamp(); + } + } else { + current_codec_ = frame->codec_type(); + last_assembled_frame_rtp_timestamp_ = frame->Timestamp(); + } + + if (buffered_frame_decryptor_ != nullptr) { + buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame)); + } else if (frame_transformer_delegate_) { + frame_transformer_delegate_->TransformFrame(std::move(frame)); + } else { + OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame))); + } +} + +void RtpVideoStreamReceiver::OnCompleteFrames( + RtpFrameReferenceFinder::ReturnVector frames) { + { + MutexLock lock(&last_seq_num_mutex_); + for (const auto& frame : frames) { + RtpFrameObject* rtp_frame = static_cast(frame.get()); + last_seq_num_for_pic_id_[rtp_frame->Id()] = rtp_frame->last_seq_num(); + } + } + for (auto& frame : frames) { + last_completed_picture_id_ = + std::max(last_completed_picture_id_, frame->Id()); + complete_frame_callback_->OnCompleteFrame(std::move(frame)); + } +} + +void RtpVideoStreamReceiver::OnDecryptedFrame( + std::unique_ptr frame) { + MutexLock lock(&reference_finder_lock_); + OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame))); +} + +void RtpVideoStreamReceiver::OnDecryptionStatusChange( + FrameDecryptorInterface::Status status) { + frames_decryptable_.store( + (status == FrameDecryptorInterface::Status::kOk) || + (status == FrameDecryptorInterface::Status::kRecoverable)); +} + +void RtpVideoStreamReceiver::SetFrameDecryptor( + rtc::scoped_refptr frame_decryptor) { + RTC_DCHECK_RUN_ON(&network_tc_); + if (buffered_frame_decryptor_ == nullptr) { + buffered_frame_decryptor_ = + std::make_unique(this, this, field_trials_); + } + buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor)); +} + +void RtpVideoStreamReceiver::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(&network_tc_); + frame_transformer_delegate_ = + rtc::make_ref_counted( + this, std::move(frame_transformer), rtc::Thread::Current(), + config_.rtp.remote_ssrc); + frame_transformer_delegate_->Init(); +} + +void RtpVideoStreamReceiver::UpdateRtt(int64_t max_rtt_ms) { + if (nack_module_) + nack_module_->UpdateRtt(max_rtt_ms); +} + +absl::optional RtpVideoStreamReceiver::LastReceivedPacketMs() const { + MutexLock lock(&sync_info_lock_); + if (last_received_rtp_system_time_) { + return absl::optional(last_received_rtp_system_time_->ms()); + } + return absl::nullopt; +} + +absl::optional RtpVideoStreamReceiver::LastReceivedKeyframePacketMs() + const { + MutexLock lock(&sync_info_lock_); + if (last_received_keyframe_rtp_system_time_) { + return absl::optional( + last_received_keyframe_rtp_system_time_->ms()); + } + return absl::nullopt; +} + +void RtpVideoStreamReceiver::AddSecondarySink(RtpPacketSinkInterface* sink) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + RTC_DCHECK(!absl::c_linear_search(secondary_sinks_, sink)); + secondary_sinks_.push_back(sink); +} + +void RtpVideoStreamReceiver::RemoveSecondarySink( + const RtpPacketSinkInterface* sink) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + auto it = absl::c_find(secondary_sinks_, sink); + if (it == secondary_sinks_.end()) { + // We might be rolling-back a call whose setup failed mid-way. In such a + // case, it's simpler to remove "everything" rather than remember what + // has already been added. + RTC_LOG(LS_WARNING) << "Removal of unknown sink."; + return; + } + secondary_sinks_.erase(it); +} + +void RtpVideoStreamReceiver::ManageFrame( + std::unique_ptr frame) { + MutexLock lock(&reference_finder_lock_); + OnCompleteFrames(reference_finder_->ManageFrame(std::move(frame))); +} + +void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) { + if (packet.payload_size() == 0) { + // Padding or keep-alive packet. + // TODO(nisse): Could drop empty packets earlier, but need to figure out how + // they should be counted in stats. + NotifyReceiverOfEmptyPacket(packet.SequenceNumber()); + return; + } + if (packet.PayloadType() == config_.rtp.red_payload_type) { + ParseAndHandleEncapsulatingHeader(packet); + return; + } + + const auto type_it = payload_type_map_.find(packet.PayloadType()); + if (type_it == payload_type_map_.end()) { + return; + } + absl::optional parsed_payload = + type_it->second->Parse(packet.PayloadBuffer()); + if (parsed_payload == absl::nullopt) { + RTC_LOG(LS_WARNING) << "Failed parsing payload."; + return; + } + + OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet, + parsed_payload->video_header); +} + +void RtpVideoStreamReceiver::ParseAndHandleEncapsulatingHeader( + const RtpPacketReceived& packet) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + if (packet.PayloadType() == config_.rtp.red_payload_type && + packet.payload_size() > 0) { + if (packet.payload()[0] == config_.rtp.ulpfec_payload_type) { + // Notify video_receiver about received FEC packets to avoid NACKing these + // packets. + NotifyReceiverOfEmptyPacket(packet.SequenceNumber()); + } + if (!ulpfec_receiver_->AddReceivedRedPacket( + packet, config_.rtp.ulpfec_payload_type)) { + return; + } + ulpfec_receiver_->ProcessReceivedFec(); + } +} + +// In the case of a video stream without picture ids and no rtx the +// RtpFrameReferenceFinder will need to know about padding to +// correctly calculate frame references. +void RtpVideoStreamReceiver::NotifyReceiverOfEmptyPacket(uint16_t seq_num) { + { + MutexLock lock(&reference_finder_lock_); + OnCompleteFrames(reference_finder_->PaddingReceived(seq_num)); + } + + video_coding::PacketBuffer::InsertResult insert_result; + { + MutexLock lock(&packet_buffer_lock_); + insert_result = packet_buffer_.InsertPadding(seq_num); + } + OnInsertedPacket(std::move(insert_result)); + + if (nack_module_) { + nack_module_->OnReceivedPacket(seq_num, /* is_keyframe = */ false, + /* is _recovered = */ false); + } + if (loss_notification_controller_) { + // TODO(bugs.webrtc.org/10336): Handle empty packets. + RTC_LOG(LS_WARNING) + << "LossNotificationController does not expect empty packets."; + } +} + +bool RtpVideoStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet, + size_t rtcp_packet_length) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + + if (!receiving_) { + return false; + } + + rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length); + + int64_t rtt = 0; + rtp_rtcp_->RTT(config_.rtp.remote_ssrc, &rtt, nullptr, nullptr, nullptr); + if (rtt == 0) { + // Waiting for valid rtt. + return true; + } + uint32_t ntp_secs = 0; + uint32_t ntp_frac = 0; + uint32_t rtp_timestamp = 0; + uint32_t received_ntp_secs = 0; + uint32_t received_ntp_frac = 0; + if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, &received_ntp_secs, + &received_ntp_frac, &rtp_timestamp) != 0) { + // Waiting for RTCP. + return true; + } + NtpTime received_ntp(received_ntp_secs, received_ntp_frac); + int64_t time_since_received = + clock_->CurrentNtpInMilliseconds() - received_ntp.ToMs(); + // Don't use old SRs to estimate time. + if (time_since_received <= 1) { + ntp_estimator_.UpdateRtcpTimestamp( + TimeDelta::Millis(rtt), NtpTime(ntp_secs, ntp_frac), rtp_timestamp); + absl::optional remote_to_local_clock_offset = + ntp_estimator_.EstimateRemoteToLocalClockOffset(); + if (remote_to_local_clock_offset.has_value()) { + capture_clock_offset_updater_.SetRemoteToLocalClockOffset( + *remote_to_local_clock_offset); + } + } + + return true; +} + +void RtpVideoStreamReceiver::FrameContinuous(int64_t picture_id) { + if (!nack_module_) + return; + + int seq_num = -1; + { + MutexLock lock(&last_seq_num_mutex_); + auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); + if (seq_num_it != last_seq_num_for_pic_id_.end()) + seq_num = seq_num_it->second; + } + if (seq_num != -1) + nack_module_->ClearUpTo(seq_num); +} + +void RtpVideoStreamReceiver::FrameDecoded(int64_t picture_id) { + int seq_num = -1; + { + MutexLock lock(&last_seq_num_mutex_); + auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); + if (seq_num_it != last_seq_num_for_pic_id_.end()) { + seq_num = seq_num_it->second; + last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(), + ++seq_num_it); + } + } + if (seq_num != -1) { + { + MutexLock lock(&packet_buffer_lock_); + packet_buffer_.ClearTo(seq_num); + int64_t unwrapped_rtp_seq_num = rtp_seq_num_unwrapper_.Unwrap(seq_num); + packet_infos_.erase(packet_infos_.begin(), + packet_infos_.upper_bound(unwrapped_rtp_seq_num)); + } + MutexLock lock(&reference_finder_lock_); + reference_finder_->ClearTo(seq_num); + } +} + +void RtpVideoStreamReceiver::SignalNetworkState(NetworkState state) { + rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode + : RtcpMode::kOff); +} + +void RtpVideoStreamReceiver::StartReceive() { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + receiving_ = true; +} + +void RtpVideoStreamReceiver::StopReceive() { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + receiving_ = false; +} + +void RtpVideoStreamReceiver::UpdateHistograms() { + FecPacketCounter counter = ulpfec_receiver_->GetPacketCounter(); + if (counter.first_packet_time_ms == -1) + return; + + int64_t elapsed_sec = + (clock_->TimeInMilliseconds() - counter.first_packet_time_ms) / 1000; + if (elapsed_sec < metrics::kMinRunTimeInSeconds) + return; + + if (counter.num_packets > 0) { + RTC_HISTOGRAM_PERCENTAGE( + "WebRTC.Video.ReceivedFecPacketsInPercent", + static_cast(counter.num_fec_packets * 100 / counter.num_packets)); + } + if (counter.num_fec_packets > 0) { + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.RecoveredMediaPacketsInPercentOfFec", + static_cast(counter.num_recovered_packets * + 100 / counter.num_fec_packets)); + } + if (config_.rtp.ulpfec_payload_type != -1) { + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.FecBitrateReceivedInKbps", + static_cast(counter.num_bytes * 8 / elapsed_sec / 1000)); + } +} + +void RtpVideoStreamReceiver::InsertSpsPpsIntoTracker(uint8_t payload_type) { + auto codec_params_it = pt_codec_params_.find(payload_type); + if (codec_params_it == pt_codec_params_.end()) + return; + + RTC_LOG(LS_INFO) << "Found out of band supplied codec parameters for" + " payload type: " + << static_cast(payload_type); + + H264SpropParameterSets sprop_decoder; + auto sprop_base64_it = + codec_params_it->second.find(cricket::kH264FmtpSpropParameterSets); + + if (sprop_base64_it == codec_params_it->second.end()) + return; + + if (!sprop_decoder.DecodeSprop(sprop_base64_it->second.c_str())) + return; + + tracker_.InsertSpsPpsNalus(sprop_decoder.sps_nalu(), + sprop_decoder.pps_nalu()); +} + +void RtpVideoStreamReceiver::UpdatePacketReceiveTimestamps( + const RtpPacketReceived& packet, + bool is_keyframe) { + Timestamp now = clock_->CurrentTime(); + { + MutexLock lock(&sync_info_lock_); + if (is_keyframe || + last_received_keyframe_rtp_timestamp_ == packet.Timestamp()) { + last_received_keyframe_rtp_timestamp_ = packet.Timestamp(); + last_received_keyframe_rtp_system_time_ = now; + } + last_received_rtp_system_time_ = now; + last_received_rtp_timestamp_ = packet.Timestamp(); + } + + // Periodically log the RTP header of incoming packets. + if (now.ms() - last_packet_log_ms_ > kPacketLogIntervalMs) { + rtc::StringBuilder ss; + ss << "Packet received on SSRC: " << packet.Ssrc() + << " with payload type: " << static_cast(packet.PayloadType()) + << ", timestamp: " << packet.Timestamp() + << ", sequence number: " << packet.SequenceNumber() + << ", arrival time: " << ToString(packet.arrival_time()); + int32_t time_offset; + if (packet.GetExtension(&time_offset)) { + ss << ", toffset: " << time_offset; + } + uint32_t send_time; + if (packet.GetExtension(&send_time)) { + ss << ", abs send time: " << send_time; + } + RTC_LOG(LS_INFO) << ss.str(); + last_packet_log_ms_ = now.ms(); + } +} + +} // namespace webrtc diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h new file mode 100644 index 0000000000..457885778a --- /dev/null +++ b/video/rtp_video_stream_receiver.h @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ +#define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/sequence_checker.h" +#include "api/transport/field_trial_based_config.h" +#include "api/units/timestamp.h" +#include "api/video/color_space.h" +#include "api/video/video_codec_type.h" +#include "call/rtp_packet_sink_interface.h" +#include "call/syncable.h" +#include "call/video_receive_stream.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" +#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "modules/rtp_rtcp/include/rtp_rtcp.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" +#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_video_header.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" +#include "modules/video_coding/h264_sps_pps_tracker.h" +#include "modules/video_coding/loss_notification_controller.h" +#include "modules/video_coding/packet_buffer.h" +#include "modules/video_coding/rtp_frame_reference_finder.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/numerics/sequence_number_util.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/thread_annotations.h" +#include "video/buffered_frame_decryptor.h" +#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" +#include "video/unique_timestamp_counter.h" + +namespace webrtc { + +class DEPRECATED_NackModule; +class PacketRouter; +class ProcessThread; +class ReceiveStatistics; +class ReceiveStatisticsProxy; +class RtcpRttStats; +class RtpPacketReceived; +class Transport; +class UlpfecReceiver; + +class RtpVideoStreamReceiver : public LossNotificationSender, + public RecoveredPacketReceiver, + public RtpPacketSinkInterface, + public KeyFrameRequestSender, + public OnDecryptedFrameCallback, + public OnDecryptionStatusChangeCallback, + public RtpVideoFrameReceiver { + public: + // A complete frame is a frame which has received all its packets and all its + // references are known. + class OnCompleteFrameCallback { + public: + virtual ~OnCompleteFrameCallback() {} + virtual void OnCompleteFrame(std::unique_ptr frame) = 0; + }; + + // DEPRECATED due to dependency on ReceiveStatisticsProxy. + RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + // The packet router is optional; if provided, the RtpRtcp module for this + // stream is registered as a candidate for sending REMB and transport + // feedback. + PacketRouter* packet_router, + const VideoReceiveStreamInterface::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + // The KeyFrameRequestSender is optional; if not provided, key frame + // requests are sent via the internal RtpRtcp module. + KeyFrameRequestSender* keyframe_request_sender, + OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer, + const FieldTrialsView* field_trials = nullptr); + + RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + // The packet router is optional; if provided, the RtpRtcp module for this + // stream is registered as a candidate for sending REMB and transport + // feedback. + PacketRouter* packet_router, + const VideoReceiveStreamInterface::Config* config, + ReceiveStatistics* rtp_receive_statistics, + RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, + RtcpCnameCallback* rtcp_cname_callback, + ProcessThread* process_thread, + NackSender* nack_sender, + // The KeyFrameRequestSender is optional; if not provided, key frame + // requests are sent via the internal RtpRtcp module. + KeyFrameRequestSender* keyframe_request_sender, + OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer, + const FieldTrialsView* field_trials = nullptr); + ~RtpVideoStreamReceiver() override; + + void AddReceiveCodec(uint8_t payload_type, + VideoCodecType codec_type, + const std::map& codec_params, + bool raw_payload); + + void StartReceive(); + void StopReceive(); + + // Produces the transport-related timestamps; current_delay_ms is left unset. + absl::optional GetSyncInfo() const; + + bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); + + void FrameContinuous(int64_t seq_num); + + void FrameDecoded(int64_t seq_num); + + void SignalNetworkState(NetworkState state); + + // Returns number of different frames seen. + int GetUniqueFramesSeen() const { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + return frame_counter_.GetUniqueSeen(); + } + + // Implements RtpPacketSinkInterface. + void OnRtpPacket(const RtpPacketReceived& packet) override; + + // Public only for tests. + void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, + const RtpPacketReceived& rtp_packet, + const RTPVideoHeader& video); + + // Implements RecoveredPacketReceiver. + void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; + + // Send an RTCP keyframe request. + void RequestKeyFrame() override; + + // Implements LossNotificationSender. + void SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) override; + + bool IsUlpfecEnabled() const; + bool IsRetransmissionsEnabled() const; + + // Returns true if a decryptor is attached and frames can be decrypted. + // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame + // Decryption not SRTP. + bool IsDecryptable() const; + + // Don't use, still experimental. + void RequestPacketRetransmit(const std::vector& sequence_numbers); + + void OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frames); + + // Implements OnDecryptedFrameCallback. + void OnDecryptedFrame(std::unique_ptr frame) override; + + // Implements OnDecryptionStatusChangeCallback. + void OnDecryptionStatusChange( + FrameDecryptorInterface::Status status) override; + + // Optionally set a frame decryptor after a stream has started. This will not + // reset the decoder state. + void SetFrameDecryptor( + rtc::scoped_refptr frame_decryptor); + + // Sets a frame transformer after a stream has started, if no transformer + // has previously been set. Does not reset the decoder state. + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer); + + // Called by VideoReceiveStreamInterface when stats are updated. + void UpdateRtt(int64_t max_rtt_ms); + + absl::optional LastReceivedPacketMs() const; + absl::optional LastReceivedKeyframePacketMs() const; + + // RtpDemuxer only forwards a given RTP packet to one sink. However, some + // sinks, such as FlexFEC, might wish to be informed of all of the packets + // a given sink receives (or any set of sinks). They may do so by registering + // themselves as secondary sinks. + void AddSecondarySink(RtpPacketSinkInterface* sink); + void RemoveSecondarySink(const RtpPacketSinkInterface* sink); + + private: + // Implements RtpVideoFrameReceiver. + void ManageFrame(std::unique_ptr frame) override; + + // Used for buffering RTCP feedback messages and sending them all together. + // Note: + // 1. Key frame requests and NACKs are mutually exclusive, with the + // former taking precedence over the latter. + // 2. Loss notifications are orthogonal to either. (That is, may be sent + // alongside either.) + class RtcpFeedbackBuffer : public KeyFrameRequestSender, + public NackSender, + public LossNotificationSender { + public: + RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, + NackSender* nack_sender, + LossNotificationSender* loss_notification_sender); + + ~RtcpFeedbackBuffer() override = default; + + // KeyFrameRequestSender implementation. + void RequestKeyFrame() RTC_LOCKS_EXCLUDED(mutex_) override; + + // NackSender implementation. + void SendNack(const std::vector& sequence_numbers, + bool buffering_allowed) RTC_LOCKS_EXCLUDED(mutex_) override; + + // LossNotificationSender implementation. + void SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) + RTC_LOCKS_EXCLUDED(mutex_) override; + + // Send all RTCP feedback messages buffered thus far. + void SendBufferedRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_); + + private: + // LNTF-related state. + struct LossNotificationState { + LossNotificationState(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag) + : last_decoded_seq_num(last_decoded_seq_num), + last_received_seq_num(last_received_seq_num), + decodability_flag(decodability_flag) {} + + uint16_t last_decoded_seq_num; + uint16_t last_received_seq_num; + bool decodability_flag; + }; + struct ConsumedRtcpFeedback { + bool request_key_frame = false; + std::vector nack_sequence_numbers; + absl::optional lntf_state; + }; + + ConsumedRtcpFeedback ConsumeRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_); + ConsumedRtcpFeedback ConsumeRtcpFeedbackLocked() + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // This method is called both with and without mutex_ held. + void SendRtcpFeedback(ConsumedRtcpFeedback feedback); + + KeyFrameRequestSender* const key_frame_request_sender_; + NackSender* const nack_sender_; + LossNotificationSender* const loss_notification_sender_; + + // NACKs are accessible from two threads due to nack_module_ being a module. + Mutex mutex_; + + // Key-frame-request-related state. + bool request_key_frame_ RTC_GUARDED_BY(mutex_); + + // NACK-related state. + std::vector nack_sequence_numbers_ RTC_GUARDED_BY(mutex_); + + absl::optional lntf_state_ RTC_GUARDED_BY(mutex_); + }; + enum ParseGenericDependenciesResult { + kDropPacket, + kHasGenericDescriptor, + kNoGenericDescriptor + }; + + // Entry point doing non-stats work for a received packet. Called + // for the same packet both before and after RED decapsulation. + void ReceivePacket(const RtpPacketReceived& packet); + // Parses and handles RED headers. + // This function assumes that it's being called from only one thread. + void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet); + void NotifyReceiverOfEmptyPacket(uint16_t seq_num); + void UpdateHistograms(); + bool IsRedEnabled() const; + void InsertSpsPpsIntoTracker(uint8_t payload_type); + void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result); + ParseGenericDependenciesResult ParseGenericDependenciesExtension( + const RtpPacketReceived& rtp_packet, + RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_); + void OnAssembledFrame(std::unique_ptr frame) + RTC_LOCKS_EXCLUDED(packet_buffer_lock_); + void UpdatePacketReceiveTimestamps(const RtpPacketReceived& packet, + bool is_keyframe) + RTC_RUN_ON(worker_task_checker_); + + const FieldTrialsView& field_trials_; + FieldTrialBasedConfig owned_field_trials_; + + Clock* const clock_; + // Ownership of this object lies with VideoReceiveStreamInterface, which owns + // `this`. + const VideoReceiveStreamInterface::Config& config_; + PacketRouter* const packet_router_; + ProcessThread* const process_thread_; + + RemoteNtpTimeEstimator ntp_estimator_; + + RtpHeaderExtensionMap rtp_header_extensions_; + // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout + // delay that is specified in the received packets. + FieldTrialOptional forced_playout_delay_max_ms_; + FieldTrialOptional forced_playout_delay_min_ms_; + ReceiveStatistics* const rtp_receive_statistics_; + std::unique_ptr ulpfec_receiver_; + + RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_task_checker_; + bool receiving_ RTC_GUARDED_BY(worker_task_checker_); + int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); + + const std::unique_ptr rtp_rtcp_; + + OnCompleteFrameCallback* complete_frame_callback_; + KeyFrameRequestSender* const keyframe_request_sender_; + + RtcpFeedbackBuffer rtcp_feedback_buffer_; + std::unique_ptr nack_module_; + std::unique_ptr loss_notification_controller_; + + mutable Mutex packet_buffer_lock_; + video_coding::PacketBuffer packet_buffer_ RTC_GUARDED_BY(packet_buffer_lock_); + UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_); + SeqNumUnwrapper frame_id_unwrapper_ + RTC_GUARDED_BY(worker_task_checker_); + + // Video structure provided in the dependency descriptor in a first packet + // of a key frame. It is required to parse dependency descriptor in the + // following delta packets. + std::unique_ptr video_structure_ + RTC_GUARDED_BY(worker_task_checker_); + // Frame id of the last frame with the attached video structure. + // absl::nullopt when `video_structure_ == nullptr`; + absl::optional video_structure_frame_id_ + RTC_GUARDED_BY(worker_task_checker_); + + Mutex reference_finder_lock_; + std::unique_ptr reference_finder_ + RTC_GUARDED_BY(reference_finder_lock_); + absl::optional current_codec_; + uint32_t last_assembled_frame_rtp_timestamp_; + + Mutex last_seq_num_mutex_; + std::map last_seq_num_for_pic_id_ + RTC_GUARDED_BY(last_seq_num_mutex_); + video_coding::H264SpsPpsTracker tracker_; + + // Maps payload id to the depacketizer. + std::map> payload_type_map_; + + // TODO(johan): Remove pt_codec_params_ once + // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. + // Maps a payload type to a map of out-of-band supplied codec parameters. + std::map> pt_codec_params_; + int16_t last_payload_type_ = -1; + + bool has_received_frame_; + + std::vector secondary_sinks_ + RTC_GUARDED_BY(worker_task_checker_); + + // Info for GetSyncInfo is updated on network or worker thread, and queried on + // the worker thread. + mutable Mutex sync_info_lock_; + absl::optional last_received_rtp_timestamp_ + RTC_GUARDED_BY(sync_info_lock_); + absl::optional last_received_keyframe_rtp_timestamp_ + RTC_GUARDED_BY(sync_info_lock_); + absl::optional last_received_rtp_system_time_ + RTC_GUARDED_BY(sync_info_lock_); + absl::optional last_received_keyframe_rtp_system_time_ + RTC_GUARDED_BY(sync_info_lock_); + + // Used to validate the buffered frame decryptor is always run on the correct + // thread. + SequenceChecker network_tc_; + // Handles incoming encrypted frames and forwards them to the + // rtp_reference_finder if they are decryptable. + std::unique_ptr buffered_frame_decryptor_ + RTC_PT_GUARDED_BY(network_tc_); + std::atomic frames_decryptable_; + absl::optional last_color_space_; + + AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_ + RTC_GUARDED_BY(worker_task_checker_); + + CaptureClockOffsetUpdater capture_clock_offset_updater_ + RTC_GUARDED_BY(worker_task_checker_); + + int64_t last_completed_picture_id_ = 0; + + rtc::scoped_refptr + frame_transformer_delegate_; + + SeqNumUnwrapper rtp_seq_num_unwrapper_ + RTC_GUARDED_BY(packet_buffer_lock_); + std::map packet_infos_ + RTC_GUARDED_BY(packet_buffer_lock_); +}; + +} // namespace webrtc + +#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ diff --git a/video/rtp_video_stream_receiver_unittest.cc b/video/rtp_video_stream_receiver_unittest.cc new file mode 100644 index 0000000000..bfe9a4cc89 --- /dev/null +++ b/video/rtp_video_stream_receiver_unittest.cc @@ -0,0 +1,1238 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/rtp_video_stream_receiver.h" + +#include +#include + +#include "api/video/video_codec_type.h" +#include "api/video/video_frame_type.h" +#include "call/test/mock_rtp_packet_sink_interface.h" +#include "common_video/h264/h264_common.h" +#include "media/base/media_constants.h" +#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" +#include "modules/rtp_rtcp/source/rtp_format.h" +#include "modules/rtp_rtcp/source/rtp_format_vp9.h" +#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" +#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_header_extensions.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/utility/include/process_thread.h" +#include "modules/video_coding/frame_object.h" +#include "modules/video_coding/include/video_coding_defines.h" +#include "modules/video_coding/rtp_frame_reference_finder.h" +#include "rtc_base/byte_buffer.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_frame_transformer.h" +#include "test/mock_transport.h" +#include "test/scoped_key_value_config.h" + +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::Invoke; +using ::testing::SizeIs; +using ::testing::Values; + +namespace webrtc { + +namespace { + +const uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01}; + +std::vector GetAbsoluteCaptureTimestamps(const EncodedFrame* frame) { + std::vector result; + for (const auto& packet_info : frame->PacketInfos()) { + if (packet_info.absolute_capture_time()) { + result.push_back( + packet_info.absolute_capture_time()->absolute_capture_timestamp); + } + } + return result; +} + +RTPVideoHeader GetGenericVideoHeader(VideoFrameType frame_type) { + RTPVideoHeader video_header; + video_header.is_first_packet_in_frame = true; + video_header.is_last_packet_in_frame = true; + video_header.codec = kVideoCodecGeneric; + video_header.frame_type = frame_type; + return video_header; +} + +class MockNackSender : public NackSender { + public: + MOCK_METHOD(void, + SendNack, + (const std::vector& sequence_numbers, + bool buffering_allowed), + (override)); +}; + +class MockKeyFrameRequestSender : public KeyFrameRequestSender { + public: + MOCK_METHOD(void, RequestKeyFrame, (), (override)); +}; + +class MockOnCompleteFrameCallback + : public RtpVideoStreamReceiver::OnCompleteFrameCallback { + public: + MOCK_METHOD(void, DoOnCompleteFrame, (EncodedFrame*), ()); + MOCK_METHOD(void, DoOnCompleteFrameFailNullptr, (EncodedFrame*), ()); + MOCK_METHOD(void, DoOnCompleteFrameFailLength, (EncodedFrame*), ()); + MOCK_METHOD(void, DoOnCompleteFrameFailBitstream, (EncodedFrame*), ()); + void OnCompleteFrame(std::unique_ptr frame) override { + if (!frame) { + DoOnCompleteFrameFailNullptr(nullptr); + return; + } + EXPECT_EQ(buffer_.Length(), frame->size()); + if (buffer_.Length() != frame->size()) { + DoOnCompleteFrameFailLength(frame.get()); + return; + } + if (frame->size() != buffer_.Length() || + memcmp(buffer_.Data(), frame->data(), buffer_.Length()) != 0) { + DoOnCompleteFrameFailBitstream(frame.get()); + return; + } + DoOnCompleteFrame(frame.get()); + } + + void ClearExpectedBitstream() { buffer_.Clear(); } + + void AppendExpectedBitstream(const uint8_t data[], size_t size_in_bytes) { + // TODO(Johan): Let rtc::ByteBuffer handle uint8_t* instead of char*. + buffer_.WriteBytes(reinterpret_cast(data), size_in_bytes); + } + rtc::ByteBufferWriter buffer_; +}; + +constexpr uint32_t kSsrc = 111; +constexpr uint16_t kSequenceNumber = 222; +constexpr int kPayloadType = 100; +constexpr int kRedPayloadType = 125; + +std::unique_ptr CreateRtpPacketReceived() { + auto packet = std::make_unique(); + packet->SetSsrc(kSsrc); + packet->SetSequenceNumber(kSequenceNumber); + packet->SetPayloadType(kPayloadType); + return packet; +} + +MATCHER_P(SamePacketAs, other, "") { + return arg.Ssrc() == other.Ssrc() && + arg.SequenceNumber() == other.SequenceNumber(); +} + +} // namespace + +class RtpVideoStreamReceiverTest : public ::testing::Test { + public: + RtpVideoStreamReceiverTest() : RtpVideoStreamReceiverTest("") {} + explicit RtpVideoStreamReceiverTest(std::string field_trials) + : field_trials_(field_trials), + config_(CreateConfig()), + process_thread_(ProcessThread::Create("TestThread")) { + rtp_receive_statistics_ = + ReceiveStatistics::Create(Clock::GetRealTimeClock()); + rtp_video_stream_receiver_ = std::make_unique( + Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, + rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(), + &mock_nack_sender_, &mock_key_frame_request_sender_, + &mock_on_complete_frame_callback_, nullptr, nullptr, &field_trials_); + rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, + kVideoCodecGeneric, {}, + /*raw_payload=*/false); + } + + RTPVideoHeader GetDefaultH264VideoHeader() { + RTPVideoHeader video_header; + video_header.codec = kVideoCodecH264; + video_header.video_type_header.emplace(); + return video_header; + } + + // TODO(Johan): refactor h264_sps_pps_tracker_unittests.cc to avoid duplicate + // code. + void AddSps(RTPVideoHeader* video_header, + uint8_t sps_id, + rtc::CopyOnWriteBuffer* data) { + NaluInfo info; + info.type = H264::NaluType::kSps; + info.sps_id = sps_id; + info.pps_id = -1; + data->AppendData({H264::NaluType::kSps, sps_id}); + auto& h264 = absl::get(video_header->video_type_header); + h264.nalus[h264.nalus_length++] = info; + } + + void AddPps(RTPVideoHeader* video_header, + uint8_t sps_id, + uint8_t pps_id, + rtc::CopyOnWriteBuffer* data) { + NaluInfo info; + info.type = H264::NaluType::kPps; + info.sps_id = sps_id; + info.pps_id = pps_id; + data->AppendData({H264::NaluType::kPps, pps_id}); + auto& h264 = absl::get(video_header->video_type_header); + h264.nalus[h264.nalus_length++] = info; + } + + void AddIdr(RTPVideoHeader* video_header, int pps_id) { + NaluInfo info; + info.type = H264::NaluType::kIdr; + info.sps_id = -1; + info.pps_id = pps_id; + auto& h264 = absl::get(video_header->video_type_header); + h264.nalus[h264.nalus_length++] = info; + } + + protected: + static VideoReceiveStreamInterface::Config CreateConfig() { + VideoReceiveStreamInterface::Config config(nullptr); + config.rtp.remote_ssrc = 1111; + config.rtp.local_ssrc = 2222; + config.rtp.red_payload_type = kRedPayloadType; + return config; + } + + webrtc::test::ScopedKeyValueConfig field_trials_; + VideoReceiveStreamInterface::Config config_; + MockNackSender mock_nack_sender_; + MockKeyFrameRequestSender mock_key_frame_request_sender_; + MockTransport mock_transport_; + MockOnCompleteFrameCallback mock_on_complete_frame_callback_; + std::unique_ptr process_thread_; + std::unique_ptr rtp_receive_statistics_; + std::unique_ptr rtp_video_stream_receiver_; +}; + +TEST_F(RtpVideoStreamReceiverTest, CacheColorSpaceFromLastPacketOfKeyframe) { + // Test that color space is cached from the last packet of a key frame and + // that it's not reset by padding packets without color space. + constexpr int kVp9PayloadType = 99; + const ColorSpace kColorSpace( + ColorSpace::PrimaryID::kFILM, ColorSpace::TransferID::kBT2020_12, + ColorSpace::MatrixID::kBT2020_NCL, ColorSpace::RangeID::kFull); + const std::vector kKeyFramePayload = {0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10}; + const std::vector kDeltaFramePayload = {0, 1, 2, 3, 4}; + + // Anonymous helper class that generates received packets. + class { + public: + void SetPayload(const std::vector& payload, + VideoFrameType video_frame_type) { + video_frame_type_ = video_frame_type; + RtpPacketizer::PayloadSizeLimits pay_load_size_limits; + // Reduce max payload length to make sure the key frame generates two + // packets. + pay_load_size_limits.max_payload_len = 8; + RTPVideoHeaderVP9 rtp_video_header_vp9; + rtp_video_header_vp9.InitRTPVideoHeaderVP9(); + rtp_video_header_vp9.inter_pic_predicted = + (video_frame_type == VideoFrameType::kVideoFrameDelta); + rtp_packetizer_ = std::make_unique( + payload, pay_load_size_limits, rtp_video_header_vp9); + } + + size_t NumPackets() { return rtp_packetizer_->NumPackets(); } + void SetColorSpace(const ColorSpace& color_space) { + color_space_ = color_space; + } + + RtpPacketReceived NextPacket() { + RtpHeaderExtensionMap extension_map; + extension_map.Register(1); + RtpPacketToSend packet_to_send(&extension_map); + packet_to_send.SetSequenceNumber(sequence_number_++); + packet_to_send.SetSsrc(kSsrc); + packet_to_send.SetPayloadType(kVp9PayloadType); + bool include_color_space = + (rtp_packetizer_->NumPackets() == 1u && + video_frame_type_ == VideoFrameType::kVideoFrameKey); + if (include_color_space) { + EXPECT_TRUE( + packet_to_send.SetExtension(color_space_)); + } + rtp_packetizer_->NextPacket(&packet_to_send); + + RtpPacketReceived received_packet(&extension_map); + received_packet.Parse(packet_to_send.data(), packet_to_send.size()); + return received_packet; + } + + private: + uint16_t sequence_number_ = 0; + VideoFrameType video_frame_type_; + ColorSpace color_space_; + std::unique_ptr rtp_packetizer_; + } received_packet_generator; + received_packet_generator.SetColorSpace(kColorSpace); + + // Prepare the receiver for VP9. + std::map codec_params; + rtp_video_stream_receiver_->AddReceiveCodec(kVp9PayloadType, kVideoCodecVP9, + codec_params, + /*raw_payload=*/false); + + // Generate key frame packets. + received_packet_generator.SetPayload(kKeyFramePayload, + VideoFrameType::kVideoFrameKey); + EXPECT_EQ(received_packet_generator.NumPackets(), 2u); + RtpPacketReceived key_frame_packet1 = received_packet_generator.NextPacket(); + RtpPacketReceived key_frame_packet2 = received_packet_generator.NextPacket(); + + // Generate delta frame packet. + received_packet_generator.SetPayload(kDeltaFramePayload, + VideoFrameType::kVideoFrameDelta); + EXPECT_EQ(received_packet_generator.NumPackets(), 1u); + RtpPacketReceived delta_frame_packet = received_packet_generator.NextPacket(); + + rtp_video_stream_receiver_->StartReceive(); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kKeyFramePayload.data(), kKeyFramePayload.size()); + + // Send the key frame and expect a callback with color space information. + EXPECT_FALSE(key_frame_packet1.GetExtension()); + EXPECT_TRUE(key_frame_packet2.GetExtension()); + rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([kColorSpace](EncodedFrame* frame) { + ASSERT_TRUE(frame->EncodedImage().ColorSpace()); + EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace); + })); + rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet2); + // Resend the first key frame packet to simulate padding for example. + rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1); + + mock_on_complete_frame_callback_.ClearExpectedBitstream(); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kDeltaFramePayload.data(), kDeltaFramePayload.size()); + + // Expect delta frame to have color space set even though color space not + // included in the RTP packet. + EXPECT_FALSE(delta_frame_packet.GetExtension()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([kColorSpace](EncodedFrame* frame) { + ASSERT_TRUE(frame->EncodedImage().ColorSpace()); + EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace); + })); + rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet); +} + +TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrame) { + RtpPacketReceived rtp_packet; + rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(1); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiverTest, PacketInfoIsPropagatedIntoVideoFrames) { + constexpr uint64_t kAbsoluteCaptureTimestamp = 12; + constexpr int kId0 = 1; + + RtpHeaderExtensionMap extension_map; + extension_map.Register(kId0); + RtpPacketReceived rtp_packet(&extension_map); + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); + rtp_packet.SetSequenceNumber(1); + rtp_packet.SetTimestamp(1); + rtp_packet.SetSsrc(kSsrc); + rtp_packet.SetExtension( + AbsoluteCaptureTime{kAbsoluteCaptureTimestamp, + /*estimated_capture_clock_offset=*/absl::nullopt}); + + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([kAbsoluteCaptureTimestamp](EncodedFrame* frame) { + EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), + ElementsAre(kAbsoluteCaptureTimestamp)); + })); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiverTest, + MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue) { + constexpr uint64_t kAbsoluteCaptureTimestamp = 12; + constexpr int kId0 = 1; + + RtpHeaderExtensionMap extension_map; + extension_map.Register(kId0); + RtpPacketReceived rtp_packet(&extension_map); + rtp_packet.SetPayloadType(kPayloadType); + + rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); + uint16_t sequence_number = 1; + uint32_t rtp_timestamp = 1; + rtp_packet.SetSequenceNumber(sequence_number); + rtp_packet.SetTimestamp(rtp_timestamp); + rtp_packet.SetSsrc(kSsrc); + rtp_packet.SetExtension( + AbsoluteCaptureTime{kAbsoluteCaptureTimestamp, + /*estimated_capture_clock_offset=*/absl::nullopt}); + + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + + // Rtp packet without absolute capture time. + rtp_packet = RtpPacketReceived(&extension_map); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(++sequence_number); + rtp_packet.SetTimestamp(++rtp_timestamp); + rtp_packet.SetSsrc(kSsrc); + + // There is no absolute capture time in the second packet. + // Expect rtp video stream receiver to extrapolate it for the resulting video + // frame using absolute capture time from the previous packet. + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([](EncodedFrame* frame) { + EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), SizeIs(1)); + })); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiverTest, NoInfiniteRecursionOnEncapsulatedRedPacket) { + const std::vector data({ + 0x80, // RTP version. + kRedPayloadType, // Payload type. + 0, 0, 0, 0, 0, 0, // Don't care. + 0, 0, 0x4, 0x57, // SSRC + kRedPayloadType, // RED header. + 0, 0, 0, 0, 0 // Don't care. + }); + RtpPacketReceived packet; + EXPECT_TRUE(packet.Parse(data.data(), data.size())); + rtp_video_stream_receiver_->StartReceive(); + rtp_video_stream_receiver_->OnRtpPacket(packet); +} + +TEST_F(RtpVideoStreamReceiverTest, + DropsPacketWithRedPayloadTypeAndEmptyPayload) { + const uint8_t kRedPayloadType = 125; + config_.rtp.red_payload_type = kRedPayloadType; + SetUp(); // re-create rtp_video_stream_receiver with red payload type. + // clang-format off + const uint8_t data[] = { + 0x80, // RTP version. + kRedPayloadType, // Payload type. + 0, 0, 0, 0, 0, 0, // Don't care. + 0, 0, 0x4, 0x57, // SSRC + // Empty rtp payload. + }; + // clang-format on + RtpPacketReceived packet; + // Manually convert to CopyOnWriteBuffer to be sure capacity == size + // and asan bot can catch read buffer overflow. + EXPECT_TRUE(packet.Parse(rtc::CopyOnWriteBuffer(data))); + rtp_video_stream_receiver_->StartReceive(); + rtp_video_stream_receiver_->OnRtpPacket(packet); + // Expect asan doesn't find anything. +} + +TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrameBitstreamError) { + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); + rtp_packet.SetSequenceNumber(1); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + constexpr uint8_t expected_bitsteam[] = {1, 2, 3, 0xff}; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + expected_bitsteam, sizeof(expected_bitsteam)); + EXPECT_CALL(mock_on_complete_frame_callback_, + DoOnCompleteFrameFailBitstream(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +class RtpVideoStreamReceiverTestH264 + : public RtpVideoStreamReceiverTest, + public ::testing::WithParamInterface { + protected: + RtpVideoStreamReceiverTestH264() : RtpVideoStreamReceiverTest(GetParam()) {} +}; + +INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe, + RtpVideoStreamReceiverTestH264, + Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/")); + +TEST_P(RtpVideoStreamReceiverTestH264, InBandSpsPps) { + rtc::CopyOnWriteBuffer sps_data; + RtpPacketReceived rtp_packet; + RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader(); + AddSps(&sps_video_header, 0, &sps_data); + rtp_packet.SetSequenceNumber(0); + rtp_packet.SetPayloadType(kPayloadType); + sps_video_header.is_first_packet_in_frame = true; + sps_video_header.frame_type = VideoFrameType::kEmptyFrame; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(), + sps_data.size()); + rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet, + sps_video_header); + + rtc::CopyOnWriteBuffer pps_data; + RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader(); + AddPps(&pps_video_header, 0, 1, &pps_data); + rtp_packet.SetSequenceNumber(1); + pps_video_header.is_first_packet_in_frame = true; + pps_video_header.frame_type = VideoFrameType::kEmptyFrame; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(), + pps_data.size()); + rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet, + pps_video_header); + + rtc::CopyOnWriteBuffer idr_data; + RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader(); + AddIdr(&idr_video_header, 1); + rtp_packet.SetSequenceNumber(2); + idr_video_header.is_first_packet_in_frame = true; + idr_video_header.is_last_packet_in_frame = true; + idr_video_header.frame_type = VideoFrameType::kVideoFrameKey; + const uint8_t idr[] = {0x65, 1, 2, 3}; + idr_data.AppendData(idr); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), + idr_data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet, + idr_video_header); +} + +TEST_P(RtpVideoStreamReceiverTestH264, OutOfBandFmtpSpsPps) { + constexpr int kPayloadType = 99; + std::map codec_params; + // Example parameter sets from https://tools.ietf.org/html/rfc3984#section-8.2 + // . + codec_params.insert( + {cricket::kH264FmtpSpropParameterSets, "Z0IACpZTBYmI,aMljiA=="}); + rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecH264, + codec_params, + /*raw_payload=*/false); + const uint8_t binary_sps[] = {0x67, 0x42, 0x00, 0x0a, 0x96, + 0x53, 0x05, 0x89, 0x88}; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_sps, + sizeof(binary_sps)); + const uint8_t binary_pps[] = {0x68, 0xc9, 0x63, 0x88}; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_pps, + sizeof(binary_pps)); + + RtpPacketReceived rtp_packet; + RTPVideoHeader video_header = GetDefaultH264VideoHeader(); + AddIdr(&video_header, 0); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(2); + video_header.is_first_packet_in_frame = true; + video_header.is_last_packet_in_frame = true; + video_header.codec = kVideoCodecH264; + video_header.frame_type = VideoFrameType::kVideoFrameKey; + rtc::CopyOnWriteBuffer data({'1', '2', '3'}); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_P(RtpVideoStreamReceiverTestH264, ForceSpsPpsIdrIsKeyframe) { + constexpr int kPayloadType = 99; + std::map codec_params; + if (GetParam() == + "") { // Forcing can be done either with field trial or codec_params. + codec_params.insert({cricket::kH264FmtpSpsPpsIdrInKeyframe, ""}); + } + rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecH264, + codec_params, + /*raw_payload=*/false); + rtc::CopyOnWriteBuffer sps_data; + RtpPacketReceived rtp_packet; + RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader(); + AddSps(&sps_video_header, 0, &sps_data); + rtp_packet.SetSequenceNumber(0); + rtp_packet.SetPayloadType(kPayloadType); + sps_video_header.is_first_packet_in_frame = true; + sps_video_header.frame_type = VideoFrameType::kEmptyFrame; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(), + sps_data.size()); + rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet, + sps_video_header); + + rtc::CopyOnWriteBuffer pps_data; + RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader(); + AddPps(&pps_video_header, 0, 1, &pps_data); + rtp_packet.SetSequenceNumber(1); + pps_video_header.is_first_packet_in_frame = true; + pps_video_header.frame_type = VideoFrameType::kEmptyFrame; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(), + pps_data.size()); + rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet, + pps_video_header); + + rtc::CopyOnWriteBuffer idr_data; + RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader(); + AddIdr(&idr_video_header, 1); + rtp_packet.SetSequenceNumber(2); + idr_video_header.is_first_packet_in_frame = true; + idr_video_header.is_last_packet_in_frame = true; + idr_video_header.frame_type = VideoFrameType::kVideoFrameKey; + const uint8_t idr[] = {0x65, 1, 2, 3}; + idr_data.AppendData(idr); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), + idr_data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce( + [&](EncodedFrame* frame) { EXPECT_TRUE(frame->is_keyframe()); }); + rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet, + idr_video_header); + mock_on_complete_frame_callback_.ClearExpectedBitstream(); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), + idr_data.size()); + rtp_packet.SetSequenceNumber(3); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce( + [&](EncodedFrame* frame) { EXPECT_FALSE(frame->is_keyframe()); }); + rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet, + idr_video_header); +} + +TEST_F(RtpVideoStreamReceiverTest, PaddingInMediaStream) { + RtpPacketReceived rtp_packet; + RTPVideoHeader video_header = GetDefaultH264VideoHeader(); + rtc::CopyOnWriteBuffer data({'1', '2', '3'}); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(2); + video_header.is_first_packet_in_frame = true; + video_header.is_last_packet_in_frame = true; + video_header.codec = kVideoCodecGeneric; + video_header.frame_type = VideoFrameType::kVideoFrameKey; + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + + rtp_packet.SetSequenceNumber(3); + rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet, + video_header); + + rtp_packet.SetSequenceNumber(4); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + video_header.frame_type = VideoFrameType::kVideoFrameDelta; + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + + rtp_packet.SetSequenceNumber(6); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_packet.SetSequenceNumber(5); + rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeIfFirstFrameIsDelta) { + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); + rtp_packet.SetSequenceNumber(1); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta); + EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame()); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeWhenPacketBufferGetsFull) { + constexpr int kPacketBufferMaxSize = 2048; + + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta); + // Incomplete frames so that the packet buffer is filling up. + video_header.is_last_packet_in_frame = false; + uint16_t start_sequence_number = 1234; + rtp_packet.SetSequenceNumber(start_sequence_number); + while (rtp_packet.SequenceNumber() - start_sequence_number < + kPacketBufferMaxSize) { + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + rtp_packet.SetSequenceNumber(rtp_packet.SequenceNumber() + 2); + } + + EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame()); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiverTest, SecondarySinksGetRtpNotifications) { + rtp_video_stream_receiver_->StartReceive(); + + MockRtpPacketSink secondary_sink_1; + MockRtpPacketSink secondary_sink_2; + + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_1); + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_2); + + auto rtp_packet = CreateRtpPacketReceived(); + EXPECT_CALL(secondary_sink_1, OnRtpPacket(SamePacketAs(*rtp_packet))); + EXPECT_CALL(secondary_sink_2, OnRtpPacket(SamePacketAs(*rtp_packet))); + + rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); + + // Test tear-down. + rtp_video_stream_receiver_->StopReceive(); + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_1); + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_2); +} + +TEST_F(RtpVideoStreamReceiverTest, RemovedSecondarySinksGetNoRtpNotifications) { + rtp_video_stream_receiver_->StartReceive(); + + MockRtpPacketSink secondary_sink; + + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); + + auto rtp_packet = CreateRtpPacketReceived(); + + EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0); + + rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); + + // Test tear-down. + rtp_video_stream_receiver_->StopReceive(); +} + +TEST_F(RtpVideoStreamReceiverTest, + OnlyRemovedSecondarySinksExcludedFromNotifications) { + rtp_video_stream_receiver_->StartReceive(); + + MockRtpPacketSink kept_secondary_sink; + MockRtpPacketSink removed_secondary_sink; + + rtp_video_stream_receiver_->AddSecondarySink(&kept_secondary_sink); + rtp_video_stream_receiver_->AddSecondarySink(&removed_secondary_sink); + rtp_video_stream_receiver_->RemoveSecondarySink(&removed_secondary_sink); + + auto rtp_packet = CreateRtpPacketReceived(); + EXPECT_CALL(kept_secondary_sink, OnRtpPacket(SamePacketAs(*rtp_packet))); + + rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); + + // Test tear-down. + rtp_video_stream_receiver_->StopReceive(); + rtp_video_stream_receiver_->RemoveSecondarySink(&kept_secondary_sink); +} + +TEST_F(RtpVideoStreamReceiverTest, + SecondariesOfNonStartedStreamGetNoNotifications) { + // Explicitly showing that the stream is not in the `started` state, + // regardless of whether streams start out `started` or `stopped`. + rtp_video_stream_receiver_->StopReceive(); + + MockRtpPacketSink secondary_sink; + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); + + auto rtp_packet = CreateRtpPacketReceived(); + EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0); + + rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); + + // Test tear-down. + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); +} + +TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorOnePacket) { + const std::vector data = {0, 1, 2, 3, 4}; + const int kSpatialIndex = 1; + + rtp_video_stream_receiver_->StartReceive(); + + RtpHeaderExtensionMap extension_map; + extension_map.Register(5); + RtpPacketReceived rtp_packet(&extension_map); + rtp_packet.SetPayloadType(kPayloadType); + + RtpGenericFrameDescriptor generic_descriptor; + generic_descriptor.SetFirstPacketInSubFrame(true); + generic_descriptor.SetLastPacketInSubFrame(true); + generic_descriptor.SetFrameId(100); + generic_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex); + generic_descriptor.AddFrameDependencyDiff(90); + generic_descriptor.AddFrameDependencyDiff(80); + ASSERT_TRUE(rtp_packet.SetExtension( + generic_descriptor)); + + uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); + memcpy(payload, data.data(), data.size()); + // The first byte is the header, so we ignore the first byte of `data`. + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, + data.size() - 1); + + rtp_packet.SetMarker(true); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(1); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce(Invoke([kSpatialIndex](EncodedFrame* frame) { + EXPECT_EQ(frame->num_references, 2U); + EXPECT_EQ(frame->references[0], frame->Id() - 90); + EXPECT_EQ(frame->references[1], frame->Id() - 80); + EXPECT_EQ(frame->SpatialIndex(), kSpatialIndex); + EXPECT_THAT(frame->PacketInfos(), SizeIs(1)); + })); + + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); +} + +TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorTwoPackets) { + const std::vector data = {0, 1, 2, 3, 4}; + const int kSpatialIndex = 1; + + rtp_video_stream_receiver_->StartReceive(); + + RtpHeaderExtensionMap extension_map; + extension_map.Register(5); + RtpPacketReceived first_packet(&extension_map); + + RtpGenericFrameDescriptor first_packet_descriptor; + first_packet_descriptor.SetFirstPacketInSubFrame(true); + first_packet_descriptor.SetLastPacketInSubFrame(false); + first_packet_descriptor.SetFrameId(100); + first_packet_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex); + first_packet_descriptor.SetResolution(480, 360); + ASSERT_TRUE(first_packet.SetExtension( + first_packet_descriptor)); + + uint8_t* first_packet_payload = first_packet.SetPayloadSize(data.size()); + memcpy(first_packet_payload, data.data(), data.size()); + // The first byte is the header, so we ignore the first byte of `data`. + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, + data.size() - 1); + + first_packet.SetPayloadType(kPayloadType); + first_packet.SetSequenceNumber(1); + rtp_video_stream_receiver_->OnRtpPacket(first_packet); + + RtpPacketReceived second_packet(&extension_map); + RtpGenericFrameDescriptor second_packet_descriptor; + second_packet_descriptor.SetFirstPacketInSubFrame(false); + second_packet_descriptor.SetLastPacketInSubFrame(true); + ASSERT_TRUE(second_packet.SetExtension( + second_packet_descriptor)); + + second_packet.SetMarker(true); + second_packet.SetPayloadType(kPayloadType); + second_packet.SetSequenceNumber(2); + + uint8_t* second_packet_payload = second_packet.SetPayloadSize(data.size()); + memcpy(second_packet_payload, data.data(), data.size()); + // The first byte is the header, so we ignore the first byte of `data`. + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, + data.size() - 1); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce(Invoke([kSpatialIndex](EncodedFrame* frame) { + EXPECT_EQ(frame->num_references, 0U); + EXPECT_EQ(frame->SpatialIndex(), kSpatialIndex); + EXPECT_EQ(frame->EncodedImage()._encodedWidth, 480u); + EXPECT_EQ(frame->EncodedImage()._encodedHeight, 360u); + EXPECT_THAT(frame->PacketInfos(), SizeIs(2)); + })); + + rtp_video_stream_receiver_->OnRtpPacket(second_packet); +} + +TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorRawPayload) { + const std::vector data = {0, 1, 2, 3, 4}; + const int kRawPayloadType = 123; + + rtp_video_stream_receiver_->AddReceiveCodec(kRawPayloadType, + kVideoCodecGeneric, {}, + /*raw_payload=*/true); + rtp_video_stream_receiver_->StartReceive(); + + RtpHeaderExtensionMap extension_map; + extension_map.Register(5); + RtpPacketReceived rtp_packet(&extension_map); + + RtpGenericFrameDescriptor generic_descriptor; + generic_descriptor.SetFirstPacketInSubFrame(true); + generic_descriptor.SetLastPacketInSubFrame(true); + ASSERT_TRUE(rtp_packet.SetExtension( + generic_descriptor)); + + uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); + memcpy(payload, data.data(), data.size()); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + + rtp_packet.SetMarker(true); + rtp_packet.SetPayloadType(kRawPayloadType); + rtp_packet.SetSequenceNumber(1); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); +} + +TEST_F(RtpVideoStreamReceiverTest, UnwrapsFrameId) { + const std::vector data = {0, 1, 2, 3, 4}; + const int kPayloadType = 123; + + rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecGeneric, + {}, + /*raw_payload=*/true); + rtp_video_stream_receiver_->StartReceive(); + RtpHeaderExtensionMap extension_map; + extension_map.Register(5); + + uint16_t rtp_sequence_number = 1; + auto inject_packet = [&](uint16_t wrapped_frame_id) { + RtpPacketReceived rtp_packet(&extension_map); + + RtpGenericFrameDescriptor generic_descriptor; + generic_descriptor.SetFirstPacketInSubFrame(true); + generic_descriptor.SetLastPacketInSubFrame(true); + generic_descriptor.SetFrameId(wrapped_frame_id); + ASSERT_TRUE(rtp_packet.SetExtension( + generic_descriptor)); + + uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); + ASSERT_TRUE(payload); + memcpy(payload, data.data(), data.size()); + mock_on_complete_frame_callback_.ClearExpectedBitstream(); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + rtp_packet.SetMarker(true); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(++rtp_sequence_number); + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); + }; + + int64_t first_picture_id; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](EncodedFrame* frame) { first_picture_id = frame->Id(); }); + inject_packet(/*wrapped_frame_id=*/0xffff); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](EncodedFrame* frame) { + EXPECT_EQ(frame->Id() - first_picture_id, 3); + }); + inject_packet(/*wrapped_frame_id=*/0x0002); +} + +class RtpVideoStreamReceiverDependencyDescriptorTest + : public RtpVideoStreamReceiverTest { + public: + RtpVideoStreamReceiverDependencyDescriptorTest() { + rtp_video_stream_receiver_->AddReceiveCodec(payload_type_, + kVideoCodecGeneric, {}, + /*raw_payload=*/true); + extension_map_.Register(7); + rtp_video_stream_receiver_->StartReceive(); + } + + // Returns some valid structure for the DependencyDescriptors. + // First template of that structure always fit for a key frame. + static FrameDependencyStructure CreateStreamStructure() { + FrameDependencyStructure stream_structure; + stream_structure.num_decode_targets = 1; + stream_structure.templates = { + FrameDependencyTemplate().Dtis("S"), + FrameDependencyTemplate().Dtis("S").FrameDiffs({1}), + }; + return stream_structure; + } + + void InjectPacketWith(const FrameDependencyStructure& stream_structure, + const DependencyDescriptor& dependency_descriptor) { + const std::vector data = {0, 1, 2, 3, 4}; + RtpPacketReceived rtp_packet(&extension_map_); + ASSERT_TRUE(rtp_packet.SetExtension( + stream_structure, dependency_descriptor)); + uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); + ASSERT_TRUE(payload); + memcpy(payload, data.data(), data.size()); + mock_on_complete_frame_callback_.ClearExpectedBitstream(); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + rtp_packet.SetMarker(true); + rtp_packet.SetPayloadType(payload_type_); + rtp_packet.SetSequenceNumber(++rtp_sequence_number_); + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); + } + + private: + const int payload_type_ = 123; + RtpHeaderExtensionMap extension_map_; + uint16_t rtp_sequence_number_ = 321; +}; + +TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest, UnwrapsFrameId) { + FrameDependencyStructure stream_structure = CreateStreamStructure(); + + DependencyDescriptor keyframe_descriptor; + keyframe_descriptor.attached_structure = + std::make_unique(stream_structure); + keyframe_descriptor.frame_dependencies = stream_structure.templates[0]; + keyframe_descriptor.frame_number = 0xfff0; + // DependencyDescriptor doesn't support reordering delta frame before + // keyframe. Thus feed a key frame first, then test reodered delta frames. + int64_t first_picture_id; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](EncodedFrame* frame) { first_picture_id = frame->Id(); }); + InjectPacketWith(stream_structure, keyframe_descriptor); + + DependencyDescriptor deltaframe1_descriptor; + deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1]; + deltaframe1_descriptor.frame_number = 0xfffe; + + DependencyDescriptor deltaframe2_descriptor; + deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1]; + deltaframe2_descriptor.frame_number = 0x0002; + + // Parser should unwrap frame ids correctly even if packets were reordered by + // the network. + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](EncodedFrame* frame) { + // 0x0002 - 0xfff0 + EXPECT_EQ(frame->Id() - first_picture_id, 18); + }) + .WillOnce([&](EncodedFrame* frame) { + // 0xfffe - 0xfff0 + EXPECT_EQ(frame->Id() - first_picture_id, 14); + }); + InjectPacketWith(stream_structure, deltaframe2_descriptor); + InjectPacketWith(stream_structure, deltaframe1_descriptor); +} + +TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest, + DropsLateDeltaFramePacketWithDependencyDescriptorExtension) { + FrameDependencyStructure stream_structure1 = CreateStreamStructure(); + FrameDependencyStructure stream_structure2 = CreateStreamStructure(); + // Make sure template ids for these two structures do not collide: + // adjust structure_id (that is also used as template id offset). + stream_structure1.structure_id = 13; + stream_structure2.structure_id = + stream_structure1.structure_id + stream_structure1.templates.size(); + + DependencyDescriptor keyframe1_descriptor; + keyframe1_descriptor.attached_structure = + std::make_unique(stream_structure1); + keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0]; + keyframe1_descriptor.frame_number = 1; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); + InjectPacketWith(stream_structure1, keyframe1_descriptor); + + // Pass in 2nd key frame with different structure. + DependencyDescriptor keyframe2_descriptor; + keyframe2_descriptor.attached_structure = + std::make_unique(stream_structure2); + keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0]; + keyframe2_descriptor.frame_number = 3; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); + InjectPacketWith(stream_structure2, keyframe2_descriptor); + + // Pass in late delta frame that uses structure of the 1st key frame. + DependencyDescriptor deltaframe_descriptor; + deltaframe_descriptor.frame_dependencies = stream_structure1.templates[0]; + deltaframe_descriptor.frame_number = 2; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame).Times(0); + InjectPacketWith(stream_structure1, deltaframe_descriptor); +} + +TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest, + DropsLateKeyFramePacketWithDependencyDescriptorExtension) { + FrameDependencyStructure stream_structure1 = CreateStreamStructure(); + FrameDependencyStructure stream_structure2 = CreateStreamStructure(); + // Make sure template ids for these two structures do not collide: + // adjust structure_id (that is also used as template id offset). + stream_structure1.structure_id = 13; + stream_structure2.structure_id = + stream_structure1.structure_id + stream_structure1.templates.size(); + + DependencyDescriptor keyframe1_descriptor; + keyframe1_descriptor.attached_structure = + std::make_unique(stream_structure1); + keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0]; + keyframe1_descriptor.frame_number = 1; + + DependencyDescriptor keyframe2_descriptor; + keyframe2_descriptor.attached_structure = + std::make_unique(stream_structure2); + keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0]; + keyframe2_descriptor.frame_number = 3; + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce( + [&](EncodedFrame* frame) { EXPECT_EQ(frame->Id() & 0xFFFF, 3); }); + InjectPacketWith(stream_structure2, keyframe2_descriptor); + InjectPacketWith(stream_structure1, keyframe1_descriptor); + + // Pass in delta frame that uses structure of the 2nd key frame. Late key + // frame shouldn't block it. + DependencyDescriptor deltaframe_descriptor; + deltaframe_descriptor.frame_dependencies = stream_structure2.templates[0]; + deltaframe_descriptor.frame_number = 4; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce( + [&](EncodedFrame* frame) { EXPECT_EQ(frame->Id() & 0xFFFF, 4); }); + InjectPacketWith(stream_structure2, deltaframe_descriptor); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +using RtpVideoStreamReceiverDeathTest = RtpVideoStreamReceiverTest; +TEST_F(RtpVideoStreamReceiverDeathTest, RepeatedSecondarySinkDisallowed) { + MockRtpPacketSink secondary_sink; + + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); + EXPECT_DEATH(rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink), + ""); + + // Test tear-down. + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); +} +#endif + +TEST_F(RtpVideoStreamReceiverTest, TransformFrame) { + auto mock_frame_transformer = + rtc::make_ref_counted>(); + EXPECT_CALL(*mock_frame_transformer, + RegisterTransformedFrameSinkCallback(_, config_.rtp.remote_ssrc)); + auto receiver = std::make_unique( + Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, + rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(), + &mock_nack_sender_, nullptr, &mock_on_complete_frame_callback_, nullptr, + mock_frame_transformer, &field_trials_); + receiver->AddReceiveCodec(kPayloadType, kVideoCodecGeneric, {}, + /*raw_payload=*/false); + + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({'1', '2', '3', '4'}); + rtp_packet.SetSequenceNumber(1); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(*mock_frame_transformer, Transform(_)); + receiver->OnReceivedPayloadData(data, rtp_packet, video_header); + + EXPECT_CALL(*mock_frame_transformer, + UnregisterTransformedFrameSinkCallback(config_.rtp.remote_ssrc)); + receiver = nullptr; +} + +// Test default behavior and when playout delay is overridden by field trial. +const VideoPlayoutDelay kTransmittedPlayoutDelay = {100, 200}; +const VideoPlayoutDelay kForcedPlayoutDelay = {70, 90}; +struct PlayoutDelayOptions { + std::string field_trial; + VideoPlayoutDelay expected_delay; +}; +const PlayoutDelayOptions kDefaultBehavior = { + /*field_trial=*/"", /*expected_delay=*/kTransmittedPlayoutDelay}; +const PlayoutDelayOptions kOverridePlayoutDelay = { + /*field_trial=*/"WebRTC-ForcePlayoutDelay/min_ms:70,max_ms:90/", + /*expected_delay=*/kForcedPlayoutDelay}; + +class RtpVideoStreamReceiverTestPlayoutDelay + : public RtpVideoStreamReceiverTest, + public ::testing::WithParamInterface { + protected: + RtpVideoStreamReceiverTestPlayoutDelay() + : RtpVideoStreamReceiverTest(GetParam().field_trial) {} +}; + +INSTANTIATE_TEST_SUITE_P(PlayoutDelay, + RtpVideoStreamReceiverTestPlayoutDelay, + Values(kDefaultBehavior, kOverridePlayoutDelay)); + +TEST_P(RtpVideoStreamReceiverTestPlayoutDelay, PlayoutDelay) { + rtc::CopyOnWriteBuffer payload_data({'1', '2', '3', '4'}); + RtpHeaderExtensionMap extension_map; + extension_map.Register(1); + RtpPacketToSend packet_to_send(&extension_map); + packet_to_send.SetPayloadType(kPayloadType); + packet_to_send.SetSequenceNumber(1); + + // Set playout delay on outgoing packet. + EXPECT_TRUE(packet_to_send.SetExtension( + kTransmittedPlayoutDelay)); + uint8_t* payload = packet_to_send.AllocatePayload(payload_data.size()); + memcpy(payload, payload_data.data(), payload_data.size()); + + RtpPacketReceived received_packet(&extension_map); + received_packet.Parse(packet_to_send.data(), packet_to_send.size()); + + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(payload_data.data(), + payload_data.size()); + // Expect the playout delay of encoded frame to be the same as the transmitted + // playout delay unless it was overridden by a field trial. + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([expected_playout_delay = + GetParam().expected_delay](EncodedFrame* frame) { + EXPECT_EQ(frame->EncodedImage().playout_delay_, expected_playout_delay); + })); + rtp_video_stream_receiver_->OnReceivedPayloadData( + received_packet.PayloadBuffer(), received_packet, video_header); +} + +} // namespace webrtc