From 5fe9510efb5a737c1b089d5c36173ed9ea923185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20M=C3=B6ller?= Date: Mon, 4 Mar 2019 16:49:25 +0100 Subject: [PATCH] Move ownership of RTPSenderVideo one more level up, to RtpVideoSender The idea is to let the RtpRtcp and RTPSender classes be responsible for media-agnostic RTP transport, and move out the media-specific processing, such as packetization and media-specific headers. Bug: webrtc:7135 Change-Id: Ib0ce45bf06713b3eb6c06acd91c5168856874e4e Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/123187 Reviewed-by: Sebastian Jansson Reviewed-by: Danil Chapovalov Commit-Queue: Niels Moller Cr-Commit-Position: refs/heads/master@{#26954} --- call/BUILD.gn | 1 + call/rtp_video_sender.cc | 249 +++++++++++------- call/rtp_video_sender.h | 24 +- modules/rtp_rtcp/include/rtp_rtcp.h | 40 ++- modules/rtp_rtcp/include/rtp_rtcp_defines.h | 13 + modules/rtp_rtcp/mocks/mock_rtp_rtcp.h | 15 +- modules/rtp_rtcp/source/nack_rtx_unittest.cc | 32 ++- .../rtp_rtcp/source/playout_delay_oracle.h | 7 +- modules/rtp_rtcp/source/rtp_rtcp_impl.cc | 125 ++++----- modules/rtp_rtcp/source/rtp_rtcp_impl.h | 25 +- .../rtp_rtcp/source/rtp_rtcp_impl_unittest.cc | 48 ++-- .../rtp_rtcp/source/rtp_sender_unittest.cc | 36 ++- modules/rtp_rtcp/source/rtp_sender_video.cc | 12 +- modules/rtp_rtcp/source/rtp_sender_video.h | 18 +- .../source/rtp_sender_video_unittest.cc | 2 + 15 files changed, 383 insertions(+), 264 deletions(-) diff --git a/call/BUILD.gn b/call/BUILD.gn index d4c7e37ed2..3a16757cef 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -126,6 +126,7 @@ rtc_source_set("rtp_sender") { "..:webrtc_common", "../api:fec_controller_api", "../api:transport_api", + "../api/transport:field_trial_based_config", "../api/transport:goog_cc", "../api/transport:network_control", "../api/units:data_rate", diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index e23df47a0c..6c22120159 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -16,10 +16,12 @@ #include #include "absl/memory/memory.h" +#include "api/transport/field_trial_based_config.h" #include "call/rtp_transport_controller_send_interface.h" #include "modules/pacing/packet_router.h" #include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/playout_delay_oracle.h" #include "modules/rtp_rtcp/source/rtp_sender.h" #include "modules/utility/include/process_thread.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -30,6 +32,20 @@ namespace webrtc { +namespace webrtc_internal_rtp_video_sender { + +RtpStreamSender::RtpStreamSender( + std::unique_ptr playout_delay_oracle, + std::unique_ptr rtp_rtcp, + std::unique_ptr sender_video) + : playout_delay_oracle(std::move(playout_delay_oracle)), + rtp_rtcp(std::move(rtp_rtcp)), + sender_video(std::move(sender_video)) {} + +RtpStreamSender::~RtpStreamSender() = default; + +} // namespace webrtc_internal_rtp_video_sender + namespace { static const int kMinSendSidePacketHistorySize = 600; // Assume an average video stream has around 3 packets per frame (1 mbps / 30 @@ -39,7 +55,9 @@ static const int kSendSideSeqNumSetMaxSize = 5500; // We don't do MTU discovery, so assume that we have the standard ethernet MTU. static const size_t kPathMTU = 1500; -std::vector> CreateRtpRtcpModules( +using webrtc_internal_rtp_video_sender::RtpStreamSender; + +std::vector CreateRtpStreamSenders( const RtpConfig& rtp_config, int rtcp_report_interval_ms, Transport* send_transport, @@ -62,6 +80,7 @@ std::vector> CreateRtpRtcpModules( RtpRtcp::Configuration configuration; configuration.audio = false; + configuration.clock = Clock::GetRealTimeClock(); configuration.receiver_only = false; configuration.outgoing_transport = send_transport; configuration.intra_frame_callback = intra_frame_callback; @@ -86,7 +105,7 @@ std::vector> CreateRtpRtcpModules( configuration.extmap_allow_mixed = rtp_config.extmap_allow_mixed; configuration.rtcp_report_interval_ms = rtcp_report_interval_ms; - std::vector> modules; + std::vector rtp_streams; const std::vector& flexfec_protected_ssrcs = rtp_config.flexfec.protected_media_ssrcs; for (uint32_t ssrc : rtp_config.ssrcs) { @@ -95,14 +114,23 @@ std::vector> CreateRtpRtcpModules( flexfec_protected_ssrcs.end(), ssrc) != flexfec_protected_ssrcs.end(); configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr; - std::unique_ptr rtp_rtcp = - std::unique_ptr(RtpRtcp::CreateRtpRtcp(configuration)); + auto playout_delay_oracle = absl::make_unique(); + + configuration.ack_observer = playout_delay_oracle.get(); + auto rtp_rtcp = absl::WrapUnique(RtpRtcp::CreateRtpRtcp(configuration)); rtp_rtcp->SetSendingStatus(false); rtp_rtcp->SetSendingMediaStatus(false); rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); - modules.push_back(std::move(rtp_rtcp)); + + auto sender_video = absl::make_unique( + configuration.clock, rtp_rtcp->RtpSender(), flexfec_sender, + playout_delay_oracle.get(), frame_encryptor, + crypto_options.sframe.require_frame_encryption, + FieldTrialBasedConfig()); + rtp_streams.emplace_back(std::move(playout_delay_oracle), + std::move(rtp_rtcp), std::move(sender_video)); } - return modules; + return rtp_streams; } bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) { @@ -198,25 +226,25 @@ RtpVideoSender::RtpVideoSender( suspended_ssrcs_(std::move(suspended_ssrcs)), flexfec_sender_(MaybeCreateFlexfecSender(rtp_config, suspended_ssrcs_)), fec_controller_(std::move(fec_controller)), - rtp_modules_(CreateRtpRtcpModules(rtp_config, - rtcp_report_interval_ms, - send_transport, - observers.intra_frame_callback, - transport->GetBandwidthObserver(), - transport, - observers.rtcp_rtt_stats, - flexfec_sender_.get(), - observers.bitrate_observer, - observers.rtcp_type_observer, - observers.send_delay_observer, - observers.send_packet_observer, - event_log, - retransmission_limiter, - this, - // TODO(srte): Remove this argument. - RtpKeepAliveConfig(), - frame_encryptor, - crypto_options)), + rtp_streams_(CreateRtpStreamSenders(rtp_config, + rtcp_report_interval_ms, + send_transport, + observers.intra_frame_callback, + transport->GetBandwidthObserver(), + transport, + observers.rtcp_rtt_stats, + flexfec_sender_.get(), + observers.bitrate_observer, + observers.rtcp_type_observer, + observers.send_delay_observer, + observers.send_packet_observer, + event_log, + retransmission_limiter, + this, + // TODO(srte): Remove this argument. + RtpKeepAliveConfig(), + frame_encryptor, + crypto_options)), rtp_config_(rtp_config), transport_(transport), transport_overhead_bytes_per_packet_(0), @@ -224,7 +252,7 @@ RtpVideoSender::RtpVideoSender( encoder_target_rate_bps_(0), frame_counts_(rtp_config.ssrcs.size()), frame_count_observer_(observers.frame_count_observer) { - RTC_DCHECK_EQ(rtp_config.ssrcs.size(), rtp_modules_.size()); + RTC_DCHECK_EQ(rtp_config.ssrcs.size(), rtp_streams_.size()); module_process_thread_checker_.DetachFromThread(); // SSRCs are assumed to be sorted in the same order as |rtp_modules|. for (uint32_t ssrc : rtp_config.ssrcs) { @@ -246,9 +274,9 @@ RtpVideoSender::RtpVideoSender( // TODO(nisse): Consider moving registration with PacketRouter last, after the // modules are fully configured. - for (auto& rtp_rtcp : rtp_modules_) { + for (const RtpStreamSender& stream : rtp_streams_) { constexpr bool remb_candidate = true; - transport->packet_router()->AddSendRtpModule(rtp_rtcp.get(), + transport->packet_router()->AddSendRtpModule(stream.rtp_rtcp.get(), remb_candidate); } @@ -256,8 +284,8 @@ RtpVideoSender::RtpVideoSender( const std::string& extension = rtp_config_.extensions[i].uri; int id = rtp_config_.extensions[i].id; RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension)); - for (auto& rtp_rtcp : rtp_modules_) { - RTC_CHECK(rtp_rtcp->RegisterRtpHeaderExtension(extension, id)); + for (const RtpStreamSender& stream : rtp_streams_) { + RTC_CHECK(stream.rtp_rtcp->RegisterRtpHeaderExtension(extension, id)); } } @@ -266,20 +294,23 @@ RtpVideoSender::RtpVideoSender( ConfigureRids(rtp_config); if (!rtp_config.mid.empty()) { - for (auto& rtp_rtcp : rtp_modules_) { - rtp_rtcp->SetMid(rtp_config.mid); + for (const RtpStreamSender& stream : rtp_streams_) { + stream.rtp_rtcp->SetMid(rtp_config.mid); } } // TODO(pbos): Should we set CNAME on all RTP modules? - rtp_modules_.front()->SetCNAME(rtp_config.c_name.c_str()); + rtp_streams_.front().rtp_rtcp->SetCNAME(rtp_config.c_name.c_str()); - for (auto& rtp_rtcp : rtp_modules_) { - rtp_rtcp->RegisterRtcpStatisticsCallback(observers.rtcp_stats); - rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(observers.rtp_stats); - rtp_rtcp->SetMaxRtpPacketSize(rtp_config.max_packet_size); - rtp_rtcp->RegisterVideoSendPayload(rtp_config.payload_type, - rtp_config.payload_name.c_str()); + for (const RtpStreamSender& stream : rtp_streams_) { + stream.rtp_rtcp->RegisterRtcpStatisticsCallback(observers.rtcp_stats); + stream.rtp_rtcp->RegisterSendChannelRtpStatisticsCallback( + observers.rtp_stats); + stream.rtp_rtcp->SetMaxRtpPacketSize(rtp_config.max_packet_size); + stream.rtp_rtcp->RegisterSendPayloadFrequency(rtp_config.payload_type, + kVideoPayloadTypeFrequency); + stream.sender_video->RegisterPayloadType(rtp_config.payload_type, + rtp_config.payload_name); } // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, // so enable that logic if either of those FEC schemes are enabled. @@ -293,8 +324,8 @@ RtpVideoSender::RtpVideoSender( } RtpVideoSender::~RtpVideoSender() { - for (auto& rtp_rtcp : rtp_modules_) { - transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp.get()); + for (const RtpStreamSender& stream : rtp_streams_) { + transport_->packet_router()->RemoveSendRtpModule(stream.rtp_rtcp.get()); } if (fec_controller_->UseLossVectorMask()) { transport_->DeRegisterPacketFeedbackObserver(this); @@ -307,42 +338,44 @@ void RtpVideoSender::RegisterProcessThread( RTC_DCHECK(!module_process_thread_); module_process_thread_ = module_process_thread; - for (auto& rtp_rtcp : rtp_modules_) - module_process_thread_->RegisterModule(rtp_rtcp.get(), RTC_FROM_HERE); + for (const RtpStreamSender& stream : rtp_streams_) { + module_process_thread_->RegisterModule(stream.rtp_rtcp.get(), + RTC_FROM_HERE); + } } void RtpVideoSender::DeRegisterProcessThread() { RTC_DCHECK_RUN_ON(&module_process_thread_checker_); - for (auto& rtp_rtcp : rtp_modules_) - module_process_thread_->DeRegisterModule(rtp_rtcp.get()); + for (const RtpStreamSender& stream : rtp_streams_) + module_process_thread_->DeRegisterModule(stream.rtp_rtcp.get()); } void RtpVideoSender::SetActive(bool active) { rtc::CritScope lock(&crit_); if (active_ == active) return; - const std::vector active_modules(rtp_modules_.size(), active); + const std::vector active_modules(rtp_streams_.size(), active); SetActiveModules(active_modules); } void RtpVideoSender::SetActiveModules(const std::vector active_modules) { rtc::CritScope lock(&crit_); - RTC_DCHECK_EQ(rtp_modules_.size(), active_modules.size()); + RTC_DCHECK_EQ(rtp_streams_.size(), active_modules.size()); active_ = false; for (size_t i = 0; i < active_modules.size(); ++i) { if (active_modules[i]) { active_ = true; } // Sends a kRtcpByeCode when going from true to false. - rtp_modules_[i]->SetSendingStatus(active_modules[i]); + rtp_streams_[i].rtp_rtcp->SetSendingStatus(active_modules[i]); // If set to false this module won't send media. - rtp_modules_[i]->SetSendingMediaStatus(active_modules[i]); + rtp_streams_[i].rtp_rtcp->SetSendingMediaStatus(active_modules[i]); } } bool RtpVideoSender::IsActive() { rtc::CritScope lock(&crit_); - return active_ && !rtp_modules_.empty(); + return active_ && !rtp_streams_.empty(); } EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( @@ -352,7 +385,7 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( fec_controller_->UpdateWithEncodedData(encoded_image.size(), encoded_image._frameType); rtc::CritScope lock(&crit_); - RTC_DCHECK(!rtp_modules_.empty()); + RTC_DCHECK(!rtp_streams_.empty()); if (!active_) return Result(Result::ERROR_SEND_FAILED); @@ -365,21 +398,33 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( // Map spatial index to simulcast. stream_index = encoded_image.SpatialIndex().value_or(0); } - RTC_DCHECK_LT(stream_index, rtp_modules_.size()); + RTC_DCHECK_LT(stream_index, rtp_streams_.size()); RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader( encoded_image, codec_specific_info, shared_frame_id_); - uint32_t frame_id; - if (!rtp_modules_[stream_index]->Sending()) { + uint32_t rtp_timestamp = + encoded_image.Timestamp() + + rtp_streams_[stream_index].rtp_rtcp->StartTimestamp(); + + // RTCPSender has it's own copy of the timestamp offset, added in + // RTCPSender::BuildSR, hence we must not add the in the offset for this call. + // TODO(nisse): Delete RTCPSender:timestamp_offset_, and see if we can confine + // knowledge of the offset to a single place. + if (!rtp_streams_[stream_index].rtp_rtcp->OnSendingRtpFrame( + encoded_image.Timestamp(), encoded_image.capture_time_ms_, + rtp_config_.payload_type, + encoded_image._frameType == kVideoFrameKey)) { // The payload router could be active but this module isn't sending. return Result(Result::ERROR_SEND_FAILED); } - bool send_result = rtp_modules_[stream_index]->SendOutgoingData( - encoded_image._frameType, rtp_config_.payload_type, - encoded_image.Timestamp(), encoded_image.capture_time_ms_, - encoded_image.data(), encoded_image.size(), fragmentation, - &rtp_video_header, &frame_id); + int64_t expected_retransmission_time_ms = + rtp_streams_[stream_index].rtp_rtcp->ExpectedRetransmissionTimeMs(); + bool send_result = rtp_streams_[stream_index].sender_video->SendVideo( + encoded_image._frameType, rtp_config_.payload_type, rtp_timestamp, + encoded_image.capture_time_ms_, encoded_image.data(), + encoded_image.size(), fragmentation, &rtp_video_header, + expected_retransmission_time_ms); if (frame_count_observer_) { FrameCounts& counts = frame_counts_[stream_index]; if (encoded_image._frameType == kVideoFrameKey) { @@ -395,29 +440,31 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( if (!send_result) return Result(Result::ERROR_SEND_FAILED); - return Result(Result::OK, frame_id); + return Result(Result::OK, rtp_timestamp); } void RtpVideoSender::OnBitrateAllocationUpdated( const VideoBitrateAllocation& bitrate) { rtc::CritScope lock(&crit_); if (IsActive()) { - if (rtp_modules_.size() == 1) { + if (rtp_streams_.size() == 1) { // If spatial scalability is enabled, it is covered by a single stream. - rtp_modules_[0]->SetVideoBitrateAllocation(bitrate); + rtp_streams_[0].rtp_rtcp->SetVideoBitrateAllocation(bitrate); } else { std::vector> layer_bitrates = bitrate.GetSimulcastAllocations(); // Simulcast is in use, split the VideoBitrateAllocation into one struct // per rtp stream, moving over the temporal layer allocation. - for (size_t i = 0; i < rtp_modules_.size(); ++i) { + for (size_t i = 0; i < rtp_streams_.size(); ++i) { // The next spatial layer could be used if the current one is // inactive. if (layer_bitrates[i]) { - rtp_modules_[i]->SetVideoBitrateAllocation(*layer_bitrates[i]); + rtp_streams_[i].rtp_rtcp->SetVideoBitrateAllocation( + *layer_bitrates[i]); } else { // Signal a 0 bitrate on a simulcast stream. - rtp_modules_[i]->SetVideoBitrateAllocation(VideoBitrateAllocation()); + rtp_streams_[i].rtp_rtcp->SetVideoBitrateAllocation( + VideoBitrateAllocation()); } } } @@ -475,11 +522,11 @@ void RtpVideoSender::ConfigureProtection(const RtpConfig& rtp_config) { DisableRedAndUlpfec(); } - for (auto& rtp_rtcp : rtp_modules_) { + for (const RtpStreamSender& stream : rtp_streams_) { // Set NACK. - rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize); + stream.rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize); // Set RED/ULPFEC information. - rtp_rtcp->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); + stream.sender_video->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); } } @@ -498,9 +545,10 @@ bool RtpVideoSender::NackEnabled() const { uint32_t RtpVideoSender::GetPacketizationOverheadRate() const { uint32_t packetization_overhead_bps = 0; - for (auto& rtp_rtcp : rtp_modules_) { - if (rtp_rtcp->SendingMedia()) { - packetization_overhead_bps += rtp_rtcp->PacketizationOverheadBps(); + for (size_t i = 0; i < rtp_streams_.size(); ++i) { + if (rtp_streams_[i].rtp_rtcp->SendingMedia()) { + packetization_overhead_bps += + rtp_streams_[i].sender_video->PacketizationOverheadBps(); } } return packetization_overhead_bps; @@ -508,15 +556,15 @@ uint32_t RtpVideoSender::GetPacketizationOverheadRate() const { void RtpVideoSender::DeliverRtcp(const uint8_t* packet, size_t length) { // Runs on a network thread. - for (auto& rtp_rtcp : rtp_modules_) - rtp_rtcp->IncomingRtcpPacket(packet, length); + for (const RtpStreamSender& stream : rtp_streams_) + stream.rtp_rtcp->IncomingRtcpPacket(packet, length); } void RtpVideoSender::ConfigureSsrcs(const RtpConfig& rtp_config) { // Configure regular SSRCs. for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) { uint32_t ssrc = rtp_config.ssrcs[i]; - RtpRtcp* const rtp_rtcp = rtp_modules_[i].get(); + RtpRtcp* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get(); rtp_rtcp->SetSSRC(ssrc); // Restore RTP state if previous existed. @@ -533,7 +581,7 @@ void RtpVideoSender::ConfigureSsrcs(const RtpConfig& rtp_config) { RTC_DCHECK_EQ(rtp_config.rtx.ssrcs.size(), rtp_config.ssrcs.size()); for (size_t i = 0; i < rtp_config.rtx.ssrcs.size(); ++i) { uint32_t ssrc = rtp_config.rtx.ssrcs[i]; - RtpRtcp* const rtp_rtcp = rtp_modules_[i].get(); + RtpRtcp* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get(); rtp_rtcp->SetRtxSsrc(ssrc); auto it = suspended_ssrcs_.find(ssrc); if (it != suspended_ssrcs_.end()) @@ -542,16 +590,18 @@ void RtpVideoSender::ConfigureSsrcs(const RtpConfig& rtp_config) { // Configure RTX payload types. RTC_DCHECK_GE(rtp_config.rtx.payload_type, 0); - for (auto& rtp_rtcp : rtp_modules_) { - rtp_rtcp->SetRtxSendPayloadType(rtp_config.rtx.payload_type, - rtp_config.payload_type); - rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads); + for (const RtpStreamSender& stream : rtp_streams_) { + stream.rtp_rtcp->SetRtxSendPayloadType(rtp_config.rtx.payload_type, + rtp_config.payload_type); + stream.rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | + kRtxRedundantPayloads); } if (rtp_config.ulpfec.red_payload_type != -1 && rtp_config.ulpfec.red_rtx_payload_type != -1) { - for (auto& rtp_rtcp : rtp_modules_) { - rtp_rtcp->SetRtxSendPayloadType(rtp_config.ulpfec.red_rtx_payload_type, - rtp_config.ulpfec.red_payload_type); + for (const RtpStreamSender& stream : rtp_streams_) { + stream.rtp_rtcp->SetRtxSendPayloadType( + rtp_config.ulpfec.red_rtx_payload_type, + rtp_config.ulpfec.red_payload_type); } } } @@ -560,18 +610,17 @@ void RtpVideoSender::ConfigureRids(const RtpConfig& rtp_config) { RTC_DCHECK(rtp_config.rids.empty() || rtp_config.rids.size() == rtp_config.ssrcs.size()); RTC_DCHECK(rtp_config.rids.empty() || - rtp_config.rids.size() == rtp_modules_.size()); + rtp_config.rids.size() == rtp_streams_.size()); for (size_t i = 0; i < rtp_config.rids.size(); ++i) { const std::string& rid = rtp_config.rids[i]; - RtpRtcp* const rtp_rtcp = rtp_modules_[i].get(); - rtp_rtcp->SetRid(rid); + rtp_streams_[i].rtp_rtcp->SetRid(rid); } } void RtpVideoSender::OnNetworkAvailability(bool network_available) { - for (auto& rtp_rtcp : rtp_modules_) { - rtp_rtcp->SetRTCPStatus(network_available ? rtp_config_.rtcp_mode - : RtcpMode::kOff); + for (const RtpStreamSender& stream : rtp_streams_) { + stream.rtp_rtcp->SetRTCPStatus(network_available ? rtp_config_.rtcp_mode + : RtcpMode::kOff); } } @@ -580,13 +629,13 @@ std::map RtpVideoSender::GetRtpStates() const { for (size_t i = 0; i < rtp_config_.ssrcs.size(); ++i) { uint32_t ssrc = rtp_config_.ssrcs[i]; - RTC_DCHECK_EQ(ssrc, rtp_modules_[i]->SSRC()); - rtp_states[ssrc] = rtp_modules_[i]->GetRtpState(); + RTC_DCHECK_EQ(ssrc, rtp_streams_[i].rtp_rtcp->SSRC()); + rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtpState(); } for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) { uint32_t ssrc = rtp_config_.rtx.ssrcs[i]; - rtp_states[ssrc] = rtp_modules_[i]->GetRtxState(); + rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtxState(); } if (flexfec_sender_) { @@ -616,8 +665,8 @@ void RtpVideoSender::OnTransportOverheadChanged( size_t max_rtp_packet_size = std::min(rtp_config_.max_packet_size, kPathMTU - transport_overhead_bytes_per_packet_); - for (auto& rtp_rtcp : rtp_modules_) { - rtp_rtcp->SetMaxRtpPacketSize(max_rtp_packet_size); + for (const RtpStreamSender& stream : rtp_streams_) { + stream.rtp_rtcp->SetMaxRtpPacketSize(max_rtp_packet_size); } } @@ -699,17 +748,15 @@ int RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params, *sent_video_rate_bps = 0; *sent_nack_rate_bps = 0; *sent_fec_rate_bps = 0; - for (auto& rtp_rtcp : rtp_modules_) { + for (const RtpStreamSender& stream : rtp_streams_) { uint32_t not_used = 0; - uint32_t module_video_rate = 0; - uint32_t module_fec_rate = 0; uint32_t module_nack_rate = 0; - rtp_rtcp->SetFecParameters(*delta_params, *key_params); - rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate, - &module_nack_rate); - *sent_video_rate_bps += module_video_rate; + stream.sender_video->SetFecParameters(*delta_params, *key_params); + *sent_video_rate_bps += stream.sender_video->VideoBitrateSent(); + *sent_fec_rate_bps += stream.sender_video->FecOverheadRate(); + stream.rtp_rtcp->BitrateSent(¬_used, /*video_rate=*/nullptr, + /*fec_rate=*/nullptr, &module_nack_rate); *sent_nack_rate_bps += module_nack_rate; - *sent_fec_rate_bps += module_fec_rate; } return 0; } diff --git a/call/rtp_video_sender.h b/call/rtp_video_sender.h index fc0272bee6..69187a9439 100644 --- a/call/rtp_video_sender.h +++ b/call/rtp_video_sender.h @@ -25,6 +25,7 @@ #include "call/rtp_video_sender_interface.h" #include "logging/rtc_event_log/rtc_event_log.h" #include "modules/rtp_rtcp/include/flexfec_sender.h" +#include "modules/rtp_rtcp/source/rtp_sender_video.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" #include "modules/utility/include/process_thread.h" #include "rtc_base/constructor_magic.h" @@ -40,6 +41,26 @@ class RTPFragmentationHeader; class RtpRtcp; class RtpTransportControllerSendInterface; +namespace webrtc_internal_rtp_video_sender { +// RTP state for a single simulcast stream. Internal to the implementation of +// RtpVideoSender. +struct RtpStreamSender { + RtpStreamSender(std::unique_ptr playout_delay_oracle, + std::unique_ptr rtp_rtcp, + std::unique_ptr sender_video); + ~RtpStreamSender(); + + RtpStreamSender(RtpStreamSender&&) = default; + RtpStreamSender& operator=(RtpStreamSender&&) = default; + + // Note: Needs pointer stability. + std::unique_ptr playout_delay_oracle; + std::unique_ptr rtp_rtcp; + std::unique_ptr sender_video; +}; + +} // namespace webrtc_internal_rtp_video_sender + // RtpVideoSender routes outgoing data to the correct sending RTP module, based // on the simulcast layer in RTPVideoHeader. class RtpVideoSender : public RtpVideoSenderInterface, @@ -145,7 +166,8 @@ class RtpVideoSender : public RtpVideoSenderInterface, std::unique_ptr flexfec_sender_; std::unique_ptr fec_controller_; // Rtp modules are assumed to be sorted in simulcast index order. - const std::vector> rtp_modules_; + const std::vector + rtp_streams_; const RtpConfig rtp_config_; RtpTransportControllerSendInterface* const transport_; diff --git a/modules/rtp_rtcp/include/rtp_rtcp.h b/modules/rtp_rtcp/include/rtp_rtcp.h index 6c201d1824..6e1b4a5e62 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp.h +++ b/modules/rtp_rtcp/include/rtp_rtcp.h @@ -24,6 +24,7 @@ #include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtp_sender.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/deprecation.h" @@ -93,6 +94,8 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { SendPacketObserver* send_packet_observer = nullptr; RateLimiter* retransmission_rate_limiter = nullptr; OverheadObserver* overhead_observer = nullptr; + RtcpAckObserver* ack_observer = nullptr; + RtpKeepAliveConfig keepalive_config; int rtcp_report_interval_ms = 0; @@ -145,8 +148,8 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { int frequency, int channels, int rate) = 0; - virtual void RegisterVideoSendPayload(int payload_type, - const char* payload_name) = 0; + virtual void RegisterSendPayloadFrequency(int payload_type, + int payload_frequency) = 0; // Unregisters a send payload. // |payload_type| - payload type of codec @@ -248,10 +251,8 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { uint32_t* fec_rate, uint32_t* nack_rate) const = 0; - // Returns the current packetization overhead rate, in bps. Note that this is - // the payload overhead, eg the VP8 payload headers, not the RTP headers - // or extension/ - virtual uint32_t PacketizationOverheadBps() const = 0; + virtual RTPSender* RtpSender() = 0; + virtual const RTPSender* RtpSender() const = 0; // Used by the codec module to deliver a video or audio frame for // packetization. @@ -274,6 +275,13 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { const RTPVideoHeader* rtp_video_header, uint32_t* transport_frame_id_out) = 0; + // Record that a frame is about to be sent. Returns true on success, and false + // if the module isn't ready to send. + virtual bool OnSendingRtpFrame(uint32_t timestamp, + int64_t capture_time_ms, + int payload_type, + bool force_sender_report) = 0; + virtual bool TimeToSendPacket(uint32_t ssrc, uint16_t sequence_number, int64_t capture_time_ms, @@ -331,6 +339,9 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { int64_t* min_rtt, int64_t* max_rtt) const = 0; + // Returns the estimated RTT, with fallback to a default value. + virtual int64_t ExpectedRetransmissionTimeMs() const = 0; + // Forces a send of a RTCP packet. Periodic SR and RR are triggered via the // process function. // Returns -1 on failure else 0. @@ -438,23 +449,6 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { // Video // ************************************************************************** - // Set RED and ULPFEC payload types. A payload type of -1 means that the - // corresponding feature is turned off. Note that we DO NOT support enabling - // ULPFEC without enabling RED, and RED is only ever used when ULPFEC is - // enabled. - virtual void SetUlpfecConfig(int red_payload_type, - int ulpfec_payload_type) = 0; - - // Set FEC rates, max frames before FEC is sent, and type of FEC masks. - // Returns false on failure. - virtual bool SetFecParameters(const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) = 0; - - // Deprecated version of member function above. - RTC_DEPRECATED - int32_t SetFecParameters(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params); - // Set method for requestion a new key frame. // Returns -1 on failure else 0. virtual int32_t SetKeyFrameRequestMethod(KeyFrameRequestMethod method) = 0; diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 03dcda3c3c..b6489426d4 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -459,5 +459,18 @@ class StreamDataCountersCallback { uint32_t ssrc) = 0; }; +class RtcpAckObserver { + public: + // This method is called on received report blocks matching the sender ssrc. + // TODO(nisse): Use of "extended" sequence number is a bit brittle, since the + // observer for this callback typically has its own sequence number unwrapper, + // and there's no guarantee that they are in sync. Change to pass raw sequence + // number, possibly augmented with timestamp (if available) to aid + // disambiguation. + virtual void OnReceivedAck(int64_t extended_highest_sequence_number) = 0; + + virtual ~RtcpAckObserver() = default; +}; + } // namespace webrtc #endif // MODULES_RTP_RTCP_INCLUDE_RTP_RTCP_DEFINES_H_ diff --git a/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h b/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h index fc94649d0b..7d4f0f1d3c 100644 --- a/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h +++ b/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h @@ -43,8 +43,8 @@ class MockRtpRtcp : public RtpRtcp { int frequency, int channels, int rate)); - MOCK_METHOD2(RegisterVideoSendPayload, - void(int payload_type, const char* payload_name)); + MOCK_METHOD2(RegisterSendPayloadFrequency, + void(int payload_type, int frequency)); MOCK_METHOD1(DeRegisterSendPayload, int32_t(int8_t payload_type)); MOCK_METHOD1(SetExtmapAllowMixed, void(bool extmap_allow_mixed)); MOCK_METHOD2(RegisterSendRtpHeaderExtension, @@ -85,7 +85,6 @@ class MockRtpRtcp : public RtpRtcp { uint32_t* video_rate, uint32_t* fec_rate, uint32_t* nack_rate)); - MOCK_CONST_METHOD0(PacketizationOverheadBps, uint32_t()); MOCK_CONST_METHOD1(EstimatedReceiveBandwidth, int(uint32_t* available_bandwidth)); MOCK_METHOD9(SendOutgoingData, @@ -98,6 +97,7 @@ class MockRtpRtcp : public RtpRtcp { const RTPFragmentationHeader* fragmentation, const RTPVideoHeader* rtp_video_header, uint32_t* frame_id_out)); + MOCK_METHOD4(OnSendingRtpFrame, bool(uint32_t, int64_t, int, bool)); MOCK_METHOD5(TimeToSendPacket, bool(uint32_t ssrc, uint16_t sequence_number, @@ -130,6 +130,7 @@ class MockRtpRtcp : public RtpRtcp { int64_t* avg_rtt, int64_t* min_rtt, int64_t* max_rtt)); + MOCK_CONST_METHOD0(ExpectedRetransmissionTimeMs, int64_t()); MOCK_METHOD1(SendRTCP, int32_t(RTCPPacketType packet_type)); MOCK_METHOD1(SendCompoundRTCP, int32_t(const std::set& packet_types)); @@ -165,11 +166,6 @@ class MockRtpRtcp : public RtpRtcp { int32_t(uint8_t key, uint16_t time_ms, uint8_t level)); MOCK_METHOD1(SetAudioLevel, int32_t(uint8_t level_dbov)); MOCK_METHOD1(SetTargetSendBitrate, void(uint32_t bitrate_bps)); - MOCK_METHOD2(SetUlpfecConfig, - void(int red_payload_type, int fec_payload_type)); - MOCK_METHOD2(SetFecParameters, - bool(const FecProtectionParams& delta_params, - const FecProtectionParams& key_params)); MOCK_METHOD1(SetKeyFrameRequestMethod, int32_t(KeyFrameRequestMethod method)); MOCK_METHOD0(RequestKeyFrame, int32_t()); MOCK_METHOD3(SendLossNotification, @@ -182,6 +178,9 @@ class MockRtpRtcp : public RtpRtcp { MOCK_CONST_METHOD0(GetSendChannelRtpStatisticsCallback, StreamDataCountersCallback*(void)); MOCK_METHOD1(SetVideoBitrateAllocation, void(const VideoBitrateAllocation&)); + MOCK_METHOD0(RtpSender, RTPSender*()); + MOCK_CONST_METHOD0(RtpSender, const RTPSender*()); + // Members. unsigned int remote_ssrc_; diff --git a/modules/rtp_rtcp/source/nack_rtx_unittest.cc b/modules/rtp_rtcp/source/nack_rtx_unittest.cc index 7ecec76474..864befe979 100644 --- a/modules/rtp_rtcp/source/nack_rtx_unittest.cc +++ b/modules/rtp_rtcp/source/nack_rtx_unittest.cc @@ -14,14 +14,18 @@ #include #include +#include "absl/memory/memory.h" #include "api/call/transport.h" +#include "api/transport/field_trial_based_config.h" #include "call/rtp_stream_receiver_controller.h" #include "call/rtx_receive_stream.h" #include "common_types.h" // NOLINT(build/include) #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/playout_delay_oracle.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_sender_video.h" #include "rtc_base/rate_limiter.h" #include "test/gtest.h" @@ -132,8 +136,10 @@ class RtpRtcpRtxNackTest : public ::testing::Test { configuration.receive_statistics = receive_statistics_.get(); configuration.outgoing_transport = &transport_; configuration.retransmission_rate_limiter = &retransmission_rate_limiter_; - rtp_rtcp_module_ = RtpRtcp::CreateRtpRtcp(configuration); - + rtp_rtcp_module_ = absl::WrapUnique(RtpRtcp::CreateRtpRtcp(configuration)); + rtp_sender_video_ = absl::make_unique( + &fake_clock, rtp_rtcp_module_->RtpSender(), nullptr, + &playout_delay_oracle_, nullptr, false, FieldTrialBasedConfig()); rtp_rtcp_module_->SetSSRC(kTestSsrc); rtp_rtcp_module_->SetRTCPStatus(RtcpMode::kCompound); rtp_rtcp_module_->SetStorePacketsStatus(true, 600); @@ -146,9 +152,9 @@ class RtpRtcpRtxNackTest : public ::testing::Test { // single rtp_rtcp module for both send and receive side. rtp_rtcp_module_->SetRemoteSSRC(kTestSsrc); - rtp_rtcp_module_->RegisterVideoSendPayload(kPayloadType, "video"); + rtp_sender_video_->RegisterPayloadType(kPayloadType, "video"); rtp_rtcp_module_->SetRtxSendPayloadType(kRtxPayloadType, kPayloadType); - transport_.SetSendModule(rtp_rtcp_module_); + transport_.SetSendModule(rtp_rtcp_module_.get()); media_receiver_ = transport_.stream_receiver_controller_.CreateReceiver( kTestSsrc, &media_stream_); @@ -203,9 +209,11 @@ class RtpRtcpRtxNackTest : public ::testing::Test { uint16_t nack_list[kVideoNackListSize]; for (int frame = 0; frame < kNumFrames; ++frame) { RTPVideoHeader video_header; - EXPECT_TRUE(rtp_rtcp_module_->SendOutgoingData( + EXPECT_TRUE(rtp_rtcp_module_->OnSendingRtpFrame(timestamp, timestamp / 90, + kPayloadType, false)); + EXPECT_TRUE(rtp_sender_video_->SendVideo( webrtc::kVideoFrameDelta, kPayloadType, timestamp, timestamp / 90, - payload_data, payload_data_length, nullptr, &video_header, nullptr)); + payload_data, payload_data_length, nullptr, &video_header, 0)); // Min required delay until retransmit = 5 + RTT ms (RTT = 0). fake_clock.AdvanceTimeMilliseconds(5); int length = BuildNackList(nack_list); @@ -219,10 +227,10 @@ class RtpRtcpRtxNackTest : public ::testing::Test { media_stream_.sequence_numbers_.sort(); } - void TearDown() override { delete rtp_rtcp_module_; } - std::unique_ptr receive_statistics_; - RtpRtcp* rtp_rtcp_module_; + std::unique_ptr rtp_rtcp_module_; + PlayoutDelayOracle playout_delay_oracle_; + std::unique_ptr rtp_sender_video_; RtxLoopBackTransport transport_; const std::map rtx_associated_payload_types_ = { {kRtxPayloadType, kPayloadType}}; @@ -252,9 +260,11 @@ TEST_F(RtpRtcpRtxNackTest, LongNackList) { // enough packets. for (int frame = 0; frame < kNumFrames; ++frame) { RTPVideoHeader video_header; - EXPECT_TRUE(rtp_rtcp_module_->SendOutgoingData( + EXPECT_TRUE(rtp_rtcp_module_->OnSendingRtpFrame(timestamp, timestamp / 90, + kPayloadType, false)); + EXPECT_TRUE(rtp_sender_video_->SendVideo( webrtc::kVideoFrameDelta, kPayloadType, timestamp, timestamp / 90, - payload_data, payload_data_length, nullptr, &video_header, nullptr)); + payload_data, payload_data_length, nullptr, &video_header, 0)); // Prepare next frame. timestamp += 3000; fake_clock.AdvanceTimeMilliseconds(33); diff --git a/modules/rtp_rtcp/source/playout_delay_oracle.h b/modules/rtp_rtcp/source/playout_delay_oracle.h index 78f8747879..6451be4cdc 100644 --- a/modules/rtp_rtcp/source/playout_delay_oracle.h +++ b/modules/rtp_rtcp/source/playout_delay_oracle.h @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "common_types.h" // NOLINT(build/include) #include "modules/include/module_common_types_public.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/critical_section.h" #include "rtc_base/thread_annotations.h" @@ -33,10 +34,10 @@ namespace webrtc { // The application specifies a minimum and maximum limit for the playout delay // which are both communicated to the receiver and the receiver can adapt // the playout delay within this range based on observed network jitter. -class PlayoutDelayOracle { +class PlayoutDelayOracle : public RtcpAckObserver { public: PlayoutDelayOracle(); - ~PlayoutDelayOracle(); + ~PlayoutDelayOracle() override; // The playout delay to be added to a packet. The input delays are provided by // the application, with -1 meaning unchanged/unspecified. The output delay @@ -49,7 +50,7 @@ class PlayoutDelayOracle { void OnSentPacket(uint16_t sequence_number, absl::optional playout_delay); - void OnReceivedAck(int64_t extended_highest_sequence_number); + void OnReceivedAck(int64_t extended_highest_sequence_number) override; private: // The playout delay information is updated from the encoder thread(s). diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 20f58c2c3f..2adddff0bf 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -53,14 +53,6 @@ RtpRtcp* RtpRtcp::CreateRtpRtcp(const RtpRtcp::Configuration& configuration) { } } -// Deprecated. -int32_t RtpRtcp::SetFecParameters(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params) { - RTC_DCHECK(delta_params); - RTC_DCHECK(key_params); - return SetFecParameters(*delta_params, *key_params) ? 0 : -1; -} - ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) : rtcp_sender_(configuration.audio, configuration.clock, @@ -96,6 +88,7 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) nack_last_seq_number_sent_(0), key_frame_req_method_(kKeyFrameReqPliRtcp), remote_bitrate_(configuration.remote_bitrate_estimator), + ack_observer_(configuration.ack_observer), rtt_stats_(configuration.rtt_stats), rtt_ms_(0) { FieldTrialBasedConfig default_trials; @@ -120,12 +113,6 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) : default_trials)); if (configuration.audio) { audio_ = absl::make_unique(clock_, rtp_sender_.get()); - } else { - video_ = absl::make_unique( - clock_, rtp_sender_.get(), configuration.flexfec_sender, - configuration.frame_encryptor, configuration.require_frame_encryption, - configuration.field_trials ? *configuration.field_trials - : default_trials); } // Make sure rtcp sender use same timestamp offset as rtp sender. rtcp_sender_.SetTimestampOffset(rtp_sender_->TimestampOffset()); @@ -289,11 +276,9 @@ void ModuleRtpRtcpImpl::RegisterAudioSendPayload(int payload_type, frequency, channels, rate)); } -void ModuleRtpRtcpImpl::RegisterVideoSendPayload(int payload_type, - const char* payload_name) { - RTC_DCHECK(video_); - rtcp_sender_.SetRtpClockRate(payload_type, kVideoPayloadTypeFrequency); - video_->RegisterPayloadType(payload_type, payload_name); +void ModuleRtpRtcpImpl::RegisterSendPayloadFrequency(int payload_type, + int payload_frequency) { + rtcp_sender_.SetRtpClockRate(payload_type, payload_frequency); } int32_t ModuleRtpRtcpImpl::DeRegisterSendPayload(const int8_t payload_type) { @@ -445,37 +430,33 @@ bool ModuleRtpRtcpImpl::SendOutgoingData( const RTPFragmentationHeader* fragmentation, const RTPVideoHeader* rtp_video_header, uint32_t* transport_frame_id_out) { - rtcp_sender_.SetLastRtpTime(time_stamp, capture_time_ms, payload_type); - // Make sure an RTCP report isn't queued behind a key frame. - if (rtcp_sender_.TimeToSendRTCPReport(kVideoFrameKey == frame_type)) { - rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport); - } - int64_t expected_retransmission_time_ms = rtt_ms(); - if (expected_retransmission_time_ms == 0) { - // No rtt available (|kRtpRtcpRttProcessTimeMs| not yet passed?), so try to - // poll avg_rtt_ms directly from rtcp receiver. - if (rtcp_receiver_.RTT(rtcp_receiver_.RemoteSSRC(), nullptr, - &expected_retransmission_time_ms, nullptr, - nullptr) == -1) { - expected_retransmission_time_ms = kDefaultExpectedRetransmissionTimeMs; - } - } + OnSendingRtpFrame(time_stamp, capture_time_ms, payload_type, + kVideoFrameKey == frame_type); const uint32_t rtp_timestamp = time_stamp + rtp_sender_->TimestampOffset(); if (transport_frame_id_out) *transport_frame_id_out = rtp_timestamp; - if (audio_) { - RTC_DCHECK(fragmentation == nullptr); + RTC_DCHECK(audio_); + RTC_DCHECK(fragmentation == nullptr); - return audio_->SendAudio(frame_type, payload_type, rtp_timestamp, - payload_data, payload_size); - } else { - return video_->SendVideo(frame_type, payload_type, rtp_timestamp, - capture_time_ms, payload_data, payload_size, - fragmentation, rtp_video_header, - expected_retransmission_time_ms); - } + return audio_->SendAudio(frame_type, payload_type, rtp_timestamp, + payload_data, payload_size); +} + +bool ModuleRtpRtcpImpl::OnSendingRtpFrame(uint32_t timestamp, + int64_t capture_time_ms, + int payload_type, + bool force_sender_report) { + if (!Sending()) + return false; + + rtcp_sender_.SetLastRtpTime(timestamp, capture_time_ms, payload_type); + // Make sure an RTCP report isn't queued behind a key frame. + if (rtcp_sender_.TimeToSendRTCPReport(force_sender_report)) + rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport); + + return true; } bool ModuleRtpRtcpImpl::TimeToSendPacket(uint32_t ssrc, @@ -560,6 +541,21 @@ int32_t ModuleRtpRtcpImpl::RTT(const uint32_t remote_ssrc, return ret; } +int64_t ModuleRtpRtcpImpl::ExpectedRetransmissionTimeMs() const { + int64_t expected_retransmission_time_ms = rtt_ms(); + if (expected_retransmission_time_ms > 0) { + return expected_retransmission_time_ms; + } + // No rtt available (|kRtpRtcpRttProcessTimeMs| not yet passed?), so try to + // poll avg_rtt_ms directly from rtcp receiver. + if (rtcp_receiver_.RTT(rtcp_receiver_.RemoteSSRC(), nullptr, + &expected_retransmission_time_ms, nullptr, + nullptr) == 0) { + return expected_retransmission_time_ms; + } + return kDefaultExpectedRetransmissionTimeMs; +} + // Force a send of an RTCP packet. // Normal SR and RR are triggered via the process function. int32_t ModuleRtpRtcpImpl::SendRTCP(RTCPPacketType packet_type) { @@ -821,42 +817,25 @@ int32_t ModuleRtpRtcpImpl::SendLossNotification(uint16_t last_decoded_seq_num, decodability_flag); } -void ModuleRtpRtcpImpl::SetUlpfecConfig(int red_payload_type, - int ulpfec_payload_type) { - RTC_DCHECK(video_); - video_->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); -} - -bool ModuleRtpRtcpImpl::SetFecParameters( - const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) { - if (!video_) { - return false; - } - video_->SetFecParameters(delta_params, key_params); - return true; -} - void ModuleRtpRtcpImpl::SetRemoteSSRC(const uint32_t ssrc) { // Inform about the incoming SSRC. rtcp_sender_.SetRemoteSSRC(ssrc); rtcp_receiver_.SetRemoteSSRC(ssrc); } +// TODO(nisse): Delete video_rate amd fec_rate arguments. void ModuleRtpRtcpImpl::BitrateSent(uint32_t* total_rate, uint32_t* video_rate, uint32_t* fec_rate, uint32_t* nack_rate) const { *total_rate = rtp_sender_->BitrateSent(); - *video_rate = video_ ? video_->VideoBitrateSent() : 0; - *fec_rate = video_ ? video_->FecOverheadRate() : 0; + if (video_rate) + *video_rate = 0; + if (fec_rate) + *fec_rate = 0; *nack_rate = rtp_sender_->NackOverheadRate(); } -uint32_t ModuleRtpRtcpImpl::PacketizationOverheadBps() const { - return video_ ? video_->PacketizationOverheadBps() : 0; -} - void ModuleRtpRtcpImpl::OnRequestSendReport() { SendRTCP(kRtcpSr); } @@ -882,12 +861,13 @@ void ModuleRtpRtcpImpl::OnReceivedNack( void ModuleRtpRtcpImpl::OnReceivedRtcpReportBlocks( const ReportBlockList& report_blocks) { - if (video_) { + if (ack_observer_) { uint32_t ssrc = SSRC(); for (const RTCPReportBlock& report_block : report_blocks) { if (ssrc == report_block.source_ssrc) { - video_->OnReceivedAck(report_block.extended_highest_sequence_number); + ack_observer_->OnReceivedAck( + report_block.extended_highest_sequence_number); } } } @@ -952,4 +932,13 @@ void ModuleRtpRtcpImpl::SetVideoBitrateAllocation( const VideoBitrateAllocation& bitrate) { rtcp_sender_.SetVideoBitrateAllocation(bitrate); } + +RTPSender* ModuleRtpRtcpImpl::RtpSender() { + return rtp_sender_.get(); +} + +const RTPSender* ModuleRtpRtcpImpl::RtpSender() const { + return rtp_sender_.get(); +} + } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.h b/modules/rtp_rtcp/source/rtp_rtcp_impl.h index b067bd13b0..20eb179a9a 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl.h +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.h @@ -70,8 +70,8 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { int frequency, int channels, int rate) override; - void RegisterVideoSendPayload(int payload_type, - const char* payload_name) override; + void RegisterSendPayloadFrequency(int payload_type, + int payload_frequency) override; int32_t DeRegisterSendPayload(int8_t payload_type) override; @@ -149,6 +149,11 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { const RTPVideoHeader* rtp_video_header, uint32_t* transport_frame_id_out) override; + bool OnSendingRtpFrame(uint32_t timestamp, + int64_t capture_time_ms, + int payload_type, + bool force_sender_report) override; + bool TimeToSendPacket(uint32_t ssrc, uint16_t sequence_number, int64_t capture_time_ms, @@ -193,6 +198,8 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { int64_t* min_rtt, int64_t* max_rtt) const override; + int64_t ExpectedRetransmissionTimeMs() const override; + // Force a send of an RTCP packet. // Normal SR and RR are triggered via the process function. int32_t SendRTCP(RTCPPacketType rtcpPacketType) override; @@ -286,11 +293,6 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { uint16_t last_received_seq_num, bool decodability_flag) override; - void SetUlpfecConfig(int red_payload_type, int ulpfec_payload_type) override; - - bool SetFecParameters(const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) override; - bool LastReceivedNTP(uint32_t* NTPsecs, uint32_t* NTPfrac, uint32_t* remote_sr) const; @@ -301,7 +303,6 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { uint32_t* video_rate, uint32_t* fec_rate, uint32_t* nackRate) const override; - uint32_t PacketizationOverheadBps() const override; void RegisterSendChannelRtpStatisticsCallback( StreamDataCountersCallback* callback) override; @@ -317,6 +318,9 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { void SetVideoBitrateAllocation( const VideoBitrateAllocation& bitrate) override; + RTPSender* RtpSender() override; + const RTPSender* RtpSender() const override; + protected: bool UpdateRTCPReceiveInformationTimers(); @@ -343,7 +347,6 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { std::unique_ptr rtp_sender_; std::unique_ptr audio_; - std::unique_ptr video_; RTCPSender rtcp_sender_; RTCPReceiver rtcp_receiver_; @@ -362,7 +365,9 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { KeyFrameRequestMethod key_frame_req_method_; - RemoteBitrateEstimator* remote_bitrate_; + RemoteBitrateEstimator* const remote_bitrate_; + + RtcpAckObserver* const ack_observer_; RtcpRttStats* const rtt_stats_; diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc index dd531f8555..da541a5098 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc @@ -12,9 +12,12 @@ #include #include +#include "absl/memory/memory.h" +#include "api/transport/field_trial_based_config.h" #include "api/video_codecs/video_codec.h" #include "modules/rtp_rtcp/include/rtp_header_parser.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/playout_delay_oracle.h" #include "modules/rtp_rtcp/source/rtcp_packet.h" #include "modules/rtp_rtcp/source/rtcp_packet/nack.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" @@ -199,11 +202,15 @@ class RtpRtcpImplTest : public ::testing::Test { sender_.impl_->SetSequenceNumber(kSequenceNumber); sender_.impl_->SetStorePacketsStatus(true, 100); + sender_video_ = absl::make_unique( + &clock_, sender_.impl_->RtpSender(), nullptr, &playout_delay_oracle_, + nullptr, false, FieldTrialBasedConfig()); + memset(&codec_, 0, sizeof(VideoCodec)); codec_.plType = 100; codec_.width = 320; codec_.height = 180; - sender_.impl_->RegisterVideoSendPayload(codec_.plType, "VP8"); + sender_video_->RegisterPayloadType(codec_.plType, "VP8"); // Receive module. EXPECT_EQ(0, receiver_.impl_->SetSendingStatus(false)); @@ -217,10 +224,14 @@ class RtpRtcpImplTest : public ::testing::Test { SimulatedClock clock_; RtpRtcpModule sender_; + PlayoutDelayOracle playout_delay_oracle_; + std::unique_ptr sender_video_; RtpRtcpModule receiver_; VideoCodec codec_; - void SendFrame(const RtpRtcpModule* module, uint8_t tid) { + void SendFrame(const RtpRtcpModule* module, + RTPSenderVideo* sender, + uint8_t tid) { RTPVideoHeaderVP8 vp8_header = {}; vp8_header.temporalIdx = tid; RTPVideoHeader rtp_video_header; @@ -236,9 +247,10 @@ class RtpRtcpImplTest : public ::testing::Test { rtp_video_header.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false}; const uint8_t payload[100] = {0}; - EXPECT_EQ(true, module->impl_->SendOutgoingData( - kVideoFrameKey, codec_.plType, 0, 0, payload, - sizeof(payload), nullptr, &rtp_video_header, nullptr)); + EXPECT_TRUE(module->impl_->OnSendingRtpFrame(0, 0, codec_.plType, true)); + EXPECT_TRUE(sender->SendVideo(kVideoFrameKey, codec_.plType, 0, 0, payload, + sizeof(payload), nullptr, &rtp_video_header, + 0)); } void IncomingRtcpNack(const RtpRtcpModule* module, uint16_t sequence_number) { @@ -258,9 +270,11 @@ class RtpRtcpImplTest : public ::testing::Test { TEST_F(RtpRtcpImplTest, RetransmitsAllLayers) { // Send frames. EXPECT_EQ(0, sender_.RtpSent()); - SendFrame(&sender_, kBaseLayerTid); // kSequenceNumber - SendFrame(&sender_, kHigherLayerTid); // kSequenceNumber + 1 - SendFrame(&sender_, kNoTemporalIdx); // kSequenceNumber + 2 + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); // kSequenceNumber + SendFrame(&sender_, sender_video_.get(), + kHigherLayerTid); // kSequenceNumber + 1 + SendFrame(&sender_, sender_video_.get(), + kNoTemporalIdx); // kSequenceNumber + 2 EXPECT_EQ(3, sender_.RtpSent()); EXPECT_EQ(kSequenceNumber + 2, sender_.LastRtpSequenceNumber()); @@ -290,7 +304,7 @@ TEST_F(RtpRtcpImplTest, Rtt) { receiver_.receive_statistics_->OnRtpPacket(packet); // Send Frame before sending an SR. - SendFrame(&sender_, kBaseLayerTid); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); // Sender module should send an SR. EXPECT_EQ(0, sender_.impl_->SendRTCP(kRtcpReport)); @@ -338,7 +352,7 @@ TEST_F(RtpRtcpImplTest, RttForReceiverOnly) { // Sender module should send a response to the last received RTRR (DLRR). clock_.AdvanceTimeMilliseconds(1000); // Send Frame before sending a SR. - SendFrame(&sender_, kBaseLayerTid); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); EXPECT_EQ(0, sender_.impl_->SendRTCP(kRtcpReport)); // Verify RTT. @@ -367,7 +381,7 @@ TEST_F(RtpRtcpImplTest, NoSrBeforeMedia) { EXPECT_EQ(-1, sender_.RtcpSent().first_packet_time_ms); EXPECT_EQ(receiver_.RtcpSent().first_packet_time_ms, current_time); - SendFrame(&sender_, kBaseLayerTid); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); EXPECT_EQ(sender_.RtcpSent().first_packet_time_ms, current_time); } @@ -459,7 +473,7 @@ TEST_F(RtpRtcpImplTest, SendsInitialNackList) { uint16_t nack_list[kNackLength] = {123}; EXPECT_EQ(0U, sender_.RtcpSent().nack_packets); // Send Frame before sending a compound RTCP that starts with SR. - SendFrame(&sender_, kBaseLayerTid); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); EXPECT_EQ(1U, sender_.RtcpSent().nack_packets); EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(123)); @@ -471,7 +485,7 @@ TEST_F(RtpRtcpImplTest, SendsExtendedNackList) { uint16_t nack_list[kNackLength] = {123}; EXPECT_EQ(0U, sender_.RtcpSent().nack_packets); // Send Frame before sending a compound RTCP that starts with SR. - SendFrame(&sender_, kBaseLayerTid); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); EXPECT_EQ(1U, sender_.RtcpSent().nack_packets); EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(123)); @@ -496,7 +510,7 @@ TEST_F(RtpRtcpImplTest, ReSendsNackListAfterRttMs) { uint16_t nack_list[kNackLength] = {123, 125}; EXPECT_EQ(0U, sender_.RtcpSent().nack_packets); // Send Frame before sending a compound RTCP that starts with SR. - SendFrame(&sender_, kBaseLayerTid); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); EXPECT_EQ(1U, sender_.RtcpSent().nack_packets); EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(123, 125)); @@ -593,7 +607,7 @@ TEST_F(RtpRtcpImplTest, SendsKeepaliveAfterTimout) { EXPECT_EQ(2U, sender_.transport_.NumKeepaliveSent()); // Send actual payload data, no keep-alive expected. - SendFrame(&sender_, 0); + SendFrame(&sender_, sender_video_.get(), 0); sender_.impl_->Process(); EXPECT_EQ(2U, sender_.transport_.NumKeepaliveSent()); @@ -615,7 +629,7 @@ TEST_F(RtpRtcpImplTest, ConfigurableRtcpReportInterval) { sender_.SetRtcpReportIntervalAndReset(kVideoReportInterval); SetUp(); - SendFrame(&sender_, kBaseLayerTid); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); // Initial state sender_.impl_->Process(); @@ -634,7 +648,7 @@ TEST_F(RtpRtcpImplTest, ConfigurableRtcpReportInterval) { EXPECT_GT(sender_.RtcpSent().first_packet_time_ms, -1); EXPECT_EQ(sender_.transport_.NumRtcpSent(), 1u); - SendFrame(&sender_, kBaseLayerTid); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); // Move ahead to the last possible second before second rtcp is expected. clock_.AdvanceTimeMilliseconds(kVideoReportInterval * 1 / 2 - 1); diff --git a/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_unittest.cc index 2bd3c949ee..1c74e7e4c1 100644 --- a/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -500,8 +500,10 @@ TEST_P(RtpSenderTestWithoutPacer, OnSendSideDelayUpdated) { &mock_rtc_event_log_, nullptr, nullptr, nullptr, false, nullptr, false, false, FieldTrialBasedConfig())); rtp_sender_->SetSSRC(kSsrc); + PlayoutDelayOracle playout_delay_oracle; RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), nullptr, - nullptr, false, FieldTrialBasedConfig()); + &playout_delay_oracle, nullptr, false, + FieldTrialBasedConfig()); const uint8_t kPayloadType = 127; const char payload_name[] = "GENERIC"; @@ -1066,8 +1068,10 @@ TEST_P(RtpSenderTest, SendRedundantPayloads) { TEST_P(RtpSenderTestWithoutPacer, SendGenericVideo) { const char payload_name[] = "GENERIC"; const uint8_t payload_type = 127; + PlayoutDelayOracle playout_delay_oracle; RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), nullptr, - nullptr, false, FieldTrialBasedConfig()); + &playout_delay_oracle, nullptr, false, + FieldTrialBasedConfig()); rtp_sender_video.RegisterPayloadType(payload_type, payload_name); uint8_t payload[] = {47, 11, 32, 93, 89}; @@ -1121,9 +1125,10 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) { rtp_sender_->SetSequenceNumber(kSeqNum); rtp_sender_->SetStorePacketsStatus(true, 10); + PlayoutDelayOracle playout_delay_oracle; RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), - &flexfec_sender, nullptr, false, - FieldTrialBasedConfig()); + &flexfec_sender, &playout_delay_oracle, + nullptr, false, FieldTrialBasedConfig()); rtp_sender_video.RegisterPayloadType(kMediaPayloadType, "GENERIC"); // Parameters selected to generate a single FEC packet per media packet. @@ -1194,9 +1199,10 @@ TEST_P(RtpSenderTest, NoFlexfecForTimingFrames) { rtp_sender_->SetSequenceNumber(kSeqNum); rtp_sender_->SetStorePacketsStatus(true, 10); + PlayoutDelayOracle playout_delay_oracle; RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), - &flexfec_sender, nullptr, false, - FieldTrialBasedConfig()); + &flexfec_sender, &playout_delay_oracle, + nullptr, false, FieldTrialBasedConfig()); rtp_sender_video.RegisterPayloadType(kMediaPayloadType, "GENERIC"); // Need extension to be registered for timing frames to be sent. @@ -1291,9 +1297,10 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) { rtp_sender_->SetSSRC(kMediaSsrc); rtp_sender_->SetSequenceNumber(kSeqNum); + PlayoutDelayOracle playout_delay_oracle; RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), - &flexfec_sender, nullptr, false, - FieldTrialBasedConfig()); + &flexfec_sender, &playout_delay_oracle, + nullptr, false, FieldTrialBasedConfig()); rtp_sender_video.RegisterPayloadType(kMediaPayloadType, "GENERIC"); // Parameters selected to generate a single FEC packet per media packet. @@ -1420,9 +1427,10 @@ TEST_P(RtpSenderTest, FecOverheadRate) { rtp_sender_->SetSSRC(kMediaSsrc); rtp_sender_->SetSequenceNumber(kSeqNum); + PlayoutDelayOracle playout_delay_oracle; RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), - &flexfec_sender, nullptr, false, - FieldTrialBasedConfig()); + &flexfec_sender, &playout_delay_oracle, + nullptr, false, FieldTrialBasedConfig()); rtp_sender_video.RegisterPayloadType(kMediaPayloadType, "GENERIC"); // Parameters selected to generate a single FEC packet per media packet. FecProtectionParams params; @@ -1489,8 +1497,10 @@ TEST_P(RtpSenderTest, BitrateCallbacks) { false, false, FieldTrialBasedConfig())); rtp_sender_->SetSSRC(kSsrc); + PlayoutDelayOracle playout_delay_oracle; RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), nullptr, - nullptr, false, FieldTrialBasedConfig()); + &playout_delay_oracle, nullptr, false, + FieldTrialBasedConfig()); const char payload_name[] = "GENERIC"; const uint8_t payload_type = 127; rtp_sender_video.RegisterPayloadType(payload_type, payload_name); @@ -1574,8 +1584,10 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacks) { const uint8_t kUlpfecPayloadType = 97; const char payload_name[] = "GENERIC"; const uint8_t payload_type = 127; + PlayoutDelayOracle playout_delay_oracle; RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), nullptr, - nullptr, false, FieldTrialBasedConfig()); + &playout_delay_oracle, nullptr, false, + FieldTrialBasedConfig()); rtp_sender_video.RegisterPayloadType(payload_type, payload_name); uint8_t payload[] = {47, 11, 32, 93, 89}; rtp_sender_->SetStorePacketsStatus(true, 1); diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 32a3621248..8b835bdc8f 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -186,6 +186,7 @@ const char* FrameTypeToString(FrameType frame_type) { RTPSenderVideo::RTPSenderVideo(Clock* clock, RTPSender* rtp_sender, FlexfecSender* flexfec_sender, + PlayoutDelayOracle* playout_delay_oracle, FrameEncryptorInterface* frame_encryptor, bool require_frame_encryption, const WebRtcKeyValueConfig& field_trials) @@ -195,6 +196,7 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock, kConditionallyRetransmitHigherLayers), last_rotation_(kVideoRotation_0), transmit_color_space_next_frame_(false), + playout_delay_oracle_(playout_delay_oracle), red_payload_type_(-1), ulpfec_payload_type_(-1), flexfec_sender_(flexfec_sender), @@ -207,7 +209,9 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock, require_frame_encryption_(require_frame_encryption), generic_descriptor_auth_experiment_( field_trials.Lookup("WebRTC-GenericDescriptorAuth").find("Enabled") == - 0) {} + 0) { + RTC_DCHECK(playout_delay_oracle_); +} RTPSenderVideo::~RTPSenderVideo() {} @@ -448,7 +452,7 @@ bool RTPSenderVideo::SendVideo(FrameType frame_type, video_header->frame_marking.temporal_id != kNoTemporalIdx; const absl::optional playout_delay = - playout_delay_oracle_.PlayoutDelayToSend(video_header->playout_delay); + playout_delay_oracle_->PlayoutDelayToSend(video_header->playout_delay); { rtc::CritScope cs(&crit_); // According to @@ -667,8 +671,8 @@ bool RTPSenderVideo::SendVideo(FrameType frame_type, packetized_payload_size += packet->payload_size(); if (i == 0) { - playout_delay_oracle_.OnSentPacket(packet->SequenceNumber(), - playout_delay); + playout_delay_oracle_->OnSentPacket(packet->SequenceNumber(), + playout_delay); } // No FEC protection for upper temporal layers, if used. bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx; diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index b3c7f305c7..9772b867aa 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -53,6 +53,7 @@ class RTPSenderVideo { RTPSenderVideo(Clock* clock, RTPSender* rtpSender, FlexfecSender* flexfec_sender, + PlayoutDelayOracle* playout_delay_oracle, FrameEncryptorInterface* frame_encryptor, bool require_frame_encryption, const WebRtcKeyValueConfig& field_trials); @@ -70,10 +71,15 @@ class RTPSenderVideo { void RegisterPayloadType(int8_t payload_type, absl::string_view payload_name); - // ULPFEC. + // Set RED and ULPFEC payload types. A payload type of -1 means that the + // corresponding feature is turned off. Note that we DO NOT support enabling + // ULPFEC without enabling RED, and RED is only ever used when ULPFEC is + // enabled. void SetUlpfecConfig(int red_payload_type, int ulpfec_payload_type); // FlexFEC/ULPFEC. + // Set FEC rates, max frames before FEC is sent, and type of FEC masks. + // Returns false on failure. void SetFecParameters(const FecProtectionParams& delta_params, const FecProtectionParams& key_params); @@ -82,11 +88,11 @@ class RTPSenderVideo { uint32_t VideoBitrateSent() const; uint32_t FecOverheadRate() const; - uint32_t PacketizationOverheadBps() const; - void OnReceivedAck(int64_t extended_highest_sequence_number) { - playout_delay_oracle_.OnReceivedAck(extended_highest_sequence_number); - } + // Returns the current packetization overhead rate, in bps. Note that this is + // the payload overhead, eg the VP8 payload headers, not the RTP headers + // or extension/ + uint32_t PacketizationOverheadBps() const; protected: static uint8_t GetTemporalId(const RTPVideoHeader& header); @@ -159,7 +165,7 @@ class RTPSenderVideo { // Tracks the current request for playout delay limits from application // and decides whether the current RTP frame should include the playout // delay extension on header. - PlayoutDelayOracle playout_delay_oracle_; + PlayoutDelayOracle* const playout_delay_oracle_; // RED/ULPFEC. int red_payload_type_ RTC_GUARDED_BY(crit_); diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index 47fabd1272..fbd15c6d99 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -101,6 +101,7 @@ class TestRtpSenderVideo : public RTPSenderVideo { : RTPSenderVideo(clock, rtp_sender, flexfec_sender, + &playout_delay_oracle_, nullptr, false, field_trials) {} @@ -113,6 +114,7 @@ class TestRtpSenderVideo : public RTPSenderVideo { retransmission_settings, expected_retransmission_time_ms); } + PlayoutDelayOracle playout_delay_oracle_; }; class FieldTrials : public WebRtcKeyValueConfig {