From 64be7fa7d8c1e2c897e70ef5d8de6dd5539ce6e8 Mon Sep 17 00:00:00 2001 From: Stefan Holmer Date: Thu, 4 Oct 2018 15:21:55 +0200 Subject: [PATCH] Move FecController to RtpVideoSender. This also moves the packet feedback tracking to RtpVideoSender. Bug: webrtc:9517 Change-Id: Ifb1ff85051730108a0b0d1dd30f6f8595ad2af6e Reviewed-on: https://webrtc-review.googlesource.com/c/95920 Reviewed-by: Sebastian Jansson Reviewed-by: Niels Moller Commit-Queue: Stefan Holmer Cr-Commit-Position: refs/heads/master@{#25019} --- audio/test/audio_bwe_integration_test.cc | 2 +- call/BUILD.gn | 3 + call/call.cc | 29 +-- call/call.h | 3 +- call/degraded_call.cc | 5 +- call/degraded_call.h | 6 +- call/rtp_transport_controller_send.cc | 22 +- call/rtp_transport_controller_send.h | 6 +- .../rtp_transport_controller_send_interface.h | 9 +- call/rtp_video_sender.cc | 200 +++++++++++++++--- call/rtp_video_sender.h | 61 ++++-- call/rtp_video_sender_interface.h | 22 +- call/rtp_video_sender_unittest.cc | 17 +- .../test/mock_rtp_transport_controller_send.h | 7 +- media/engine/fakewebrtccall.cc | 21 +- media/engine/fakewebrtccall.h | 7 +- media/engine/webrtcvideoengine.cc | 4 +- media/engine/webrtcvoiceengine.cc | 3 +- test/scenario/audio_stream.cc | 4 +- video/video_send_stream.cc | 9 - video/video_send_stream.h | 2 - video/video_send_stream_impl.cc | 180 ++-------------- video/video_send_stream_impl.h | 33 +-- video/video_send_stream_impl_unittest.cc | 101 +++++---- video/video_send_stream_tests.cc | 10 +- 25 files changed, 375 insertions(+), 391 deletions(-) diff --git a/audio/test/audio_bwe_integration_test.cc b/audio/test/audio_bwe_integration_test.cc index 10c8784271..8ec2a7cf4a 100644 --- a/audio/test/audio_bwe_integration_test.cc +++ b/audio/test/audio_bwe_integration_test.cc @@ -138,7 +138,7 @@ class NoBandwidthDropAfterDtx : public AudioBweTest { void PerformTest() override { stats_poller_.PostDelayedTask( std::unique_ptr(new StatsPollTask(sender_call_)), 100); - sender_call_->OnTransportOverheadChanged(webrtc::MediaType::AUDIO, 0); + sender_call_->OnAudioTransportOverheadChanged(0); AudioBweTest::PerformTest(); } diff --git a/call/BUILD.gn b/call/BUILD.gn index 7c1cb3f62b..611d9e8adb 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -65,6 +65,7 @@ rtc_source_set("rtp_interfaces") { ] deps = [ "../api:array_view", + "../api:fec_controller_api", "../api:libjingle_peerconnection_api", "../api/transport:bitrate_settings", "../logging:rtc_event_log_api", @@ -117,6 +118,7 @@ rtc_source_set("rtp_sender") { ":bitrate_configurator", ":rtp_interfaces", "..:webrtc_common", + "../api:fec_controller_api", "../api:transport_api", "../api/transport:network_control", "../api/video_codecs:video_codecs_api", @@ -341,6 +343,7 @@ if (rtc_include_tests) { "../modules/rtp_rtcp:rtp_rtcp_format", "../modules/utility:mock_process_thread", "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding", "../rtc_base:checks", "../rtc_base:rate_limiter", "../rtc_base:rtc_base_approved", diff --git a/call/call.cc b/call/call.cc index 1062baf49d..635874b2d6 100644 --- a/call/call.cc +++ b/call/call.cc @@ -221,8 +221,8 @@ class Call final : public webrtc::Call, void SignalChannelNetworkState(MediaType media, NetworkState state) override; - void OnTransportOverheadChanged(MediaType media, - int transport_overhead_per_packet) override; + void OnAudioTransportOverheadChanged( + int transport_overhead_per_packet) override; void OnSentPacket(const rtc::SentPacket& sent_packet) override; @@ -997,27 +997,10 @@ void Call::SignalChannelNetworkState(MediaType media, NetworkState state) { } } -void Call::OnTransportOverheadChanged(MediaType media, - int transport_overhead_per_packet) { - switch (media) { - case MediaType::AUDIO: { - ReadLockScoped read_lock(*send_crit_); - for (auto& kv : audio_send_ssrcs_) { - kv.second->SetTransportOverhead(transport_overhead_per_packet); - } - break; - } - case MediaType::VIDEO: { - ReadLockScoped read_lock(*send_crit_); - for (auto& kv : video_send_ssrcs_) { - kv.second->SetTransportOverhead(transport_overhead_per_packet); - } - break; - } - case MediaType::ANY: - case MediaType::DATA: - RTC_NOTREACHED(); - break; +void Call::OnAudioTransportOverheadChanged(int transport_overhead_per_packet) { + ReadLockScoped read_lock(*send_crit_); + for (auto& kv : audio_send_ssrcs_) { + kv.second->SetTransportOverhead(transport_overhead_per_packet); } } diff --git a/call/call.h b/call/call.h index d8b1b5777d..4167296d75 100644 --- a/call/call.h +++ b/call/call.h @@ -113,8 +113,7 @@ class Call { virtual void SignalChannelNetworkState(MediaType media, NetworkState state) = 0; - virtual void OnTransportOverheadChanged( - MediaType media, + virtual void OnAudioTransportOverheadChanged( int transport_overhead_per_packet) = 0; virtual void OnSentPacket(const rtc::SentPacket& sent_packet) = 0; diff --git a/call/degraded_call.cc b/call/degraded_call.cc index df9869729a..8181310e78 100644 --- a/call/degraded_call.cc +++ b/call/degraded_call.cc @@ -157,10 +157,9 @@ void DegradedCall::SignalChannelNetworkState(MediaType media, call_->SignalChannelNetworkState(media, state); } -void DegradedCall::OnTransportOverheadChanged( - MediaType media, +void DegradedCall::OnAudioTransportOverheadChanged( int transport_overhead_per_packet) { - call_->OnTransportOverheadChanged(media, transport_overhead_per_packet); + call_->OnAudioTransportOverheadChanged(transport_overhead_per_packet); } void DegradedCall::OnSentPacket(const rtc::SentPacket& sent_packet) { diff --git a/call/degraded_call.h b/call/degraded_call.h index 257ee0be25..3c0b80df86 100644 --- a/call/degraded_call.h +++ b/call/degraded_call.h @@ -70,10 +70,8 @@ class DegradedCall : public Call, private Transport, private PacketReceiver { bitrate_allocation_strategy) override; void SignalChannelNetworkState(MediaType media, NetworkState state) override; - - void OnTransportOverheadChanged(MediaType media, - int transport_overhead_per_packet) override; - + void OnAudioTransportOverheadChanged( + int transport_overhead_per_packet) override; void OnSentPacket(const rtc::SentPacket& sent_packet) override; protected: diff --git a/call/rtp_transport_controller_send.cc b/call/rtp_transport_controller_send.cc index 92d972911d..54aef21fd9 100644 --- a/call/rtp_transport_controller_send.cc +++ b/call/rtp_transport_controller_send.cc @@ -22,6 +22,7 @@ namespace webrtc { namespace { static const int64_t kRetransmitWindowSizeMs = 500; +static const size_t kMaxOverheadBytes = 500; const char kTaskQueueExperiment[] = "WebRTC-TaskQueueCongestionControl"; using TaskQueueController = webrtc::webrtc_cc::SendSideCongestionController; @@ -92,13 +93,15 @@ RtpVideoSenderInterface* RtpTransportControllerSend::CreateRtpVideoSender( const RtcpConfig& rtcp_config, Transport* send_transport, const RtpSenderObservers& observers, - RtcEventLog* event_log) { + RtcEventLog* event_log, + std::unique_ptr fec_controller) { video_rtp_senders_.push_back(absl::make_unique( ssrcs, suspended_ssrcs, states, rtp_config, rtcp_config, send_transport, observers, // TODO(holmer): Remove this circular dependency by injecting // the parts of RtpTransportControllerSendInterface that are really used. - this, event_log, &retransmission_rate_limiter_)); + this, event_log, &retransmission_rate_limiter_, + std::move(fec_controller))); return video_rtp_senders_.back().get(); } @@ -315,4 +318,19 @@ void RtpTransportControllerSend::SetAllocatedBitrateWithoutFeedback( send_side_cc_->SetAllocatedBitrateWithoutFeedback(bitrate_bps); } } + +void RtpTransportControllerSend::OnTransportOverheadChanged( + size_t transport_overhead_bytes_per_packet) { + if (transport_overhead_bytes_per_packet >= kMaxOverheadBytes) { + RTC_LOG(LS_ERROR) << "Transport overhead exceeds " << kMaxOverheadBytes; + return; + } + + // TODO(holmer): Call AudioRtpSenders when they have been moved to + // RtpTransportControllerSend. + for (auto& rtp_video_sender : video_rtp_senders_) { + rtp_video_sender->OnTransportOverheadChanged( + transport_overhead_bytes_per_packet); + } +} } // namespace webrtc diff --git a/call/rtp_transport_controller_send.h b/call/rtp_transport_controller_send.h index 496a00ede5..6aaf29d6bc 100644 --- a/call/rtp_transport_controller_send.h +++ b/call/rtp_transport_controller_send.h @@ -55,7 +55,8 @@ class RtpTransportControllerSend final const RtcpConfig& rtcp_config, Transport* send_transport, const RtpSenderObservers& observers, - RtcEventLog* event_log) override; + RtcEventLog* event_log, + std::unique_ptr fec_controller) override; void DestroyRtpVideoSender( RtpVideoSenderInterface* rtp_video_sender) override; @@ -102,6 +103,9 @@ class RtpTransportControllerSend final void SetAllocatedBitrateWithoutFeedback(uint32_t bitrate_bps) override; + void OnTransportOverheadChanged( + size_t transport_overhead_per_packet) override; + private: const Clock* const clock_; PacketRouter packet_router_; diff --git a/call/rtp_transport_controller_send_interface.h b/call/rtp_transport_controller_send_interface.h index 828fec40cf..5c51c54fe1 100644 --- a/call/rtp_transport_controller_send_interface.h +++ b/call/rtp_transport_controller_send_interface.h @@ -14,11 +14,13 @@ #include #include +#include #include #include #include "absl/types/optional.h" #include "api/bitrate_constraints.h" +#include "api/fec_controller.h" #include "api/transport/bitrate_settings.h" #include "call/rtp_config.h" #include "logging/rtc_event_log/rtc_event_log.h" @@ -58,7 +60,6 @@ struct RtpSenderObservers { RtcpPacketTypeCounterObserver* rtcp_type_observer; SendSideDelayObserver* send_delay_observer; SendPacketObserver* send_packet_observer; - OverheadObserver* overhead_observer; }; // An RtpTransportController should own everything related to the RTP @@ -99,7 +100,8 @@ class RtpTransportControllerSendInterface { const RtcpConfig& rtcp_config, Transport* send_transport, const RtpSenderObservers& observers, - RtcEventLog* event_log) = 0; + RtcEventLog* event_log, + std::unique_ptr fec_controller) = 0; virtual void DestroyRtpVideoSender( RtpVideoSenderInterface* rtp_video_sender) = 0; @@ -149,6 +151,9 @@ class RtpTransportControllerSendInterface { const BitrateSettings& preferences) = 0; virtual void SetAllocatedBitrateWithoutFeedback(uint32_t bitrate_bps) = 0; + + virtual void OnTransportOverheadChanged( + size_t transport_overhead_per_packet) = 0; }; } // namespace webrtc diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 8a550e14f5..10d7e7daed 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -31,6 +31,12 @@ namespace webrtc { namespace { static const int kMinSendSidePacketHistorySize = 600; +// Assume an average video stream has around 3 packets per frame (1 mbps / 30 +// fps / 1400B) A sequence number set with size 5500 will be able to store +// packet sequence number for at least last 60 seconds. +static const int kSendSideSeqNumSetMaxSize = 5500; +// We don't do MTU discovery, so assume that we have the standard ethernet MTU. +static const size_t kPathMTU = 1500; std::vector> CreateRtpRtcpModules( const std::vector& ssrcs, @@ -149,6 +155,21 @@ std::unique_ptr MaybeCreateFlexfecSender( rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions, RTPSender::FecExtensionSizes(), rtp_state, Clock::GetRealTimeClock()); } + +uint32_t CalculateOverheadRateBps(int packets_per_second, + size_t overhead_bytes_per_packet, + uint32_t max_overhead_bps) { + uint32_t overhead_bps = + static_cast(8 * overhead_bytes_per_packet * packets_per_second); + return std::min(overhead_bps, max_overhead_bps); +} + +int CalculatePacketRate(uint32_t bitrate_bps, size_t packet_size_bytes) { + size_t packet_size_bits = 8 * packet_size_bytes; + // Ceil for int value of bitrate_bps / packet_size_bits. + return static_cast((bitrate_bps + packet_size_bits - 1) / + packet_size_bits); +} } // namespace RtpVideoSender::RtpVideoSender( @@ -161,11 +182,15 @@ RtpVideoSender::RtpVideoSender( const RtpSenderObservers& observers, RtpTransportControllerSendInterface* transport, RtcEventLog* event_log, - RateLimiter* retransmission_limiter) - : active_(false), + RateLimiter* retransmission_limiter, + std::unique_ptr fec_controller) + : send_side_bwe_with_overhead_( + webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), + active_(false), module_process_thread_(nullptr), suspended_ssrcs_(std::move(suspended_ssrcs)), flexfec_sender_(MaybeCreateFlexfecSender(rtp_config, suspended_ssrcs_)), + fec_controller_(std::move(fec_controller)), rtp_modules_( CreateRtpRtcpModules(ssrcs, rtp_config.flexfec.protected_media_ssrcs, @@ -183,10 +208,13 @@ RtpVideoSender::RtpVideoSender( observers.send_packet_observer, event_log, retransmission_limiter, - observers.overhead_observer, + this, transport->keepalive_config())), rtp_config_(rtp_config), - transport_(transport) { + transport_(transport), + transport_overhead_bytes_per_packet_(0), + overhead_bytes_per_packet_(0), + encoder_target_rate_bps_(0) { RTC_DCHECK_EQ(ssrcs.size(), rtp_modules_.size()); module_process_thread_checker_.DetachFromThread(); // SSRCs are assumed to be sorted in the same order as |rtp_modules|. @@ -246,12 +274,24 @@ RtpVideoSender::RtpVideoSender( rtp_rtcp->RegisterVideoSendPayload(rtp_config.payload_type, rtp_config.payload_name.c_str()); } + // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, + // so enable that logic if either of those FEC schemes are enabled. + fec_controller_->SetProtectionMethod(FecEnabled(), NackEnabled()); + + fec_controller_->SetProtectionCallback(this); + // Signal congestion controller this object is ready for OnPacket* callbacks. + if (fec_controller_->UseLossVectorMask()) { + transport_->RegisterPacketFeedbackObserver(this); + } } RtpVideoSender::~RtpVideoSender() { for (auto& rtp_rtcp : rtp_modules_) { transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp.get()); } + if (fec_controller_->UseLossVectorMask()) { + transport_->DeRegisterPacketFeedbackObserver(this); + } } void RtpVideoSender::RegisterProcessThread( @@ -302,6 +342,8 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) { + fec_controller_->UpdateWithEncodedData(encoded_image._length, + encoded_image._frameType); rtc::CritScope lock(&crit_); RTC_DCHECK(!rtp_modules_.empty()); if (!active_) @@ -440,34 +482,6 @@ void RtpVideoSender::DeliverRtcp(const uint8_t* packet, size_t length) { rtp_rtcp->IncomingRtcpPacket(packet, length); } -void RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) { - *sent_video_rate_bps = 0; - *sent_nack_rate_bps = 0; - *sent_fec_rate_bps = 0; - for (auto& rtp_rtcp : rtp_modules_) { - uint32_t not_used = 0; - uint32_t module_video_rate = 0; - uint32_t module_fec_rate = 0; - uint32_t module_nack_rate = 0; - rtp_rtcp->SetFecParameters(*delta_params, *key_params); - rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate, - &module_nack_rate); - *sent_video_rate_bps += module_video_rate; - *sent_nack_rate_bps += module_nack_rate; - *sent_fec_rate_bps += module_fec_rate; - } -} - -void RtpVideoSender::SetMaxRtpPacketSize(size_t max_rtp_packet_size) { - for (auto& rtp_rtcp : rtp_modules_) { - rtp_rtcp->SetMaxRtpPacketSize(max_rtp_packet_size); - } -} - void RtpVideoSender::ConfigureSsrcs(const RtpConfig& rtp_config) { // Configure regular SSRCs. for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) { @@ -551,4 +565,126 @@ std::map RtpVideoSender::GetRtpPayloadStates() } return payload_states; } + +void RtpVideoSender::OnTransportOverheadChanged( + size_t transport_overhead_bytes_per_packet) { + rtc::CritScope lock(&crit_); + transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet; + + size_t max_rtp_packet_size = + std::min(rtp_config_.max_packet_size, + kPathMTU - transport_overhead_bytes_per_packet_); + for (auto& rtp_rtcp : rtp_modules_) { + rtp_rtcp->SetMaxRtpPacketSize(max_rtp_packet_size); + } +} + +void RtpVideoSender::OnOverheadChanged(size_t overhead_bytes_per_packet) { + rtc::CritScope lock(&crit_); + overhead_bytes_per_packet_ = overhead_bytes_per_packet; +} + +void RtpVideoSender::OnBitrateUpdated(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt, + int framerate) { + // Substract overhead from bitrate. + rtc::CritScope lock(&crit_); + uint32_t payload_bitrate_bps = bitrate_bps; + if (send_side_bwe_with_overhead_) { + payload_bitrate_bps -= CalculateOverheadRateBps( + CalculatePacketRate( + bitrate_bps, + rtp_config_.max_packet_size + transport_overhead_bytes_per_packet_), + overhead_bytes_per_packet_ + transport_overhead_bytes_per_packet_, + bitrate_bps); + } + + // Get the encoder target rate. It is the estimated network rate - + // protection overhead. + encoder_target_rate_bps_ = fec_controller_->UpdateFecRates( + payload_bitrate_bps, framerate, fraction_loss, loss_mask_vector_, rtt); + loss_mask_vector_.clear(); + + uint32_t encoder_overhead_rate_bps = + send_side_bwe_with_overhead_ + ? CalculateOverheadRateBps( + CalculatePacketRate(encoder_target_rate_bps_, + rtp_config_.max_packet_size + + transport_overhead_bytes_per_packet_ - + overhead_bytes_per_packet_), + overhead_bytes_per_packet_ + + transport_overhead_bytes_per_packet_, + bitrate_bps - encoder_target_rate_bps_) + : 0; + + // When the field trial "WebRTC-SendSideBwe-WithOverhead" is enabled + // protection_bitrate includes overhead. + protection_bitrate_bps_ = + bitrate_bps - (encoder_target_rate_bps_ + encoder_overhead_rate_bps); +} + +uint32_t RtpVideoSender::GetPayloadBitrateBps() const { + return encoder_target_rate_bps_; +} + +uint32_t RtpVideoSender::GetProtectionBitrateBps() const { + return protection_bitrate_bps_; +} + +int RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) { + *sent_video_rate_bps = 0; + *sent_nack_rate_bps = 0; + *sent_fec_rate_bps = 0; + for (auto& rtp_rtcp : rtp_modules_) { + uint32_t not_used = 0; + uint32_t module_video_rate = 0; + uint32_t module_fec_rate = 0; + uint32_t module_nack_rate = 0; + rtp_rtcp->SetFecParameters(*delta_params, *key_params); + rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate, + &module_nack_rate); + *sent_video_rate_bps += module_video_rate; + *sent_nack_rate_bps += module_nack_rate; + *sent_fec_rate_bps += module_fec_rate; + } + return 0; +} + +void RtpVideoSender::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) { + const auto ssrcs = rtp_config_.ssrcs; + if (std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()) { + feedback_packet_seq_num_set_.insert(seq_num); + if (feedback_packet_seq_num_set_.size() > kSendSideSeqNumSetMaxSize) { + RTC_LOG(LS_WARNING) << "Feedback packet sequence number set exceed it's " + "max size', will get reset."; + feedback_packet_seq_num_set_.clear(); + } + } +} + +void RtpVideoSender::OnPacketFeedbackVector( + const std::vector& packet_feedback_vector) { + rtc::CritScope lock(&crit_); + // Lost feedbacks are not considered to be lost packets. + for (const PacketFeedback& packet : packet_feedback_vector) { + auto it = feedback_packet_seq_num_set_.find(packet.sequence_number); + if (it != feedback_packet_seq_num_set_.end()) { + const bool lost = packet.arrival_time_ms == PacketFeedback::kNotReceived; + loss_mask_vector_.push_back(lost); + feedback_packet_seq_num_set_.erase(it); + } + } +} + +void RtpVideoSender::SetEncodingData(size_t width, + size_t height, + size_t num_temporal_layers) { + fec_controller_->SetEncodingData(width, height, num_temporal_layers, + rtp_config_.max_packet_size); +} } // namespace webrtc diff --git a/call/rtp_video_sender.h b/call/rtp_video_sender.h index 1735dc751a..db329be00b 100644 --- a/call/rtp_video_sender.h +++ b/call/rtp_video_sender.h @@ -13,9 +13,11 @@ #include #include +#include #include #include "api/call/transport.h" +#include "api/fec_controller.h" #include "api/video_codecs/video_encoder.h" #include "call/rtp_config.h" #include "call/rtp_payload_params.h" @@ -40,7 +42,10 @@ class RtpTransportControllerSendInterface; // RtpVideoSender routes outgoing data to the correct sending RTP module, based // on the simulcast layer in RTPVideoHeader. -class RtpVideoSender : public RtpVideoSenderInterface { +class RtpVideoSender : public RtpVideoSenderInterface, + public OverheadObserver, + public VCMProtectionCallback, + public PacketFeedbackObserver { public: // Rtp modules are assumed to be sorted in simulcast index order. RtpVideoSender( @@ -53,7 +58,8 @@ class RtpVideoSender : public RtpVideoSenderInterface { const RtpSenderObservers& observers, RtpTransportControllerSendInterface* transport, RtcEventLog* event_log, - RateLimiter* retransmission_limiter); // move inside RtpTransport + RateLimiter* retransmission_limiter, // move inside RtpTransport + std::unique_ptr fec_controller); ~RtpVideoSender() override; // RegisterProcessThread register |module_process_thread| with those objects @@ -76,19 +82,14 @@ class RtpVideoSender : public RtpVideoSenderInterface { std::map GetRtpStates() const override; std::map GetRtpPayloadStates() const override; - bool FecEnabled() const override; - - bool NackEnabled() const override; - void DeliverRtcp(const uint8_t* packet, size_t length) override; - void ProtectionRequest(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) override; - - void SetMaxRtpPacketSize(size_t max_rtp_packet_size) override; + // Implements webrtc::VCMProtectionCallback. + int ProtectionRequest(const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) override; // Implements EncodedImageCallback. // Returns 0 if the packet was routed / sent, -1 otherwise. @@ -100,11 +101,36 @@ class RtpVideoSender : public RtpVideoSenderInterface { void OnBitrateAllocationUpdated( const VideoBitrateAllocation& bitrate) override; + void OnTransportOverheadChanged( + size_t transport_overhead_bytes_per_packet) override; + // Implements OverheadObserver. + void OnOverheadChanged(size_t overhead_bytes_per_packet) override; + void OnBitrateUpdated(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt, + int framerate) override; + uint32_t GetPayloadBitrateBps() const override; + uint32_t GetProtectionBitrateBps() const override; + void SetEncodingData(size_t width, + size_t height, + size_t num_temporal_layers) override; + + // From PacketFeedbackObserver. + void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override; + void OnPacketFeedbackVector( + const std::vector& packet_feedback_vector) override; + private: void UpdateModuleSendingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); void ConfigureProtection(const RtpConfig& rtp_config); void ConfigureSsrcs(const RtpConfig& rtp_config); + bool FecEnabled() const; + bool NackEnabled() const; + const bool send_side_bwe_with_overhead_; + + // TODO(holmer): Remove crit_ once RtpVideoSender runs on the + // transport task queue. rtc::CriticalSection crit_; bool active_ RTC_GUARDED_BY(crit_); @@ -113,6 +139,7 @@ class RtpVideoSender : public RtpVideoSenderInterface { std::map suspended_ssrcs_; std::unique_ptr flexfec_sender_; + std::unique_ptr fec_controller_; // Rtp modules are assumed to be sorted in simulcast index order. const std::vector> rtp_modules_; const RtpConfig rtp_config_; @@ -125,6 +152,14 @@ class RtpVideoSender : public RtpVideoSenderInterface { int64_t shared_frame_id_ = 0; std::vector params_ RTC_GUARDED_BY(crit_); + size_t transport_overhead_bytes_per_packet_ RTC_GUARDED_BY(crit_); + size_t overhead_bytes_per_packet_ RTC_GUARDED_BY(crit_); + uint32_t protection_bitrate_bps_; + uint32_t encoder_target_rate_bps_; + + std::unordered_set feedback_packet_seq_num_set_; + std::vector loss_mask_vector_ RTC_GUARDED_BY(crit_); + RTC_DISALLOW_COPY_AND_ASSIGN(RtpVideoSender); }; diff --git a/call/rtp_video_sender_interface.h b/call/rtp_video_sender_interface.h index c69f1bab02..ecaca9b256 100644 --- a/call/rtp_video_sender_interface.h +++ b/call/rtp_video_sender_interface.h @@ -40,21 +40,21 @@ class RtpVideoSenderInterface : public EncodedImageCallback { virtual std::map GetRtpStates() const = 0; virtual std::map GetRtpPayloadStates() const = 0; - virtual bool FecEnabled() const = 0; - - virtual bool NackEnabled() const = 0; - virtual void DeliverRtcp(const uint8_t* packet, size_t length) = 0; - virtual void ProtectionRequest(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) = 0; - - virtual void SetMaxRtpPacketSize(size_t max_rtp_packet_size) = 0; virtual void OnBitrateAllocationUpdated( const VideoBitrateAllocation& bitrate) = 0; + virtual void OnBitrateUpdated(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt, + int framerate) = 0; + virtual void OnTransportOverheadChanged( + size_t transport_overhead_bytes_per_packet) = 0; + virtual uint32_t GetPayloadBitrateBps() const = 0; + virtual uint32_t GetProtectionBitrateBps() const = 0; + virtual void SetEncodingData(size_t width, + size_t height, + size_t num_temporal_layers) = 0; }; } // namespace webrtc #endif // CALL_RTP_VIDEO_SENDER_INTERFACE_H_ diff --git a/call/rtp_video_sender_unittest.cc b/call/rtp_video_sender_unittest.cc index 29fc2b6020..e91bfc4aeb 100644 --- a/call/rtp_video_sender_unittest.cc +++ b/call/rtp_video_sender_unittest.cc @@ -13,6 +13,7 @@ #include "call/rtp_transport_controller_send.h" #include "call/rtp_video_sender.h" +#include "modules/video_coding/fec_controller_default.h" #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/rate_limiter.h" #include "test/field_trial.h" @@ -46,11 +47,6 @@ class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver { MOCK_METHOD1(OnReceivedIntraFrameRequest, void(uint32_t)); }; -class MockOverheadObserver : public OverheadObserver { - public: - MOCK_METHOD1(OnOverheadChanged, void(size_t overhead_bytes_per_packet)); -}; - class MockCongestionObserver : public NetworkChangedObserver { public: MOCK_METHOD4(OnNetworkChanged, @@ -69,8 +65,7 @@ RtpSenderObservers CreateObservers( FrameCountObserver* frame_count_observer, RtcpPacketTypeCounterObserver* rtcp_type_observer, SendSideDelayObserver* send_delay_observer, - SendPacketObserver* send_packet_observer, - OverheadObserver* overhead_observer) { + SendPacketObserver* send_packet_observer) { RtpSenderObservers observers; observers.rtcp_rtt_stats = rtcp_rtt_stats; observers.intra_frame_callback = intra_frame_callback; @@ -81,7 +76,6 @@ RtpSenderObservers CreateObservers( observers.rtcp_type_observer = rtcp_type_observer; observers.send_delay_observer = send_delay_observer; observers.send_packet_observer = send_packet_observer; - observers.overhead_observer = overhead_observer; return observers; } @@ -111,9 +105,9 @@ class RtpVideoSenderTestFixture { config_.rtp, config_.rtcp, &transport_, CreateObservers(&call_stats_, &encoder_feedback_, &stats_proxy_, &stats_proxy_, &stats_proxy_, &stats_proxy_, - &stats_proxy_, &stats_proxy_, &send_delay_stats_, - &overhead_observer_), - &transport_controller_, &event_log_, &retransmission_rate_limiter_); + &stats_proxy_, &stats_proxy_, &send_delay_stats_), + &transport_controller_, &event_log_, &retransmission_rate_limiter_, + absl::make_unique(&clock_)); } RtpVideoSender* router() { return router_.get(); } @@ -121,7 +115,6 @@ class RtpVideoSenderTestFixture { private: NiceMock transport_; NiceMock congestion_observer_; - NiceMock overhead_observer_; NiceMock encoder_feedback_; SimulatedClock clock_; RtcEventLogNullImpl event_log_; diff --git a/call/test/mock_rtp_transport_controller_send.h b/call/test/mock_rtp_transport_controller_send.h index 828b030378..37587b2b76 100644 --- a/call/test/mock_rtp_transport_controller_send.h +++ b/call/test/mock_rtp_transport_controller_send.h @@ -12,6 +12,7 @@ #define CALL_TEST_MOCK_RTP_TRANSPORT_CONTROLLER_SEND_H_ #include +#include #include #include @@ -29,7 +30,7 @@ namespace webrtc { class MockRtpTransportControllerSend : public RtpTransportControllerSendInterface { public: - MOCK_METHOD8( + MOCK_METHOD9( CreateRtpVideoSender, RtpVideoSenderInterface*(const std::vector&, std::map, @@ -38,7 +39,8 @@ class MockRtpTransportControllerSend const RtcpConfig&, Transport*, const RtpSenderObservers&, - RtcEventLog*)); + RtcEventLog*, + std::unique_ptr)); MOCK_METHOD1(DestroyRtpVideoSender, void(RtpVideoSenderInterface*)); MOCK_METHOD0(GetWorkerQueue, rtc::TaskQueue*()); MOCK_METHOD0(packet_router, PacketRouter*()); @@ -65,6 +67,7 @@ class MockRtpTransportControllerSend MOCK_METHOD1(SetSdpBitrateParameters, void(const BitrateConstraints&)); MOCK_METHOD1(SetClientBitratePreferences, void(const BitrateSettings&)); MOCK_METHOD1(SetAllocatedBitrateWithoutFeedback, void(uint32_t)); + MOCK_METHOD1(OnTransportOverheadChanged, void(size_t)); }; } // namespace webrtc #endif // CALL_TEST_MOCK_RTP_TRANSPORT_CONTROLLER_SEND_H_ diff --git a/media/engine/fakewebrtccall.cc b/media/engine/fakewebrtccall.cc index 4a923bcb0e..ce99b9c103 100644 --- a/media/engine/fakewebrtccall.cc +++ b/media/engine/fakewebrtccall.cc @@ -390,9 +390,7 @@ FakeCall::FakeCall() : audio_network_state_(webrtc::kNetworkUp), video_network_state_(webrtc::kNetworkUp), num_created_send_streams_(0), - num_created_receive_streams_(0), - audio_transport_overhead_(0), - video_transport_overhead_(0) {} + num_created_receive_streams_(0) {} FakeCall::~FakeCall() { EXPECT_EQ(0u, video_send_streams_.size()); @@ -635,21 +633,8 @@ void FakeCall::SignalChannelNetworkState(webrtc::MediaType media, } } -void FakeCall::OnTransportOverheadChanged(webrtc::MediaType media, - int transport_overhead_per_packet) { - switch (media) { - case webrtc::MediaType::AUDIO: - audio_transport_overhead_ = transport_overhead_per_packet; - break; - case webrtc::MediaType::VIDEO: - video_transport_overhead_ = transport_overhead_per_packet; - break; - case webrtc::MediaType::DATA: - case webrtc::MediaType::ANY: - ADD_FAILURE() - << "SignalChannelNetworkState called with unknown parameter."; - } -} +void FakeCall::OnAudioTransportOverheadChanged( + int transport_overhead_per_packet) {} void FakeCall::OnSentPacket(const rtc::SentPacket& sent_packet) { last_sent_packet_ = sent_packet; diff --git a/media/engine/fakewebrtccall.h b/media/engine/fakewebrtccall.h index edb3585f3d..dbcedb8f41 100644 --- a/media/engine/fakewebrtccall.h +++ b/media/engine/fakewebrtccall.h @@ -317,8 +317,8 @@ class FakeCall final : public webrtc::Call, public webrtc::PacketReceiver { void SignalChannelNetworkState(webrtc::MediaType media, webrtc::NetworkState state) override; - void OnTransportOverheadChanged(webrtc::MediaType media, - int transport_overhead_per_packet) override; + void OnAudioTransportOverheadChanged( + int transport_overhead_per_packet) override; void OnSentPacket(const rtc::SentPacket& sent_packet) override; testing::NiceMock @@ -338,9 +338,6 @@ class FakeCall final : public webrtc::Call, public webrtc::PacketReceiver { int num_created_send_streams_; int num_created_receive_streams_; - - int audio_transport_overhead_; - int video_transport_overhead_; }; } // namespace cricket diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc index f35bcb6eeb..40b0c45a3d 100644 --- a/media/engine/webrtcvideoengine.cc +++ b/media/engine/webrtcvideoengine.cc @@ -1408,8 +1408,8 @@ void WebRtcVideoChannel::OnNetworkRouteChanged( const rtc::NetworkRoute& network_route) { call_->GetTransportControllerSend()->OnNetworkRouteChanged(transport_name, network_route); - call_->OnTransportOverheadChanged(webrtc::MediaType::VIDEO, - network_route.packet_overhead); + call_->GetTransportControllerSend()->OnTransportOverheadChanged( + network_route.packet_overhead); } void WebRtcVideoChannel::SetInterface(NetworkInterface* iface) { diff --git a/media/engine/webrtcvoiceengine.cc b/media/engine/webrtcvoiceengine.cc index 6b14c23d01..8ae7b991e8 100644 --- a/media/engine/webrtcvoiceengine.cc +++ b/media/engine/webrtcvoiceengine.cc @@ -2059,8 +2059,7 @@ void WebRtcVoiceMediaChannel::OnNetworkRouteChanged( RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); call_->GetTransportControllerSend()->OnNetworkRouteChanged(transport_name, network_route); - call_->OnTransportOverheadChanged(webrtc::MediaType::AUDIO, - network_route.packet_overhead); + call_->OnAudioTransportOverheadChanged(network_route.packet_overhead); } bool WebRtcVoiceMediaChannel::MuteStream(uint32_t ssrc, bool muted) { diff --git a/test/scenario/audio_stream.cc b/test/scenario/audio_stream.cc index e1b645e704..bd317d97d8 100644 --- a/test/scenario/audio_stream.cc +++ b/test/scenario/audio_stream.cc @@ -75,8 +75,8 @@ SendAudioStream::SendAudioStream( } send_stream_ = sender_->call_->CreateAudioSendStream(send_config); if (field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")) { - sender->call_->OnTransportOverheadChanged( - MediaType::AUDIO, config.stream.packet_overhead.bytes()); + sender->call_->OnAudioTransportOverheadChanged( + config.stream.packet_overhead.bytes()); } } diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index 2a47b9d18d..054d27e421 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -197,15 +197,6 @@ void VideoSendStream::StopPermanentlyAndGetRtpStates( thread_sync_event_.Wait(rtc::Event::kForever); } -void VideoSendStream::SetTransportOverhead( - size_t transport_overhead_per_packet) { - RTC_DCHECK_RUN_ON(&thread_checker_); - VideoSendStreamImpl* send_stream = send_stream_.get(); - worker_queue_->PostTask([send_stream, transport_overhead_per_packet] { - send_stream->SetTransportOverhead(transport_overhead_per_packet); - }); -} - bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { // Called on a network thread. return send_stream_->DeliverRtcp(packet, length); diff --git a/video/video_send_stream.h b/video/video_send_stream.h index c55f194581..0945203df3 100644 --- a/video/video_send_stream.h +++ b/video/video_send_stream.h @@ -88,8 +88,6 @@ class VideoSendStream : public webrtc::VideoSendStream { void StopPermanentlyAndGetRtpStates(RtpStateMap* rtp_state_map, RtpPayloadStateMap* payload_state_map); - void SetTransportOverhead(size_t transport_overhead_per_packet); - private: friend class test::VideoSendStreamPeer; diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index 721e84fa0a..58caf47774 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -28,19 +28,12 @@ namespace webrtc { namespace internal { namespace { -// Assume an average video stream has around 3 packets per frame (1 mbps / 30 -// fps / 1400B) A sequence number set with size 5500 will be able to store -// packet sequence number for at least last 60 seconds. -static const int kSendSideSeqNumSetMaxSize = 5500; // Max positive size difference to treat allocations as "similar". static constexpr int kMaxVbaSizeDifferencePercent = 10; // Max time we will throttle similar video bitrate allocations. static constexpr int64_t kMaxVbaThrottleTimeMs = 500; -// We don't do MTU discovery, so assume that we have the standard ethernet MTU. -const size_t kPathMTU = 1500; - bool TransportSeqNumExtensionConfigured(const VideoSendStream::Config& config) { const std::vector& extensions = config.rtp.extensions; return std::find_if( @@ -118,26 +111,10 @@ int CalculateMaxPadBitrateBps(const std::vector& streams, return pad_up_to_bitrate_bps; } -uint32_t CalculateOverheadRateBps(int packets_per_second, - size_t overhead_bytes_per_packet, - uint32_t max_overhead_bps) { - uint32_t overhead_bps = - static_cast(8 * overhead_bytes_per_packet * packets_per_second); - return std::min(overhead_bps, max_overhead_bps); -} - -int CalculatePacketRate(uint32_t bitrate_bps, size_t packet_size_bytes) { - size_t packet_size_bits = 8 * packet_size_bytes; - // Ceil for int value of bitrate_bps / packet_size_bits. - return static_cast((bitrate_bps + packet_size_bits - 1) / - packet_size_bits); -} - RtpSenderObservers CreateObservers(CallStats* call_stats, EncoderRtcpFeedback* encoder_feedback, SendStatisticsProxy* stats_proxy, - SendDelayStats* send_delay_stats, - OverheadObserver* overhead_observer) { + SendDelayStats* send_delay_stats) { RtpSenderObservers observers; observers.rtcp_rtt_stats = call_stats; observers.intra_frame_callback = encoder_feedback; @@ -148,7 +125,6 @@ RtpSenderObservers CreateObservers(CallStats* call_stats, observers.rtcp_type_observer = stats_proxy; observers.send_delay_observer = stats_proxy; observers.send_packet_observer = send_delay_stats; - observers.overhead_observer = overhead_observer; return observers; } @@ -243,13 +219,10 @@ VideoSendStreamImpl::VideoSendStreamImpl( std::map suspended_payload_states, VideoEncoderConfig::ContentType content_type, std::unique_ptr fec_controller) - : send_side_bwe_with_overhead_( - webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), - has_alr_probing_(config->periodic_alr_bandwidth_probing || + : has_alr_probing_(config->periodic_alr_bandwidth_probing || GetAlrSettings(content_type)), stats_proxy_(stats_proxy), config_(config), - fec_controller_(std::move(fec_controller)), worker_queue_(worker_queue), check_encoder_activity_task_(nullptr), call_stats_(call_stats), @@ -275,12 +248,10 @@ VideoSendStreamImpl::VideoSendStreamImpl( CreateObservers(call_stats, &encoder_feedback_, stats_proxy_, - send_delay_stats, - this), - event_log)), - weak_ptr_factory_(this), - overhead_bytes_per_packet_(0), - transport_overhead_bytes_per_packet_(0) { + send_delay_stats), + event_log, + std::move(fec_controller))), + weak_ptr_factory_(this) { RTC_DCHECK_RUN_ON(worker_queue_); RTC_LOG(LS_INFO) << "VideoSendStreamInternal: " << config_->ToString(); weak_ptr_ = weak_ptr_factory_.GetWeakPtr(); @@ -332,17 +303,6 @@ VideoSendStreamImpl::VideoSendStreamImpl( transport->EnablePeriodicAlrProbing(true); } - // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, - // so enable that logic if either of those FEC schemes are enabled. - fec_controller_->SetProtectionMethod(rtp_video_sender_->FecEnabled(), - rtp_video_sender_->NackEnabled()); - - fec_controller_->SetProtectionCallback(this); - // Signal congestion controller this object is ready for OnPacket* callbacks. - if (fec_controller_->UseLossVectorMask()) { - transport_->RegisterPacketFeedbackObserver(this); - } - RTC_DCHECK_GE(config_->rtp.payload_type, 0); RTC_DCHECK_LE(config_->rtp.payload_type, 127); @@ -368,9 +328,6 @@ VideoSendStreamImpl::~VideoSendStreamImpl() { RTC_DCHECK(!rtp_video_sender_->IsActive()) << "VideoSendStreamImpl::Stop not called"; RTC_LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString(); - if (fec_controller_->UseLossVectorMask()) { - transport_->DeRegisterPacketFeedbackObserver(this); - } transport_->DestroyRtpVideoSender(rtp_video_sender_); } @@ -582,9 +539,9 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( const size_t num_temporal_layers = streams.back().num_temporal_layers.value_or(1); - fec_controller_->SetEncodingData(streams[0].width, streams[0].height, - num_temporal_layers, - config_->rtp.max_packet_size); + + rtp_video_sender_->SetEncodingData(streams[0].width, streams[0].height, + num_temporal_layers); if (rtp_video_sender_->IsActive()) { // The send stream is started already. Update the allocator with new bitrate @@ -623,8 +580,6 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( check_encoder_activity_task_->UpdateEncoderActivity(); } - fec_controller_->UpdateWithEncodedData(encoded_image._length, - encoded_image._frameType); EncodedImageCallback::Result result = rtp_video_sender_->OnEncodedImage( encoded_image, codec_specific_info, fragmentation); @@ -666,125 +621,16 @@ uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps, RTC_DCHECK(rtp_video_sender_->IsActive()) << "VideoSendStream::Start has not been called."; - // Substract overhead from bitrate. - rtc::CritScope lock(&overhead_bytes_per_packet_crit_); - uint32_t payload_bitrate_bps = bitrate_bps; - if (send_side_bwe_with_overhead_) { - payload_bitrate_bps -= CalculateOverheadRateBps( - CalculatePacketRate(bitrate_bps, - config_->rtp.max_packet_size + - transport_overhead_bytes_per_packet_), - overhead_bytes_per_packet_ + transport_overhead_bytes_per_packet_, - bitrate_bps); - } - - // Get the encoder target rate. It is the estimated network rate - - // protection overhead. - encoder_target_rate_bps_ = fec_controller_->UpdateFecRates( - payload_bitrate_bps, stats_proxy_->GetSendFrameRate(), fraction_loss, - loss_mask_vector_, rtt); - loss_mask_vector_.clear(); - - uint32_t encoder_overhead_rate_bps = - send_side_bwe_with_overhead_ - ? CalculateOverheadRateBps( - CalculatePacketRate(encoder_target_rate_bps_, - config_->rtp.max_packet_size + - transport_overhead_bytes_per_packet_ - - overhead_bytes_per_packet_), - overhead_bytes_per_packet_ + - transport_overhead_bytes_per_packet_, - bitrate_bps - encoder_target_rate_bps_) - : 0; - - // When the field trial "WebRTC-SendSideBwe-WithOverhead" is enabled - // protection_bitrate includes overhead. - uint32_t protection_bitrate = - bitrate_bps - (encoder_target_rate_bps_ + encoder_overhead_rate_bps); - + rtp_video_sender_->OnBitrateUpdated(bitrate_bps, fraction_loss, rtt, + stats_proxy_->GetSendFrameRate()); + encoder_target_rate_bps_ = rtp_video_sender_->GetPayloadBitrateBps(); encoder_target_rate_bps_ = std::min(encoder_max_bitrate_bps_, encoder_target_rate_bps_); video_stream_encoder_->OnBitrateUpdated(encoder_target_rate_bps_, fraction_loss, rtt); stats_proxy_->OnSetEncoderTargetRate(encoder_target_rate_bps_); - return protection_bitrate; + return rtp_video_sender_->GetProtectionBitrateBps(); } -int VideoSendStreamImpl::ProtectionRequest( - const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) { - RTC_DCHECK_RUN_ON(worker_queue_); - rtp_video_sender_->ProtectionRequest(delta_params, key_params, - sent_video_rate_bps, sent_nack_rate_bps, - sent_fec_rate_bps); - return 0; -} - -void VideoSendStreamImpl::OnOverheadChanged(size_t overhead_bytes_per_packet) { - rtc::CritScope lock(&overhead_bytes_per_packet_crit_); - overhead_bytes_per_packet_ = overhead_bytes_per_packet; -} - -void VideoSendStreamImpl::SetTransportOverhead( - size_t transport_overhead_bytes_per_packet) { - if (transport_overhead_bytes_per_packet >= static_cast(kPathMTU)) { - RTC_LOG(LS_ERROR) << "Transport overhead exceeds size of ethernet frame"; - return; - } - - transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet; - - size_t rtp_packet_size = - std::min(config_->rtp.max_packet_size, - kPathMTU - transport_overhead_bytes_per_packet_); - - rtp_video_sender_->SetMaxRtpPacketSize(rtp_packet_size); -} - -void VideoSendStreamImpl::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) { - if (!worker_queue_->IsCurrent()) { - auto ptr = weak_ptr_; - worker_queue_->PostTask([=] { - if (!ptr.get()) - return; - ptr->OnPacketAdded(ssrc, seq_num); - }); - return; - } - const auto ssrcs = config_->rtp.ssrcs; - if (std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()) { - feedback_packet_seq_num_set_.insert(seq_num); - if (feedback_packet_seq_num_set_.size() > kSendSideSeqNumSetMaxSize) { - RTC_LOG(LS_WARNING) << "Feedback packet sequence number set exceed it's " - "max size', will get reset."; - feedback_packet_seq_num_set_.clear(); - } - } -} - -void VideoSendStreamImpl::OnPacketFeedbackVector( - const std::vector& packet_feedback_vector) { - if (!worker_queue_->IsCurrent()) { - auto ptr = weak_ptr_; - worker_queue_->PostTask([=] { - if (!ptr.get()) - return; - ptr->OnPacketFeedbackVector(packet_feedback_vector); - }); - return; - } - // Lost feedbacks are not considered to be lost packets. - for (const PacketFeedback& packet : packet_feedback_vector) { - auto it = feedback_packet_seq_num_set_.find(packet.sequence_number); - if (it != feedback_packet_seq_num_set_.end()) { - const bool lost = packet.arrival_time_ms == PacketFeedback::kNotReceived; - loss_mask_vector_.push_back(lost); - feedback_packet_seq_num_set_.erase(it); - } - } -} } // namespace internal } // namespace webrtc diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h index c87a32e488..eed9f567fb 100644 --- a/video/video_send_stream_impl.h +++ b/video/video_send_stream_impl.h @@ -12,7 +12,6 @@ #include #include -#include #include #include "api/video/video_bitrate_allocator.h" @@ -39,11 +38,8 @@ namespace internal { // An encoder may deliver frames through the EncodedImageCallback on an // arbitrary thread. class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, - public webrtc::OverheadObserver, - public webrtc::VCMProtectionCallback, public VideoStreamEncoderInterface::EncoderSink, - public VideoBitrateAllocationObserver, - public webrtc::PacketFeedbackObserver { + public VideoBitrateAllocationObserver { public: VideoSendStreamImpl( SendStatisticsProxy* stats_proxy, @@ -81,15 +77,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, std::map GetRtpPayloadStates() const; - void SetTransportOverhead(size_t transport_overhead_per_packet); - absl::optional configured_pacing_factor_; - // From PacketFeedbackObserver. - void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override; - void OnPacketFeedbackVector( - const std::vector& packet_feedback_vector) override; - private: class CheckEncoderActivityTask; @@ -99,16 +88,6 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, int64_t rtt, int64_t probing_interval_ms) override; - // Implements webrtc::VCMProtectionCallback. - int ProtectionRequest(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) override; - - // Implements OverheadObserver. - void OnOverheadChanged(size_t overhead_bytes_per_packet) override; - void OnEncoderConfigurationChanged(std::vector streams, int min_transmit_bitrate_bps) override; @@ -135,13 +114,11 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, void SignalEncoderTimedOut(); void SignalEncoderActive(); - const bool send_side_bwe_with_overhead_; const bool has_alr_probing_; SendStatisticsProxy* const stats_proxy_; const VideoSendStream::Config* const config_; - std::unique_ptr fec_controller_; rtc::TaskQueue* const worker_queue_; rtc::CriticalSection encoder_activity_crit_sect_; @@ -175,14 +152,6 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, // invalidated before any other members are destroyed. rtc::WeakPtrFactory weak_ptr_factory_; - rtc::CriticalSection overhead_bytes_per_packet_crit_; - size_t overhead_bytes_per_packet_ - RTC_GUARDED_BY(overhead_bytes_per_packet_crit_); - size_t transport_overhead_bytes_per_packet_; - - std::unordered_set feedback_packet_seq_num_set_; - std::vector loss_mask_vector_; - // Context for the most recent and last sent video bitrate allocation. Used to // throttle sending of similar bitrate allocations. struct VbaSendContext { diff --git a/video/video_send_stream_impl_unittest.cc b/video/video_send_stream_impl_unittest.cc index 03c62442b0..dd036909a6 100644 --- a/video/video_send_stream_impl_unittest.cc +++ b/video/video_send_stream_impl_unittest.cc @@ -45,7 +45,7 @@ std::string GetAlrProbingExperimentString() { AlrExperimentSettings::kScreenshareProbingBweExperimentName) + "/1.0,2875,80,40,-60,3/"; } -class MockPayloadRouter : public RtpVideoSenderInterface { +class MockRtpVideoSender : public RtpVideoSenderInterface { public: MOCK_METHOD1(RegisterProcessThread, void(ProcessThread*)); MOCK_METHOD0(DeRegisterProcessThread, void()); @@ -56,21 +56,18 @@ class MockPayloadRouter : public RtpVideoSenderInterface { MOCK_CONST_METHOD0(GetRtpStates, std::map()); MOCK_CONST_METHOD0(GetRtpPayloadStates, std::map()); - MOCK_CONST_METHOD0(FecEnabled, bool()); - MOCK_CONST_METHOD0(NackEnabled, bool()); MOCK_METHOD2(DeliverRtcp, void(const uint8_t*, size_t)); - MOCK_METHOD5(ProtectionRequest, - void(const FecProtectionParams*, - const FecProtectionParams*, - uint32_t*, - uint32_t*, - uint32_t*)); - MOCK_METHOD1(SetMaxRtpPacketSize, void(size_t)); MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&)); MOCK_METHOD3(OnEncodedImage, EncodedImageCallback::Result(const EncodedImage&, const CodecSpecificInfo*, const RTPFragmentationHeader*)); + MOCK_METHOD1(OnTransportOverheadChanged, void(size_t)); + MOCK_METHOD1(OnOverheadChanged, void(size_t)); + MOCK_METHOD4(OnBitrateUpdated, void(uint32_t, uint8_t, int64_t, int)); + MOCK_CONST_METHOD0(GetPayloadBitrateBps, uint32_t()); + MOCK_CONST_METHOD0(GetProtectionBitrateBps, uint32_t()); + MOCK_METHOD3(SetEncodingData, void(size_t, size_t, size_t)); }; } // namespace @@ -94,14 +91,14 @@ class VideoSendStreamImplTest : public ::testing::Test { EXPECT_CALL(transport_controller_, packet_router()) .WillRepeatedly(Return(&packet_router_)); EXPECT_CALL(transport_controller_, - CreateRtpVideoSender(_, _, _, _, _, _, _, _)) - .WillRepeatedly(Return(&payload_router_)); - EXPECT_CALL(payload_router_, SetActive(_)) + CreateRtpVideoSender(_, _, _, _, _, _, _, _, _)) + .WillRepeatedly(Return(&rtp_video_sender_)); + EXPECT_CALL(rtp_video_sender_, SetActive(_)) .WillRepeatedly(testing::Invoke( - [&](bool active) { payload_router_active_ = active; })); - EXPECT_CALL(payload_router_, IsActive()) + [&](bool active) { rtp_video_sender_active_ = active; })); + EXPECT_CALL(rtp_video_sender_, IsActive()) .WillRepeatedly( - testing::Invoke([&]() { return payload_router_active_; })); + testing::Invoke([&]() { return rtp_video_sender_active_; })); } ~VideoSendStreamImplTest() {} @@ -127,9 +124,9 @@ class VideoSendStreamImplTest : public ::testing::Test { NiceMock transport_controller_; NiceMock bitrate_allocator_; NiceMock video_stream_encoder_; - NiceMock payload_router_; + NiceMock rtp_video_sender_; - bool payload_router_active_ = false; + bool rtp_video_sender_active_ = false; SimulatedClock clock_; RtcEventLogNullImpl event_log_; VideoSendStream::Config config_; @@ -361,19 +358,26 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationWhenEnabled) { alloc.SetBitrate(1, 1, 40000); // Encoder starts out paused, don't forward allocation. - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(0); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)).Times(0); observer->OnBitrateAllocationUpdated(alloc); // Unpause encoder, allocation should be passed through. + const uint32_t kBitrateBps = 100000; + EXPECT_CALL(rtp_video_sender_, GetPayloadBitrateBps()) + .Times(1) + .WillOnce(Return(kBitrateBps)); static_cast(vss_impl.get()) - ->OnBitrateUpdated(100000, 0, 0, 0); - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(1); + ->OnBitrateUpdated(kBitrateBps, 0, 0, 0); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)).Times(1); observer->OnBitrateAllocationUpdated(alloc); // Pause encoder again, and block allocations. + EXPECT_CALL(rtp_video_sender_, GetPayloadBitrateBps()) + .Times(1) + .WillOnce(Return(0)); static_cast(vss_impl.get()) ->OnBitrateUpdated(0, 0, 0, 0); - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(0); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)).Times(0); observer->OnBitrateAllocationUpdated(alloc); vss_impl->Stop(); @@ -387,8 +391,12 @@ TEST_F(VideoSendStreamImplTest, ThrottlesVideoBitrateAllocationWhenTooSimilar) { VideoEncoderConfig::ContentType::kScreen); vss_impl->Start(); // Unpause encoder, to allows allocations to be passed through. + const uint32_t kBitrateBps = 100000; + EXPECT_CALL(rtp_video_sender_, GetPayloadBitrateBps()) + .Times(1) + .WillOnce(Return(kBitrateBps)); static_cast(vss_impl.get()) - ->OnBitrateUpdated(100000, 0, 0, 0); + ->OnBitrateUpdated(kBitrateBps, 0, 0, 0); VideoBitrateAllocationObserver* const observer = static_cast(vss_impl.get()); @@ -400,7 +408,7 @@ TEST_F(VideoSendStreamImplTest, ThrottlesVideoBitrateAllocationWhenTooSimilar) { alloc.SetBitrate(1, 1, 40000); // Initial value. - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(1); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)).Times(1); observer->OnBitrateAllocationUpdated(alloc); VideoBitrateAllocation updated_alloc = alloc; @@ -410,19 +418,19 @@ TEST_F(VideoSendStreamImplTest, ThrottlesVideoBitrateAllocationWhenTooSimilar) { // Too small increase, don't forward. updated_alloc.SetBitrate(0, 0, base_layer_min_update_bitrate_bps - 1); - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(_)).Times(0); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(_)).Times(0); observer->OnBitrateAllocationUpdated(updated_alloc); // Large enough increase, do forward. updated_alloc.SetBitrate(0, 0, base_layer_min_update_bitrate_bps); - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(updated_alloc)) + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(updated_alloc)) .Times(1); observer->OnBitrateAllocationUpdated(updated_alloc); // This is now a decrease compared to last forward allocation, forward // immediately. updated_alloc.SetBitrate(0, 0, base_layer_min_update_bitrate_bps - 1); - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(updated_alloc)) + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(updated_alloc)) .Times(1); observer->OnBitrateAllocationUpdated(updated_alloc); @@ -437,8 +445,12 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationOnLayerChange) { VideoEncoderConfig::ContentType::kScreen); vss_impl->Start(); // Unpause encoder, to allows allocations to be passed through. + const uint32_t kBitrateBps = 100000; + EXPECT_CALL(rtp_video_sender_, GetPayloadBitrateBps()) + .Times(1) + .WillOnce(Return(kBitrateBps)); static_cast(vss_impl.get()) - ->OnBitrateUpdated(100000, 0, 0, 0); + ->OnBitrateUpdated(kBitrateBps, 0, 0, 0); VideoBitrateAllocationObserver* const observer = static_cast(vss_impl.get()); @@ -450,7 +462,7 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationOnLayerChange) { alloc.SetBitrate(1, 1, 40000); // Initial value. - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(1); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)).Times(1); observer->OnBitrateAllocationUpdated(alloc); // Move some bitrate from one layer to a new one, but keep sum the same. @@ -459,7 +471,7 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationOnLayerChange) { updated_alloc.SetBitrate(2, 0, 10000); updated_alloc.SetBitrate(1, 1, alloc.GetBitrate(1, 1) - 10000); EXPECT_EQ(alloc.get_sum_bps(), updated_alloc.get_sum_bps()); - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(updated_alloc)) + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(updated_alloc)) .Times(1); observer->OnBitrateAllocationUpdated(updated_alloc); @@ -476,9 +488,13 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationAfterTimeout) { kDefaultInitialBitrateBps, kDefaultBitratePriority, VideoEncoderConfig::ContentType::kScreen); vss_impl->Start(); + const uint32_t kBitrateBps = 100000; // Unpause encoder, to allows allocations to be passed through. + EXPECT_CALL(rtp_video_sender_, GetPayloadBitrateBps()) + .Times(1) + .WillRepeatedly(Return(kBitrateBps)); static_cast(vss_impl.get()) - ->OnBitrateUpdated(100000, 0, 0, 0); + ->OnBitrateUpdated(kBitrateBps, 0, 0, 0); VideoBitrateAllocationObserver* const observer = static_cast(vss_impl.get()); @@ -491,7 +507,7 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationAfterTimeout) { EncodedImage encoded_image; CodecSpecificInfo codec_specific; - EXPECT_CALL(payload_router_, OnEncodedImage(_, _, _)) + EXPECT_CALL(rtp_video_sender_, OnEncodedImage(_, _, _)) .WillRepeatedly(Return( EncodedImageCallback::Result(EncodedImageCallback::Result::OK))); @@ -500,13 +516,15 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationAfterTimeout) { { // Initial value. - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(1); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)) + .Times(1); observer->OnBitrateAllocationUpdated(alloc); } { // Sending same allocation again, this one should be throttled. - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(0); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)) + .Times(0); observer->OnBitrateAllocationUpdated(alloc); } @@ -514,19 +532,22 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationAfterTimeout) { { // Sending similar allocation again after timeout, should forward. - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(1); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)) + .Times(1); observer->OnBitrateAllocationUpdated(alloc); } { // Sending similar allocation again without timeout, throttle. - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(0); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)) + .Times(0); observer->OnBitrateAllocationUpdated(alloc); } { // Send encoded image, should be a noop. - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(0); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)) + .Times(0); static_cast(vss_impl.get()) ->OnEncodedImage(encoded_image, &codec_specific, nullptr); } @@ -535,7 +556,8 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationAfterTimeout) { // Advance time and send encoded image, this should wake up and send // cached bitrate allocation. fake_clock.AdvanceTimeMicros(kMaxVbaThrottleTimeMs * 1000); - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(1); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)) + .Times(1); static_cast(vss_impl.get()) ->OnEncodedImage(encoded_image, &codec_specific, nullptr); } @@ -544,7 +566,8 @@ TEST_F(VideoSendStreamImplTest, ForwardsVideoBitrateAllocationAfterTimeout) { // Advance time and send encoded image, there should be no cached // allocation to send. fake_clock.AdvanceTimeMicros(kMaxVbaThrottleTimeMs * 1000); - EXPECT_CALL(payload_router_, OnBitrateAllocationUpdated(alloc)).Times(0); + EXPECT_CALL(rtp_video_sender_, OnBitrateAllocationUpdated(alloc)) + .Times(0); static_cast(vss_impl.get()) ->OnEncodedImage(encoded_image, &codec_specific, nullptr); } diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index a4d5e195e4..09ccf1c498 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -1734,8 +1734,8 @@ TEST_P(VideoSendStreamTest, ChangingTransportOverhead) { void PerformTest() override { task_queue_->SendTask([this]() { transport_overhead_ = 100; - call_->OnTransportOverheadChanged(webrtc::MediaType::VIDEO, - transport_overhead_); + call_->GetTransportControllerSend()->OnTransportOverheadChanged( + transport_overhead_); }); EXPECT_TRUE(Wait()); @@ -1747,8 +1747,8 @@ TEST_P(VideoSendStreamTest, ChangingTransportOverhead) { task_queue_->SendTask([this]() { transport_overhead_ = 500; - call_->OnTransportOverheadChanged(webrtc::MediaType::VIDEO, - transport_overhead_); + call_->GetTransportControllerSend()->OnTransportOverheadChanged( + transport_overhead_); }); EXPECT_TRUE(Wait()); @@ -3725,7 +3725,7 @@ TEST_P(VideoSendStreamTest, RemoveOverheadFromBandwidth) { task_queue_->SendTask([this, &bitrate_config]() { call_->GetTransportControllerSend()->SetSdpBitrateParameters( bitrate_config); - call_->OnTransportOverheadChanged(webrtc::MediaType::VIDEO, 40); + call_->GetTransportControllerSend()->OnTransportOverheadChanged(40); }); // At a bitrate of 60kbps with a packet size of 1200B video and an