diff --git a/api/video/video_stream_encoder_interface.h b/api/video/video_stream_encoder_interface.h index fad3bbdfcb..44dc6f4972 100644 --- a/api/video/video_stream_encoder_interface.h +++ b/api/video/video_stream_encoder_interface.h @@ -47,7 +47,7 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { int min_transmit_bitrate_bps) = 0; }; - virtual ~VideoStreamEncoderInterface() = default; + ~VideoStreamEncoderInterface() override = default; // Sets the source that will provide video frames to the VideoStreamEncoder's // OnFrame method. |degradation_preference| control whether or not resolution diff --git a/call/BUILD.gn b/call/BUILD.gn index 821a164a01..7204dcc140 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -62,6 +62,8 @@ rtc_source_set("rtp_interfaces") { "../api:array_view", "../api:libjingle_peerconnection_api", "../api/transport:bitrate_settings", + "../logging:rtc_event_log_api", + "../modules/rtp_rtcp:rtp_rtcp_format", "../rtc_base:rtc_base_approved", "//third_party/abseil-cpp/absl/types:optional", ] @@ -104,13 +106,16 @@ rtc_source_set("rtp_sender") { "rtp_payload_params.h", "rtp_transport_controller_send.cc", "rtp_transport_controller_send.h", + "video_rtp_sender_interface.h", ] deps = [ ":bitrate_configurator", ":rtp_interfaces", "..:webrtc_common", + "../api:transport_api", "../api/transport:network_control", "../api/video_codecs:video_codecs_api", + "../logging:rtc_event_log_api", "../modules/congestion_controller", "../modules/congestion_controller/rtp:congestion_controller", "../modules/pacing", @@ -120,6 +125,7 @@ rtc_source_set("rtp_sender") { "../modules/utility", "../modules/video_coding:video_codec_interface", "../rtc_base:checks", + "../rtc_base:rate_limiter", "../rtc_base:rtc_base", "../rtc_base:rtc_base_approved", "../rtc_base:rtc_task_queue", @@ -318,6 +324,7 @@ if (rtc_include_tests) { "../modules/utility:mock_process_thread", "../modules/video_coding:video_codec_interface", "../rtc_base:checks", + "../rtc_base:rate_limiter", "../rtc_base:rtc_base_approved", "../system_wrappers", "../test:audio_codec_mocks", @@ -326,6 +333,7 @@ if (rtc_include_tests) { "../test:test_common", "../test:test_support", "../test:video_test_common", + "../video:video", "//testing/gtest", "//third_party/abseil-cpp/absl/memory", ] diff --git a/call/bitrate_allocator.h b/call/bitrate_allocator.h index 36d05de607..c29ea5e089 100644 --- a/call/bitrate_allocator.h +++ b/call/bitrate_allocator.h @@ -98,7 +98,7 @@ class BitrateAllocator : public BitrateAllocatorInterface { }; explicit BitrateAllocator(LimitObserver* limit_observer); - ~BitrateAllocator(); + ~BitrateAllocator() override; // Allocate target_bitrate across the registered BitrateAllocatorObservers. void OnNetworkChanged(uint32_t target_bitrate_bps, diff --git a/call/call.cc b/call/call.cc index 8b4da25302..4f27146e50 100644 --- a/call/call.cc +++ b/call/call.cc @@ -51,7 +51,6 @@ #include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_minmax.h" -#include "rtc_base/rate_limiter.h" #include "rtc_base/sequenced_task_checker.h" #include "rtc_base/strings/string_builder.h" #include "rtc_base/synchronization/rw_lock_wrapper.h" @@ -70,8 +69,6 @@ namespace webrtc { namespace { -static const int64_t kRetransmitWindowSizeMs = 500; - // TODO(nisse): This really begs for a shared context struct. bool UseSendSideBwe(const std::vector& extensions, bool transport_cc) { @@ -361,7 +358,6 @@ class Call final : public webrtc::Call, RTC_GUARDED_BY(&bitrate_crit_); AvgCounter pacer_bitrate_kbps_counter_ RTC_GUARDED_BY(&bitrate_crit_); - RateLimiter retransmission_rate_limiter_; ReceiveSideCongestionController receive_side_cc_; const std::unique_ptr receive_time_calculator_; @@ -442,7 +438,6 @@ Call::Call(const Call::Config& config, configured_max_padding_bitrate_bps_(0), estimated_send_bitrate_kbps_counter_(clock_, nullptr, true), pacer_bitrate_kbps_counter_(clock_, nullptr, true), - retransmission_rate_limiter_(clock_, kRetransmitWindowSizeMs), receive_side_cc_(clock_, transport_send->packet_router()), receive_time_calculator_(ReceiveTimeCalculator::CreateFromFieldTrial()), video_send_delay_stats_(new SendDelayStats(clock_)), @@ -732,8 +727,7 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream( transport_send_ptr_, bitrate_allocator_.get(), video_send_delay_stats_.get(), event_log_, std::move(config), std::move(encoder_config), suspended_video_send_ssrcs_, - suspended_video_payload_states_, std::move(fec_controller), - &retransmission_rate_limiter_); + suspended_video_payload_states_, std::move(fec_controller)); { WriteLockScoped write_lock(*send_crit_); @@ -743,7 +737,6 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream( } video_send_streams_.insert(send_stream); } - send_stream->SignalNetworkState(video_network_state_); UpdateAggregateNetworkState(); return send_stream; @@ -991,9 +984,6 @@ void Call::SignalChannelNetworkState(MediaType media, NetworkState state) { for (auto& kv : audio_send_ssrcs_) { kv.second->SignalNetworkState(audio_network_state_); } - for (auto& kv : video_send_ssrcs_) { - kv.second->SignalNetworkState(video_network_state_); - } } { ReadLockScoped read_lock(*receive_crit_); @@ -1081,7 +1071,6 @@ void Call::OnTargetTransferRate(TargetTransferRate msg) { rtc::CritScope cs(&last_bandwidth_bps_crit_); last_bandwidth_bps_ = bandwidth_bps; } - retransmission_rate_limiter_.SetMaxRate(bandwidth_bps); // For controlling the rate of feedback messages. receive_side_cc_.OnBitrateChanged(target_bitrate_bps); bitrate_allocator_->OnNetworkChanged(target_bitrate_bps, fraction_loss, diff --git a/call/payload_router.cc b/call/payload_router.cc index cca4bd3418..4e7d13eef4 100644 --- a/call/payload_router.cc +++ b/call/payload_router.cc @@ -10,14 +10,90 @@ #include "call/payload_router.h" +#include +#include +#include + +#include "call/rtp_transport_controller_send_interface.h" +#include "modules/pacing/packet_router.h" #include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtp_sender.h" +#include "modules/utility/include/process_thread.h" #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/checks.h" +#include "rtc_base/location.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { +static const int kMinSendSidePacketHistorySize = 600; + +std::vector> CreateRtpRtcpModules( + const std::vector& ssrcs, + const std::vector& protected_media_ssrcs, + const RtcpConfig& rtcp_config, + Transport* send_transport, + RtcpIntraFrameObserver* intra_frame_callback, + RtcpBandwidthObserver* bandwidth_callback, + RtpTransportControllerSendInterface* transport, + RtcpRttStats* rtt_stats, + FlexfecSender* flexfec_sender, + BitrateStatisticsObserver* bitrate_observer, + FrameCountObserver* frame_count_observer, + RtcpPacketTypeCounterObserver* rtcp_type_observer, + SendSideDelayObserver* send_delay_observer, + SendPacketObserver* send_packet_observer, + RtcEventLog* event_log, + RateLimiter* retransmission_rate_limiter, + OverheadObserver* overhead_observer, + RtpKeepAliveConfig keepalive_config) { + RTC_DCHECK_GT(ssrcs.size(), 0); + RtpRtcp::Configuration configuration; + configuration.audio = false; + configuration.receiver_only = false; + configuration.outgoing_transport = send_transport; + configuration.intra_frame_callback = intra_frame_callback; + configuration.bandwidth_callback = bandwidth_callback; + configuration.transport_feedback_callback = + transport->transport_feedback_observer(); + configuration.rtt_stats = rtt_stats; + configuration.rtcp_packet_type_counter_observer = rtcp_type_observer; + configuration.paced_sender = transport->packet_sender(); + configuration.transport_sequence_number_allocator = + transport->packet_router(); + configuration.send_bitrate_observer = bitrate_observer; + configuration.send_frame_count_observer = frame_count_observer; + configuration.send_side_delay_observer = send_delay_observer; + configuration.send_packet_observer = send_packet_observer; + configuration.event_log = event_log; + configuration.retransmission_rate_limiter = retransmission_rate_limiter; + configuration.overhead_observer = overhead_observer; + configuration.keepalive_config = keepalive_config; + configuration.rtcp_interval_config.video_interval_ms = + rtcp_config.video_report_interval_ms; + configuration.rtcp_interval_config.audio_interval_ms = + rtcp_config.audio_report_interval_ms; + std::vector> modules; + const std::vector& flexfec_protected_ssrcs = protected_media_ssrcs; + for (uint32_t ssrc : ssrcs) { + bool enable_flexfec = flexfec_sender != nullptr && + std::find(flexfec_protected_ssrcs.begin(), + flexfec_protected_ssrcs.end(), + ssrc) != flexfec_protected_ssrcs.end(); + configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr; + std::unique_ptr rtp_rtcp = + std::unique_ptr(RtpRtcp::CreateRtpRtcp(configuration)); + rtp_rtcp->SetSendingStatus(false); + rtp_rtcp->SetSendingMediaStatus(false); + rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); + modules.push_back(std::move(rtp_rtcp)); + } + return modules; +} + absl::optional GetSimulcastIdx(const CodecSpecificInfo* info) { if (!info) return absl::nullopt; @@ -33,14 +109,95 @@ absl::optional GetSimulcastIdx(const CodecSpecificInfo* info) { return absl::nullopt; } } +bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) { + const VideoCodecType codecType = PayloadStringToCodecType(payload_name); + if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) { + return true; + } + return false; +} + +// TODO(brandtr): Update this function when we support multistream protection. +std::unique_ptr MaybeCreateFlexfecSender( + const RtpConfig& rtp, + const std::map& suspended_ssrcs) { + if (rtp.flexfec.payload_type < 0) { + return nullptr; + } + RTC_DCHECK_GE(rtp.flexfec.payload_type, 0); + RTC_DCHECK_LE(rtp.flexfec.payload_type, 127); + if (rtp.flexfec.ssrc == 0) { + RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " + "Therefore disabling FlexFEC."; + return nullptr; + } + if (rtp.flexfec.protected_media_ssrcs.empty()) { + RTC_LOG(LS_WARNING) + << "FlexFEC is enabled, but no protected media SSRC given. " + "Therefore disabling FlexFEC."; + return nullptr; + } + + if (rtp.flexfec.protected_media_ssrcs.size() > 1) { + RTC_LOG(LS_WARNING) + << "The supplied FlexfecConfig contained multiple protected " + "media streams, but our implementation currently only " + "supports protecting a single media stream. " + "To avoid confusion, disabling FlexFEC completely."; + return nullptr; + } + + const RtpState* rtp_state = nullptr; + auto it = suspended_ssrcs.find(rtp.flexfec.ssrc); + if (it != suspended_ssrcs.end()) { + rtp_state = &it->second; + } + + RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size()); + return absl::make_unique( + rtp.flexfec.payload_type, rtp.flexfec.ssrc, + rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions, + RTPSender::FecExtensionSizes(), rtp_state, Clock::GetRealTimeClock()); +} } // namespace -PayloadRouter::PayloadRouter(const std::vector& rtp_modules, - const std::vector& ssrcs, - int payload_type, - const std::map& states) - : active_(false), rtp_modules_(rtp_modules), payload_type_(payload_type) { - RTC_DCHECK_EQ(ssrcs.size(), rtp_modules.size()); +PayloadRouter::PayloadRouter(const std::vector& ssrcs, + std::map suspended_ssrcs, + const std::map& states, + const RtpConfig& rtp_config, + const RtcpConfig& rtcp_config, + Transport* send_transport, + const RtpSenderObservers& observers, + RtpTransportControllerSendInterface* transport, + RtcEventLog* event_log, + RateLimiter* retransmission_limiter) + : active_(false), + module_process_thread_(nullptr), + suspended_ssrcs_(std::move(suspended_ssrcs)), + flexfec_sender_(MaybeCreateFlexfecSender(rtp_config, suspended_ssrcs_)), + rtp_modules_( + CreateRtpRtcpModules(ssrcs, + rtp_config.flexfec.protected_media_ssrcs, + rtcp_config, + send_transport, + observers.intra_frame_callback, + transport->GetBandwidthObserver(), + transport, + observers.rtcp_rtt_stats, + flexfec_sender_.get(), + observers.bitrate_observer, + observers.frame_count_observer, + observers.rtcp_type_observer, + observers.send_delay_observer, + observers.send_packet_observer, + event_log, + retransmission_limiter, + observers.overhead_observer, + transport->keepalive_config())), + rtp_config_(rtp_config), + transport_(transport) { + RTC_DCHECK_EQ(ssrcs.size(), rtp_modules_.size()); + module_process_thread_checker_.DetachFromThread(); // SSRCs are assumed to be sorted in the same order as |rtp_modules|. for (uint32_t ssrc : ssrcs) { // Restore state if it previously existed. @@ -51,9 +208,73 @@ PayloadRouter::PayloadRouter(const std::vector& rtp_modules, } params_.push_back(RtpPayloadParams(ssrc, state)); } + + // RTP/RTCP initialization. + + // We add the highest spatial layer first to ensure it'll be prioritized + // when sending padding, with the hope that the packet rate will be smaller, + // and that it's more important to protect than the lower layers. + for (auto& rtp_rtcp : rtp_modules_) { + constexpr bool remb_candidate = true; + transport->packet_router()->AddSendRtpModule(rtp_rtcp.get(), + remb_candidate); + } + + for (size_t i = 0; i < rtp_config_.extensions.size(); ++i) { + const std::string& extension = rtp_config_.extensions[i].uri; + int id = rtp_config_.extensions[i].id; + // One-byte-extension local identifiers are in the range 1-14 inclusive. + RTC_DCHECK_GE(id, 1); + RTC_DCHECK_LE(id, 14); + RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension)); + for (auto& rtp_rtcp : rtp_modules_) { + RTC_CHECK_EQ(0, rtp_rtcp->RegisterSendRtpHeaderExtension( + StringToRtpExtensionType(extension), id)); + } + } + + ConfigureProtection(rtp_config); + ConfigureSsrcs(rtp_config); + + if (!rtp_config.mid.empty()) { + for (auto& rtp_rtcp : rtp_modules_) { + rtp_rtcp->SetMid(rtp_config.mid); + } + } + + // TODO(pbos): Should we set CNAME on all RTP modules? + rtp_modules_.front()->SetCNAME(rtp_config.c_name.c_str()); + + for (auto& rtp_rtcp : rtp_modules_) { + rtp_rtcp->RegisterRtcpStatisticsCallback(observers.rtcp_stats); + rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(observers.rtp_stats); + rtp_rtcp->SetMaxRtpPacketSize(rtp_config.max_packet_size); + rtp_rtcp->RegisterVideoSendPayload(rtp_config.payload_type, + rtp_config.payload_name.c_str()); + } } -PayloadRouter::~PayloadRouter() {} +PayloadRouter::~PayloadRouter() { + for (auto& rtp_rtcp : rtp_modules_) { + transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp.get()); + } +} + +void PayloadRouter::RegisterProcessThread( + ProcessThread* module_process_thread) { + RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + RTC_DCHECK(!module_process_thread_); + module_process_thread_ = module_process_thread; + + for (auto& rtp_rtcp : rtp_modules_) + module_process_thread_->RegisterModule(rtp_rtcp.get(), RTC_FROM_HERE); +} + +void PayloadRouter::DeRegisterProcessThread() { + RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + for (auto& rtp_rtcp : rtp_modules_) + module_process_thread_->DeRegisterModule(rtp_rtcp.get()); +} void PayloadRouter::SetActive(bool active) { rtc::CritScope lock(&crit_); @@ -83,15 +304,6 @@ bool PayloadRouter::IsActive() { return active_ && !rtp_modules_.empty(); } -std::map PayloadRouter::GetRtpPayloadStates() const { - rtc::CritScope lock(&crit_); - std::map payload_states; - for (const auto& param : params_) { - payload_states[param.ssrc()] = param.state(); - } - return payload_states; -} - EncodedImageCallback::Result PayloadRouter::OnEncodedImage( const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info, @@ -112,9 +324,10 @@ EncodedImageCallback::Result PayloadRouter::OnEncodedImage( return Result(Result::ERROR_SEND_FAILED); } bool send_result = rtp_modules_[stream_index]->SendOutgoingData( - encoded_image._frameType, payload_type_, encoded_image._timeStamp, - encoded_image.capture_time_ms_, encoded_image._buffer, - encoded_image._length, fragmentation, &rtp_video_header, &frame_id); + encoded_image._frameType, rtp_config_.payload_type, + encoded_image._timeStamp, encoded_image.capture_time_ms_, + encoded_image._buffer, encoded_image._length, fragmentation, + &rtp_video_header, &frame_id); if (!send_result) return Result(Result::ERROR_SEND_FAILED); @@ -144,4 +357,189 @@ void PayloadRouter::OnBitrateAllocationUpdated( } } +void PayloadRouter::ConfigureProtection(const RtpConfig& rtp_config) { + // Consistency of FlexFEC parameters is checked in MaybeCreateFlexfecSender. + const bool flexfec_enabled = (flexfec_sender_ != nullptr); + + // Consistency of NACK and RED+ULPFEC parameters is checked in this function. + const bool nack_enabled = rtp_config.nack.rtp_history_ms > 0; + int red_payload_type = rtp_config.ulpfec.red_payload_type; + int ulpfec_payload_type = rtp_config.ulpfec.ulpfec_payload_type; + + // Shorthands. + auto IsRedEnabled = [&]() { return red_payload_type >= 0; }; + auto IsUlpfecEnabled = [&]() { return ulpfec_payload_type >= 0; }; + auto DisableRedAndUlpfec = [&]() { + red_payload_type = -1; + ulpfec_payload_type = -1; + }; + + if (webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment")) { + RTC_LOG(LS_INFO) << "Experiment to disable sending ULPFEC is enabled."; + DisableRedAndUlpfec(); + } + + // If enabled, FlexFEC takes priority over RED+ULPFEC. + if (flexfec_enabled) { + if (IsUlpfecEnabled()) { + RTC_LOG(LS_INFO) + << "Both FlexFEC and ULPFEC are configured. Disabling ULPFEC."; + } + DisableRedAndUlpfec(); + } + + // Payload types without picture ID cannot determine that a stream is complete + // without retransmitting FEC, so using ULPFEC + NACK for H.264 (for instance) + // is a waste of bandwidth since FEC packets still have to be transmitted. + // Note that this is not the case with FlexFEC. + if (nack_enabled && IsUlpfecEnabled() && + !PayloadTypeSupportsSkippingFecPackets(rtp_config.payload_name)) { + RTC_LOG(LS_WARNING) + << "Transmitting payload type without picture ID using " + "NACK+ULPFEC is a waste of bandwidth since ULPFEC packets " + "also have to be retransmitted. Disabling ULPFEC."; + DisableRedAndUlpfec(); + } + + // Verify payload types. + if (IsUlpfecEnabled() ^ IsRedEnabled()) { + RTC_LOG(LS_WARNING) + << "Only RED or only ULPFEC enabled, but not both. Disabling both."; + DisableRedAndUlpfec(); + } + + for (auto& rtp_rtcp : rtp_modules_) { + // Set NACK. + rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize); + // Set RED/ULPFEC information. + rtp_rtcp->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); + } +} + +bool PayloadRouter::FecEnabled() const { + const bool flexfec_enabled = (flexfec_sender_ != nullptr); + int ulpfec_payload_type = rtp_config_.ulpfec.ulpfec_payload_type; + return flexfec_enabled || ulpfec_payload_type >= 0; +} + +bool PayloadRouter::NackEnabled() const { + const bool nack_enabled = rtp_config_.nack.rtp_history_ms > 0; + return nack_enabled; +} + +void PayloadRouter::DeliverRtcp(const uint8_t* packet, size_t length) { + // Runs on a network thread. + for (auto& rtp_rtcp : rtp_modules_) + rtp_rtcp->IncomingRtcpPacket(packet, length); +} + +void PayloadRouter::ProtectionRequest(const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) { + *sent_video_rate_bps = 0; + *sent_nack_rate_bps = 0; + *sent_fec_rate_bps = 0; + for (auto& rtp_rtcp : rtp_modules_) { + uint32_t not_used = 0; + uint32_t module_video_rate = 0; + uint32_t module_fec_rate = 0; + uint32_t module_nack_rate = 0; + rtp_rtcp->SetFecParameters(*delta_params, *key_params); + rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate, + &module_nack_rate); + *sent_video_rate_bps += module_video_rate; + *sent_nack_rate_bps += module_nack_rate; + *sent_fec_rate_bps += module_fec_rate; + } +} + +void PayloadRouter::SetMaxRtpPacketSize(size_t max_rtp_packet_size) { + for (auto& rtp_rtcp : rtp_modules_) { + rtp_rtcp->SetMaxRtpPacketSize(max_rtp_packet_size); + } +} + +void PayloadRouter::ConfigureSsrcs(const RtpConfig& rtp_config) { + // Configure regular SSRCs. + for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) { + uint32_t ssrc = rtp_config.ssrcs[i]; + RtpRtcp* const rtp_rtcp = rtp_modules_[i].get(); + rtp_rtcp->SetSSRC(ssrc); + + // Restore RTP state if previous existed. + auto it = suspended_ssrcs_.find(ssrc); + if (it != suspended_ssrcs_.end()) + rtp_rtcp->SetRtpState(it->second); + } + + // Set up RTX if available. + if (rtp_config.rtx.ssrcs.empty()) + return; + + // Configure RTX SSRCs. + RTC_DCHECK_EQ(rtp_config.rtx.ssrcs.size(), rtp_config.ssrcs.size()); + for (size_t i = 0; i < rtp_config.rtx.ssrcs.size(); ++i) { + uint32_t ssrc = rtp_config.rtx.ssrcs[i]; + RtpRtcp* const rtp_rtcp = rtp_modules_[i].get(); + rtp_rtcp->SetRtxSsrc(ssrc); + auto it = suspended_ssrcs_.find(ssrc); + if (it != suspended_ssrcs_.end()) + rtp_rtcp->SetRtxState(it->second); + } + + // Configure RTX payload types. + RTC_DCHECK_GE(rtp_config.rtx.payload_type, 0); + for (auto& rtp_rtcp : rtp_modules_) { + rtp_rtcp->SetRtxSendPayloadType(rtp_config.rtx.payload_type, + rtp_config.payload_type); + rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads); + } + if (rtp_config.ulpfec.red_payload_type != -1 && + rtp_config.ulpfec.red_rtx_payload_type != -1) { + for (auto& rtp_rtcp : rtp_modules_) { + rtp_rtcp->SetRtxSendPayloadType(rtp_config.ulpfec.red_rtx_payload_type, + rtp_config.ulpfec.red_payload_type); + } + } +} + +void PayloadRouter::OnNetworkAvailability(bool network_available) { + for (auto& rtp_rtcp : rtp_modules_) { + rtp_rtcp->SetRTCPStatus(network_available ? rtp_config_.rtcp_mode + : RtcpMode::kOff); + } +} + +std::map PayloadRouter::GetRtpStates() const { + std::map rtp_states; + + for (size_t i = 0; i < rtp_config_.ssrcs.size(); ++i) { + uint32_t ssrc = rtp_config_.ssrcs[i]; + RTC_DCHECK_EQ(ssrc, rtp_modules_[i]->SSRC()); + rtp_states[ssrc] = rtp_modules_[i]->GetRtpState(); + } + + for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) { + uint32_t ssrc = rtp_config_.rtx.ssrcs[i]; + rtp_states[ssrc] = rtp_modules_[i]->GetRtxState(); + } + + if (flexfec_sender_) { + uint32_t ssrc = rtp_config_.flexfec.ssrc; + rtp_states[ssrc] = flexfec_sender_->GetRtpState(); + } + + return rtp_states; +} + +std::map PayloadRouter::GetRtpPayloadStates() const { + rtc::CritScope lock(&crit_); + std::map payload_states; + for (const auto& param : params_) { + payload_states[param.ssrc()] = param.state(); + } + return payload_states; +} } // namespace webrtc diff --git a/call/payload_router.h b/call/payload_router.h index c62bc75f97..cb43f27e29 100644 --- a/call/payload_router.h +++ b/call/payload_router.h @@ -12,41 +12,83 @@ #define CALL_PAYLOAD_ROUTER_H_ #include +#include #include +#include "api/call/transport.h" #include "api/video_codecs/video_encoder.h" +#include "call/rtp_config.h" #include "call/rtp_payload_params.h" +#include "call/rtp_transport_controller_send_interface.h" +#include "call/video_rtp_sender_interface.h" #include "common_types.h" // NOLINT(build/include) +#include "logging/rtc_event_log/rtc_event_log.h" +#include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" +#include "modules/utility/include/process_thread.h" #include "rtc_base/constructormagic.h" #include "rtc_base/criticalsection.h" +#include "rtc_base/rate_limiter.h" #include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" namespace webrtc { class RTPFragmentationHeader; class RtpRtcp; +class RtpTransportControllerSendInterface; // PayloadRouter routes outgoing data to the correct sending RTP module, based // on the simulcast layer in RTPVideoHeader. -class PayloadRouter : public EncodedImageCallback { +class PayloadRouter : public VideoRtpSenderInterface { public: // Rtp modules are assumed to be sorted in simulcast index order. - PayloadRouter(const std::vector& rtp_modules, - const std::vector& ssrcs, - int payload_type, - const std::map& states); + PayloadRouter( + const std::vector& ssrcs, + std::map suspended_ssrcs, + const std::map& states, + const RtpConfig& rtp_config, + const RtcpConfig& rtcp_config, + Transport* send_transport, + const RtpSenderObservers& observers, + RtpTransportControllerSendInterface* transport, + RtcEventLog* event_log, + RateLimiter* retransmission_limiter); // move inside RtpTransport ~PayloadRouter() override; + // RegisterProcessThread register |module_process_thread| with those objects + // that use it. Registration has to happen on the thread were + // |module_process_thread| was created (libjingle's worker thread). + // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, + // maybe |worker_queue|. + void RegisterProcessThread(ProcessThread* module_process_thread) override; + void DeRegisterProcessThread() override; + // PayloadRouter will only route packets if being active, all packets will be // dropped otherwise. - void SetActive(bool active); + void SetActive(bool active) override; // Sets the sending status of the rtp modules and appropriately sets the // payload router to active if any rtp modules are active. - void SetActiveModules(const std::vector active_modules); - bool IsActive(); + void SetActiveModules(const std::vector active_modules) override; + bool IsActive() override; - std::map GetRtpPayloadStates() const; + void OnNetworkAvailability(bool network_available) override; + std::map GetRtpStates() const override; + std::map GetRtpPayloadStates() const override; + + bool FecEnabled() const override; + + bool NackEnabled() const override; + + void DeliverRtcp(const uint8_t* packet, size_t length) override; + + void ProtectionRequest(const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) override; + + void SetMaxRtpPacketSize(size_t max_rtp_packet_size) override; // Implements EncodedImageCallback. // Returns 0 if the packet was routed / sent, -1 otherwise. @@ -55,17 +97,26 @@ class PayloadRouter : public EncodedImageCallback { const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) override; - void OnBitrateAllocationUpdated(const VideoBitrateAllocation& bitrate); + void OnBitrateAllocationUpdated( + const VideoBitrateAllocation& bitrate) override; private: void UpdateModuleSendingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + void ConfigureProtection(const RtpConfig& rtp_config); + void ConfigureSsrcs(const RtpConfig& rtp_config); rtc::CriticalSection crit_; bool active_ RTC_GUARDED_BY(crit_); + ProcessThread* module_process_thread_; + rtc::ThreadChecker module_process_thread_checker_; + std::map suspended_ssrcs_; + + std::unique_ptr flexfec_sender_; // Rtp modules are assumed to be sorted in simulcast index order. Not owned. - const std::vector rtp_modules_; - const int payload_type_; + const std::vector> rtp_modules_; + const RtpConfig rtp_config_; + RtpTransportControllerSendInterface* const transport_; std::vector params_ RTC_GUARDED_BY(crit_); diff --git a/call/payload_router_unittest.cc b/call/payload_router_unittest.cc index 9c3e1deac2..c02bad9b2d 100644 --- a/call/payload_router_unittest.cc +++ b/call/payload_router_unittest.cc @@ -12,12 +12,16 @@ #include #include "call/payload_router.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" -#include "modules/rtp_rtcp/mocks/mock_rtp_rtcp.h" +#include "call/rtp_transport_controller_send.h" #include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/rate_limiter.h" #include "test/field_trial.h" #include "test/gmock.h" #include "test/gtest.h" +#include "test/mock_transport.h" +#include "video/call_stats.h" +#include "video/send_delay_stats.h" +#include "video/send_statistics_proxy.h" using ::testing::_; using ::testing::AnyNumber; @@ -35,12 +39,105 @@ const int16_t kInitialPictureId1 = 222; const int16_t kInitialPictureId2 = 44; const int16_t kInitialTl0PicIdx1 = 99; const int16_t kInitialTl0PicIdx2 = 199; +const int64_t kRetransmitWindowSizeMs = 500; + +class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver { + public: + MOCK_METHOD1(OnReceivedIntraFrameRequest, void(uint32_t)); +}; + +class MockOverheadObserver : public OverheadObserver { + public: + MOCK_METHOD1(OnOverheadChanged, void(size_t overhead_bytes_per_packet)); +}; + +class MockCongestionObserver : public NetworkChangedObserver { + public: + MOCK_METHOD4(OnNetworkChanged, + void(uint32_t bitrate_bps, + uint8_t fraction_loss, + int64_t rtt_ms, + int64_t probing_interval_ms)); +}; + +RtpSenderObservers CreateObservers( + RtcpRttStats* rtcp_rtt_stats, + RtcpIntraFrameObserver* intra_frame_callback, + RtcpStatisticsCallback* rtcp_stats, + StreamDataCountersCallback* rtp_stats, + BitrateStatisticsObserver* bitrate_observer, + FrameCountObserver* frame_count_observer, + RtcpPacketTypeCounterObserver* rtcp_type_observer, + SendSideDelayObserver* send_delay_observer, + SendPacketObserver* send_packet_observer, + OverheadObserver* overhead_observer) { + RtpSenderObservers observers; + observers.rtcp_rtt_stats = rtcp_rtt_stats; + observers.intra_frame_callback = intra_frame_callback; + observers.rtcp_stats = rtcp_stats; + observers.rtp_stats = rtp_stats; + observers.bitrate_observer = bitrate_observer; + observers.frame_count_observer = frame_count_observer; + observers.rtcp_type_observer = rtcp_type_observer; + observers.send_delay_observer = send_delay_observer; + observers.send_packet_observer = send_packet_observer; + observers.overhead_observer = overhead_observer; + return observers; +} + +class PayloadRouterTestFixture { + public: + PayloadRouterTestFixture( + const std::vector& ssrcs, + int payload_type, + const std::map& suspended_payload_states) + : clock_(0), + config_(&transport_), + send_delay_stats_(&clock_), + transport_controller_(&clock_, &event_log_, nullptr, bitrate_config_), + process_thread_(ProcessThread::Create("test_thread")), + call_stats_(&clock_, process_thread_.get()), + stats_proxy_(&clock_, + config_, + VideoEncoderConfig::ContentType::kRealtimeVideo), + retransmission_rate_limiter_(&clock_, kRetransmitWindowSizeMs) { + for (uint32_t ssrc : ssrcs) { + config_.rtp.ssrcs.push_back(ssrc); + } + config_.rtp.payload_type = payload_type; + std::map suspended_ssrcs; + router_ = absl::make_unique( + config_.rtp.ssrcs, suspended_ssrcs, suspended_payload_states, + config_.rtp, config_.rtcp, &transport_, + CreateObservers(&call_stats_, &encoder_feedback_, &stats_proxy_, + &stats_proxy_, &stats_proxy_, &stats_proxy_, + &stats_proxy_, &stats_proxy_, &send_delay_stats_, + &overhead_observer_), + &transport_controller_, &event_log_, &retransmission_rate_limiter_); + } + + PayloadRouter* router() { return router_.get(); } + + private: + NiceMock transport_; + NiceMock congestion_observer_; + NiceMock overhead_observer_; + NiceMock encoder_feedback_; + SimulatedClock clock_; + RtcEventLogNullImpl event_log_; + VideoSendStream::Config config_; + SendDelayStats send_delay_stats_; + BitrateConstraints bitrate_config_; + RtpTransportControllerSend transport_controller_; + std::unique_ptr process_thread_; + CallStats call_stats_; + SendStatisticsProxy stats_proxy_; + RateLimiter retransmission_rate_limiter_; + std::unique_ptr router_; +}; } // namespace TEST(PayloadRouterTest, SendOnOneModule) { - NiceMock rtp; - std::vector modules(1, &rtp); - uint8_t payload = 'a'; EncodedImage encoded_image; encoded_image._timeStamp = 1; @@ -49,57 +146,28 @@ TEST(PayloadRouterTest, SendOnOneModule) { encoded_image._buffer = &payload; encoded_image._length = 1; - PayloadRouter payload_router(modules, {kSsrc1}, kPayloadType, {}); - - EXPECT_CALL(rtp, SendOutgoingData(encoded_image._frameType, kPayloadType, - encoded_image._timeStamp, - encoded_image.capture_time_ms_, &payload, - encoded_image._length, nullptr, _, _)) - .Times(0); + PayloadRouterTestFixture test({kSsrc1}, kPayloadType, {}); EXPECT_NE( EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, nullptr, nullptr).error); + test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); - payload_router.SetActive(true); - EXPECT_CALL(rtp, SendOutgoingData(encoded_image._frameType, kPayloadType, - encoded_image._timeStamp, - encoded_image.capture_time_ms_, &payload, - encoded_image._length, nullptr, _, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(rtp, Sending()).WillOnce(Return(true)); + test.router()->SetActive(true); EXPECT_EQ( EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, nullptr, nullptr).error); + test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); - payload_router.SetActive(false); - EXPECT_CALL(rtp, SendOutgoingData(encoded_image._frameType, kPayloadType, - encoded_image._timeStamp, - encoded_image.capture_time_ms_, &payload, - encoded_image._length, nullptr, _, _)) - .Times(0); + test.router()->SetActive(false); EXPECT_NE( EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, nullptr, nullptr).error); + test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); - payload_router.SetActive(true); - EXPECT_CALL(rtp, SendOutgoingData(encoded_image._frameType, kPayloadType, - encoded_image._timeStamp, - encoded_image.capture_time_ms_, &payload, - encoded_image._length, nullptr, _, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(rtp, Sending()).WillOnce(Return(true)); + test.router()->SetActive(true); EXPECT_EQ( EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, nullptr, nullptr).error); + test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); } TEST(PayloadRouterTest, SendSimulcastSetActive) { - NiceMock rtp_1; - NiceMock rtp_2; - std::vector modules = {&rtp_1, &rtp_2}; - uint8_t payload = 'a'; EncodedImage encoded_image; encoded_image._timeStamp = 1; @@ -108,64 +176,45 @@ TEST(PayloadRouterTest, SendSimulcastSetActive) { encoded_image._buffer = &payload; encoded_image._length = 1; - PayloadRouter payload_router(modules, {kSsrc1, kSsrc2}, kPayloadType, {}); + PayloadRouterTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {}); CodecSpecificInfo codec_info_1; memset(&codec_info_1, 0, sizeof(CodecSpecificInfo)); codec_info_1.codecType = kVideoCodecVP8; codec_info_1.codecSpecific.VP8.simulcastIdx = 0; - payload_router.SetActive(true); - EXPECT_CALL(rtp_1, Sending()).WillOnce(Return(true)); - EXPECT_CALL(rtp_1, SendOutgoingData(encoded_image._frameType, kPayloadType, - encoded_image._timeStamp, - encoded_image.capture_time_ms_, &payload, - encoded_image._length, nullptr, _, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(rtp_2, SendOutgoingData(_, _, _, _, _, _, _, _, _)).Times(0); + test.router()->SetActive(true); EXPECT_EQ(EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, &codec_info_1, nullptr) + test.router() + ->OnEncodedImage(encoded_image, &codec_info_1, nullptr) .error); CodecSpecificInfo codec_info_2; memset(&codec_info_2, 0, sizeof(CodecSpecificInfo)); codec_info_2.codecType = kVideoCodecVP8; codec_info_2.codecSpecific.VP8.simulcastIdx = 1; - - EXPECT_CALL(rtp_2, Sending()).WillOnce(Return(true)); - EXPECT_CALL(rtp_2, SendOutgoingData(encoded_image._frameType, kPayloadType, - encoded_image._timeStamp, - encoded_image.capture_time_ms_, &payload, - encoded_image._length, nullptr, _, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(rtp_1, SendOutgoingData(_, _, _, _, _, _, _, _, _)).Times(0); EXPECT_EQ(EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, &codec_info_2, nullptr) + test.router() + ->OnEncodedImage(encoded_image, &codec_info_2, nullptr) .error); // Inactive. - payload_router.SetActive(false); - EXPECT_CALL(rtp_1, SendOutgoingData(_, _, _, _, _, _, _, _, _)).Times(0); - EXPECT_CALL(rtp_2, SendOutgoingData(_, _, _, _, _, _, _, _, _)).Times(0); + test.router()->SetActive(false); EXPECT_NE(EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, &codec_info_1, nullptr) + test.router() + ->OnEncodedImage(encoded_image, &codec_info_1, nullptr) .error); EXPECT_NE(EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, &codec_info_2, nullptr) + test.router() + ->OnEncodedImage(encoded_image, &codec_info_2, nullptr) .error); } // Tests how setting individual rtp modules to active affects the overall // behavior of the payload router. First sets one module to active and checks -// that outgoing data can be sent on this module, and checks that no data can be -// sent if both modules are inactive. +// that outgoing data can be sent on this module, and checks that no data can +// be sent if both modules are inactive. TEST(PayloadRouterTest, SendSimulcastSetActiveModules) { - NiceMock rtp_1; - NiceMock rtp_2; - std::vector modules = {&rtp_1, &rtp_2}; - uint8_t payload = 'a'; EncodedImage encoded_image; encoded_image._timeStamp = 1; @@ -173,7 +222,8 @@ TEST(PayloadRouterTest, SendSimulcastSetActiveModules) { encoded_image._frameType = kVideoFrameKey; encoded_image._buffer = &payload; encoded_image._length = 1; - PayloadRouter payload_router(modules, {kSsrc1, kSsrc2}, kPayloadType, {}); + + PayloadRouterTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {}); CodecSpecificInfo codec_info_1; memset(&codec_info_1, 0, sizeof(CodecSpecificInfo)); codec_info_1.codecType = kVideoCodecVP8; @@ -186,45 +236,34 @@ TEST(PayloadRouterTest, SendSimulcastSetActiveModules) { // Only setting one stream to active will still set the payload router to // active and allow sending data on the active stream. std::vector active_modules({true, false}); - payload_router.SetActiveModules(active_modules); - - EXPECT_CALL(rtp_1, Sending()).WillOnce(Return(true)); - EXPECT_CALL(rtp_1, SendOutgoingData(encoded_image._frameType, kPayloadType, - encoded_image._timeStamp, - encoded_image.capture_time_ms_, &payload, - encoded_image._length, nullptr, _, _)) - .Times(1) - .WillOnce(Return(true)); + test.router()->SetActiveModules(active_modules); EXPECT_EQ(EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, &codec_info_1, nullptr) + test.router() + ->OnEncodedImage(encoded_image, &codec_info_1, nullptr) .error); - // Setting both streams to inactive will turn the payload router to inactive. + // Setting both streams to inactive will turn the payload router to + // inactive. active_modules = {false, false}; - payload_router.SetActiveModules(active_modules); + test.router()->SetActiveModules(active_modules); // An incoming encoded image will not ask the module to send outgoing data // because the payload router is inactive. - EXPECT_CALL(rtp_1, SendOutgoingData(_, _, _, _, _, _, _, _, _)).Times(0); - EXPECT_CALL(rtp_1, Sending()).Times(0); - EXPECT_CALL(rtp_2, SendOutgoingData(_, _, _, _, _, _, _, _, _)).Times(0); - EXPECT_CALL(rtp_2, Sending()).Times(0); EXPECT_NE(EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, &codec_info_1, nullptr) + test.router() + ->OnEncodedImage(encoded_image, &codec_info_1, nullptr) .error); EXPECT_NE(EncodedImageCallback::Result::OK, - payload_router.OnEncodedImage(encoded_image, &codec_info_2, nullptr) + test.router() + ->OnEncodedImage(encoded_image, &codec_info_2, nullptr) .error); } TEST(PayloadRouterTest, CreateWithNoPreviousStates) { - NiceMock rtp1; - NiceMock rtp2; - std::vector modules = {&rtp1, &rtp2}; - PayloadRouter payload_router(modules, {kSsrc1, kSsrc2}, kPayloadType, {}); - payload_router.SetActive(true); + PayloadRouterTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {}); + test.router()->SetActive(true); std::map initial_states = - payload_router.GetRtpPayloadStates(); + test.router()->GetRtpPayloadStates(); EXPECT_EQ(2u, initial_states.size()); EXPECT_NE(initial_states.find(kSsrc1), initial_states.end()); EXPECT_NE(initial_states.find(kSsrc2), initial_states.end()); @@ -240,14 +279,11 @@ TEST(PayloadRouterTest, CreateWithPreviousStates) { std::map states = {{kSsrc1, state1}, {kSsrc2, state2}}; - NiceMock rtp1; - NiceMock rtp2; - std::vector modules = {&rtp1, &rtp2}; - PayloadRouter payload_router(modules, {kSsrc1, kSsrc2}, kPayloadType, states); - payload_router.SetActive(true); + PayloadRouterTestFixture test({kSsrc1, kSsrc2}, kPayloadType, states); + test.router()->SetActive(true); std::map initial_states = - payload_router.GetRtpPayloadStates(); + test.router()->GetRtpPayloadStates(); EXPECT_EQ(2u, initial_states.size()); EXPECT_EQ(kInitialPictureId1, initial_states[kSsrc1].picture_id); EXPECT_EQ(kInitialTl0PicIdx1, initial_states[kSsrc1].tl0_pic_idx); diff --git a/call/rtp_config.cc b/call/rtp_config.cc index 71322f9940..1445c2552e 100644 --- a/call/rtp_config.cc +++ b/call/rtp_config.cc @@ -9,6 +9,7 @@ */ #include "call/rtp_config.h" + #include "rtc_base/strings/string_builder.h" namespace webrtc { @@ -36,4 +37,89 @@ bool UlpfecConfig::operator==(const UlpfecConfig& other) const { red_payload_type == other.red_payload_type && red_rtx_payload_type == other.red_rtx_payload_type; } + +RtpConfig::RtpConfig() = default; +RtpConfig::RtpConfig(const RtpConfig&) = default; +RtpConfig::~RtpConfig() = default; + +RtpConfig::Flexfec::Flexfec() = default; +RtpConfig::Flexfec::Flexfec(const Flexfec&) = default; +RtpConfig::Flexfec::~Flexfec() = default; + +std::string RtpConfig::ToString() const { + char buf[2 * 1024]; + rtc::SimpleStringBuilder ss(buf); + ss << "{ssrcs: ["; + for (size_t i = 0; i < ssrcs.size(); ++i) { + ss << ssrcs[i]; + if (i != ssrcs.size() - 1) + ss << ", "; + } + ss << ']'; + ss << ", rtcp_mode: " + << (rtcp_mode == RtcpMode::kCompound ? "RtcpMode::kCompound" + : "RtcpMode::kReducedSize"); + ss << ", max_packet_size: " << max_packet_size; + ss << ", extensions: ["; + for (size_t i = 0; i < extensions.size(); ++i) { + ss << extensions[i].ToString(); + if (i != extensions.size() - 1) + ss << ", "; + } + ss << ']'; + + ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}'; + ss << ", ulpfec: " << ulpfec.ToString(); + ss << ", payload_name: " << payload_name; + ss << ", payload_type: " << payload_type; + + ss << ", flexfec: {payload_type: " << flexfec.payload_type; + ss << ", ssrc: " << flexfec.ssrc; + ss << ", protected_media_ssrcs: ["; + for (size_t i = 0; i < flexfec.protected_media_ssrcs.size(); ++i) { + ss << flexfec.protected_media_ssrcs[i]; + if (i != flexfec.protected_media_ssrcs.size() - 1) + ss << ", "; + } + ss << "]}"; + + ss << ", rtx: " << rtx.ToString(); + ss << ", c_name: " << c_name; + ss << '}'; + return ss.str(); +} + +RtpConfig::Rtx::Rtx() = default; +RtpConfig::Rtx::Rtx(const Rtx&) = default; +RtpConfig::Rtx::~Rtx() = default; + +std::string RtpConfig::Rtx::ToString() const { + char buf[1024]; + rtc::SimpleStringBuilder ss(buf); + ss << "{ssrcs: ["; + for (size_t i = 0; i < ssrcs.size(); ++i) { + ss << ssrcs[i]; + if (i != ssrcs.size() - 1) + ss << ", "; + } + ss << ']'; + + ss << ", payload_type: " << payload_type; + ss << '}'; + return ss.str(); +} + +RtcpConfig::RtcpConfig() = default; +RtcpConfig::RtcpConfig(const RtcpConfig&) = default; +RtcpConfig::~RtcpConfig() = default; + +std::string RtcpConfig::ToString() const { + char buf[1024]; + rtc::SimpleStringBuilder ss(buf); + ss << "{video_report_interval_ms: " << video_report_interval_ms; + ss << ", audio_report_interval_ms: " << audio_report_interval_ms; + ss << '}'; + return ss.str(); +} + } // namespace webrtc diff --git a/call/rtp_config.h b/call/rtp_config.h index 86d32ac104..96fe15fbb1 100644 --- a/call/rtp_config.h +++ b/call/rtp_config.h @@ -12,8 +12,17 @@ #define CALL_RTP_CONFIG_H_ #include +#include + +#include "api/rtp_headers.h" +#include "api/rtpparameters.h" namespace webrtc { +// Currently only VP8/VP9 specific. +struct RtpPayloadState { + int16_t picture_id = -1; + uint8_t tl0_pic_idx = 0; +}; // Settings for NACK, see RFC 4585 for details. struct NackConfig { NackConfig() : rtp_history_ms(0) {} @@ -44,5 +53,92 @@ struct UlpfecConfig { // RTX payload type for RED payload. int red_rtx_payload_type; }; + +static const size_t kDefaultMaxPacketSize = 1500 - 40; // TCP over IPv4. +struct RtpConfig { + RtpConfig(); + RtpConfig(const RtpConfig&); + ~RtpConfig(); + std::string ToString() const; + + std::vector ssrcs; + + // The value to send in the MID RTP header extension if the extension is + // included in the list of extensions. + std::string mid; + + // See RtcpMode for description. + RtcpMode rtcp_mode = RtcpMode::kCompound; + + // Max RTP packet size delivered to send transport from VideoEngine. + size_t max_packet_size = kDefaultMaxPacketSize; + + // RTP header extensions to use for this send stream. + std::vector extensions; + + // TODO(nisse): For now, these are fixed, but we'd like to support + // changing codec without recreating the VideoSendStream. Then these + // fields must be removed, and association between payload type and codec + // must move above the per-stream level. Ownership could be with + // RtpTransportControllerSend, with a reference from PayloadRouter, where + // the latter would be responsible for mapping the codec type of encoded + // images to the right payload type. + std::string payload_name; + int payload_type = -1; + + // See NackConfig for description. + NackConfig nack; + + // See UlpfecConfig for description. + UlpfecConfig ulpfec; + + struct Flexfec { + Flexfec(); + Flexfec(const Flexfec&); + ~Flexfec(); + // Payload type of FlexFEC. Set to -1 to disable sending FlexFEC. + int payload_type = -1; + + // SSRC of FlexFEC stream. + uint32_t ssrc = 0; + + // Vector containing a single element, corresponding to the SSRC of the + // media stream being protected by this FlexFEC stream. + // The vector MUST have size 1. + // + // TODO(brandtr): Update comment above when we support + // multistream protection. + std::vector protected_media_ssrcs; + } flexfec; + + // Settings for RTP retransmission payload format, see RFC 4588 for + // details. + struct Rtx { + Rtx(); + Rtx(const Rtx&); + ~Rtx(); + std::string ToString() const; + // SSRCs to use for the RTX streams. + std::vector ssrcs; + + // Payload type to use for the RTX stream. + int payload_type = -1; + } rtx; + + // RTCP CNAME, see RFC 3550. + std::string c_name; +}; + +struct RtcpConfig { + RtcpConfig(); + RtcpConfig(const RtcpConfig&); + ~RtcpConfig(); + std::string ToString() const; + + // Time interval between RTCP report for video + int64_t video_report_interval_ms = 1000; + // Time interval between RTCP report for audio + int64_t audio_report_interval_ms = 5000; +}; } // namespace webrtc #endif // CALL_RTP_CONFIG_H_ diff --git a/call/rtp_payload_params.h b/call/rtp_payload_params.h index b85fb42789..0c71a7b5f5 100644 --- a/call/rtp_payload_params.h +++ b/call/rtp_payload_params.h @@ -15,6 +15,7 @@ #include #include "api/video_codecs/video_encoder.h" +#include "call/rtp_config.h" #include "common_types.h" // NOLINT(build/include) #include "modules/rtp_rtcp/source/rtp_video_header.h" @@ -23,12 +24,6 @@ namespace webrtc { class RTPFragmentationHeader; class RtpRtcp; -// Currently only VP8/VP9 specific. -struct RtpPayloadState { - int16_t picture_id = -1; - uint8_t tl0_pic_idx = 0; -}; - // State for setting picture id and tl0 pic idx, for VP8 and VP9 // TODO(nisse): Make these properties not codec specific. class RtpPayloadParams final { diff --git a/call/rtp_transport_controller_send.cc b/call/rtp_transport_controller_send.cc index e2b8a5edba..10b39e5aff 100644 --- a/call/rtp_transport_controller_send.cc +++ b/call/rtp_transport_controller_send.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ #include +#include #include "absl/memory/memory.h" #include "call/rtp_transport_controller_send.h" @@ -15,10 +16,12 @@ #include "modules/congestion_controller/rtp/include/send_side_congestion_controller.h" #include "rtc_base/location.h" #include "rtc_base/logging.h" +#include "rtc_base/rate_limiter.h" #include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { +static const int64_t kRetransmitWindowSizeMs = 500; const char kTaskQueueExperiment[] = "WebRTC-TaskQueueCongestionControl"; using TaskQueueController = webrtc::webrtc_cc::SendSideCongestionController; @@ -63,6 +66,7 @@ RtpTransportControllerSend::RtpTransportControllerSend( bitrate_configurator_(bitrate_config), process_thread_(ProcessThread::Create("SendControllerThread")), observer_(nullptr), + retransmission_rate_limiter_(clock, kRetransmitWindowSizeMs), task_queue_("rtp_send_controller") { // Created after task_queue to be able to post to the task queue internally. send_side_cc_ = @@ -80,6 +84,24 @@ RtpTransportControllerSend::~RtpTransportControllerSend() { process_thread_->DeRegisterModule(&pacer_); } +PayloadRouter* RtpTransportControllerSend::CreateVideoRtpSender( + const std::vector& ssrcs, + std::map suspended_ssrcs, + const std::map& states, + const RtpConfig& rtp_config, + const RtcpConfig& rtcp_config, + Transport* send_transport, + const RtpSenderObservers& observers, + RtcEventLog* event_log) { + video_rtp_senders_.push_back(absl::make_unique( + ssrcs, suspended_ssrcs, states, rtp_config, rtcp_config, send_transport, + observers, + // TODO(holmer): Remove this circular dependency by injecting + // the parts of RtpTransportControllerSendInterface that are really used. + this, event_log, &retransmission_rate_limiter_)); + return video_rtp_senders_.back().get(); +} + void RtpTransportControllerSend::OnNetworkChanged(uint32_t bitrate_bps, uint8_t fraction_loss, int64_t rtt_ms, @@ -97,16 +119,18 @@ void RtpTransportControllerSend::OnNetworkChanged(uint32_t bitrate_bps, msg.network_estimate.loss_rate_ratio = fraction_loss / 255.0; msg.network_estimate.round_trip_time = TimeDelta::ms(rtt_ms); + retransmission_rate_limiter_.SetMaxRate(bandwidth_bps); + if (!task_queue_.IsCurrent()) { task_queue_.PostTask([this, msg] { rtc::CritScope cs(&observer_crit_); - // We won't register as observer until we have an observer. + // We won't register as observer until we have an observers. RTC_DCHECK(observer_ != nullptr); observer_->OnTargetTransferRate(msg); }); } else { rtc::CritScope cs(&observer_crit_); - // We won't register as observer until we have an observer. + // We won't register as observer until we have an observers. RTC_DCHECK(observer_ != nullptr); observer_->OnTargetTransferRate(msg); } @@ -214,6 +238,9 @@ void RtpTransportControllerSend::OnNetworkRouteChanged( void RtpTransportControllerSend::OnNetworkAvailability(bool network_available) { send_side_cc_->SignalNetworkState(network_available ? kNetworkUp : kNetworkDown); + for (auto& rtp_sender : video_rtp_senders_) { + rtp_sender->OnNetworkAvailability(network_available); + } } RtcpBandwidthObserver* RtpTransportControllerSend::GetBandwidthObserver() { return send_side_cc_->GetBandwidthObserver(); diff --git a/call/rtp_transport_controller_send.h b/call/rtp_transport_controller_send.h index d9a4e18369..ce7ee1e328 100644 --- a/call/rtp_transport_controller_send.h +++ b/call/rtp_transport_controller_send.h @@ -14,8 +14,10 @@ #include #include #include +#include #include "api/transport/network_control.h" +#include "call/payload_router.h" #include "call/rtp_bitrate_configurator.h" #include "call/rtp_transport_controller_send_interface.h" #include "common_types.h" // NOLINT(build/include) @@ -44,6 +46,17 @@ class RtpTransportControllerSend final const BitrateConstraints& bitrate_config); ~RtpTransportControllerSend() override; + PayloadRouter* CreateVideoRtpSender( + const std::vector& ssrcs, + std::map suspended_ssrcs, + const std::map& + states, // move states into RtpTransportControllerSend + const RtpConfig& rtp_config, + const RtcpConfig& rtcp_config, + Transport* send_transport, + const RtpSenderObservers& observers, + RtcEventLog* event_log) override; + // Implements NetworkChangedObserver interface. void OnNetworkChanged(uint32_t bitrate_bps, uint8_t fraction_loss, @@ -90,6 +103,7 @@ class RtpTransportControllerSend final private: const Clock* const clock_; PacketRouter packet_router_; + std::vector> video_rtp_senders_; PacedSender pacer_; RtpKeepAliveConfig keepalive_; RtpBitrateConfigurator bitrate_configurator_; @@ -98,6 +112,8 @@ class RtpTransportControllerSend final rtc::CriticalSection observer_crit_; TargetTransferRateObserver* observer_ RTC_GUARDED_BY(observer_crit_); std::unique_ptr send_side_cc_; + RateLimiter retransmission_rate_limiter_; + // TODO(perkj): |task_queue_| is supposed to replace |process_thread_|. // |task_queue_| is defined last to ensure all pending tasks are cancelled // and deleted before any other members. diff --git a/call/rtp_transport_controller_send_interface.h b/call/rtp_transport_controller_send_interface.h index c3a56ad4ce..e954b021f3 100644 --- a/call/rtp_transport_controller_send_interface.h +++ b/call/rtp_transport_controller_send_interface.h @@ -13,11 +13,16 @@ #include #include +#include #include +#include #include "absl/types/optional.h" #include "api/bitrate_constraints.h" #include "api/transport/bitrate_settings.h" +#include "call/rtp_config.h" +#include "logging/rtc_event_log/rtc_event_log.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" namespace rtc { struct SentPacket; @@ -26,18 +31,36 @@ class TaskQueue; } // namespace rtc namespace webrtc { +class CallStats; class CallStatsObserver; class TargetTransferRateObserver; +class Transport; class Module; class PacedSender; class PacketFeedbackObserver; class PacketRouter; +class VideoRtpSenderInterface; class RateLimiter; class RtcpBandwidthObserver; class RtpPacketSender; struct RtpKeepAliveConfig; +class SendDelayStats; +class SendStatisticsProxy; class TransportFeedbackObserver; +struct RtpSenderObservers { + RtcpRttStats* rtcp_rtt_stats; + RtcpIntraFrameObserver* intra_frame_callback; + RtcpStatisticsCallback* rtcp_stats; + StreamDataCountersCallback* rtp_stats; + BitrateStatisticsObserver* bitrate_observer; + FrameCountObserver* frame_count_observer; + RtcpPacketTypeCounterObserver* rtcp_type_observer; + SendSideDelayObserver* send_delay_observer; + SendPacketObserver* send_packet_observer; + OverheadObserver* overhead_observer; +}; + // An RtpTransportController should own everything related to the RTP // transport to/from a remote endpoint. We should have separate // interfaces for send and receive side, even if they are implemented @@ -66,6 +89,18 @@ class RtpTransportControllerSendInterface { virtual ~RtpTransportControllerSendInterface() {} virtual rtc::TaskQueue* GetWorkerQueue() = 0; virtual PacketRouter* packet_router() = 0; + + virtual VideoRtpSenderInterface* CreateVideoRtpSender( + const std::vector& ssrcs, + std::map suspended_ssrcs, + // TODO(holmer): Move states into RtpTransportControllerSend. + const std::map& states, + const RtpConfig& rtp_config, + const RtcpConfig& rtcp_config, + Transport* send_transport, + const RtpSenderObservers& observers, + RtcEventLog* event_log) = 0; + virtual TransportFeedbackObserver* transport_feedback_observer() = 0; virtual RtpPacketSender* packet_sender() = 0; diff --git a/call/test/mock_rtp_transport_controller_send.h b/call/test/mock_rtp_transport_controller_send.h index 419ad77103..d184e699e0 100644 --- a/call/test/mock_rtp_transport_controller_send.h +++ b/call/test/mock_rtp_transport_controller_send.h @@ -11,7 +11,9 @@ #ifndef CALL_TEST_MOCK_RTP_TRANSPORT_CONTROLLER_SEND_H_ #define CALL_TEST_MOCK_RTP_TRANSPORT_CONTROLLER_SEND_H_ +#include #include +#include #include "api/bitrate_constraints.h" #include "call/rtp_transport_controller_send_interface.h" @@ -27,6 +29,16 @@ namespace webrtc { class MockRtpTransportControllerSend : public RtpTransportControllerSendInterface { public: + MOCK_METHOD8( + CreateVideoRtpSender, + VideoRtpSenderInterface*(const std::vector&, + std::map, + const std::map&, + const RtpConfig&, + const RtcpConfig&, + Transport*, + const RtpSenderObservers&, + RtcEventLog*)); MOCK_METHOD0(GetWorkerQueue, rtc::TaskQueue*()); MOCK_METHOD0(packet_router, PacketRouter*()); MOCK_METHOD0(transport_feedback_observer, TransportFeedbackObserver*()); diff --git a/call/video_rtp_sender_interface.h b/call/video_rtp_sender_interface.h new file mode 100644 index 0000000000..0d47845944 --- /dev/null +++ b/call/video_rtp_sender_interface.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_VIDEO_RTP_SENDER_INTERFACE_H_ +#define CALL_VIDEO_RTP_SENDER_INTERFACE_H_ + +#include +#include + +#include "call/rtp_config.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/utility/include/process_thread.h" +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { +class VideoBitrateAllocation; +struct FecProtectionParams; + +class VideoRtpSenderInterface : public EncodedImageCallback { + public: + virtual void RegisterProcessThread(ProcessThread* module_process_thread) = 0; + virtual void DeRegisterProcessThread() = 0; + + // PayloadRouter will only route packets if being active, all packets will be + // dropped otherwise. + virtual void SetActive(bool active) = 0; + // Sets the sending status of the rtp modules and appropriately sets the + // payload router to active if any rtp modules are active. + virtual void SetActiveModules(const std::vector active_modules) = 0; + virtual bool IsActive() = 0; + + virtual void OnNetworkAvailability(bool network_available) = 0; + virtual std::map GetRtpStates() const = 0; + virtual std::map GetRtpPayloadStates() const = 0; + + virtual bool FecEnabled() const = 0; + + virtual bool NackEnabled() const = 0; + + virtual void DeliverRtcp(const uint8_t* packet, size_t length) = 0; + + virtual void ProtectionRequest(const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) = 0; + + virtual void SetMaxRtpPacketSize(size_t max_rtp_packet_size) = 0; + virtual void OnBitrateAllocationUpdated( + const VideoBitrateAllocation& bitrate) = 0; +}; +} // namespace webrtc +#endif // CALL_VIDEO_RTP_SENDER_INTERFACE_H_ diff --git a/call/video_send_stream.cc b/call/video_send_stream.cc index 9024e3a6a5..bb590fa170 100644 --- a/call/video_send_stream.cc +++ b/call/video_send_stream.cc @@ -95,89 +95,4 @@ std::string VideoSendStream::Config::EncoderSettings::ToString() const { ss << '}'; return ss.str(); } - -VideoSendStream::Config::Rtp::Rtp() = default; -VideoSendStream::Config::Rtp::Rtp(const Rtp&) = default; -VideoSendStream::Config::Rtp::~Rtp() = default; - -VideoSendStream::Config::Rtp::Flexfec::Flexfec() = default; -VideoSendStream::Config::Rtp::Flexfec::Flexfec(const Flexfec&) = default; -VideoSendStream::Config::Rtp::Flexfec::~Flexfec() = default; - -std::string VideoSendStream::Config::Rtp::ToString() const { - char buf[2 * 1024]; - rtc::SimpleStringBuilder ss(buf); - ss << "{ssrcs: ["; - for (size_t i = 0; i < ssrcs.size(); ++i) { - ss << ssrcs[i]; - if (i != ssrcs.size() - 1) - ss << ", "; - } - ss << ']'; - ss << ", rtcp_mode: " - << (rtcp_mode == RtcpMode::kCompound ? "RtcpMode::kCompound" - : "RtcpMode::kReducedSize"); - ss << ", max_packet_size: " << max_packet_size; - ss << ", extensions: ["; - for (size_t i = 0; i < extensions.size(); ++i) { - ss << extensions[i].ToString(); - if (i != extensions.size() - 1) - ss << ", "; - } - ss << ']'; - - ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}'; - ss << ", ulpfec: " << ulpfec.ToString(); - ss << ", payload_name: " << payload_name; - ss << ", payload_type: " << payload_type; - - ss << ", flexfec: {payload_type: " << flexfec.payload_type; - ss << ", ssrc: " << flexfec.ssrc; - ss << ", protected_media_ssrcs: ["; - for (size_t i = 0; i < flexfec.protected_media_ssrcs.size(); ++i) { - ss << flexfec.protected_media_ssrcs[i]; - if (i != flexfec.protected_media_ssrcs.size() - 1) - ss << ", "; - } - ss << "]}"; - - ss << ", rtx: " << rtx.ToString(); - ss << ", c_name: " << c_name; - ss << '}'; - return ss.str(); -} - -VideoSendStream::Config::Rtp::Rtx::Rtx() = default; -VideoSendStream::Config::Rtp::Rtx::Rtx(const Rtx&) = default; -VideoSendStream::Config::Rtp::Rtx::~Rtx() = default; - -std::string VideoSendStream::Config::Rtp::Rtx::ToString() const { - char buf[1024]; - rtc::SimpleStringBuilder ss(buf); - ss << "{ssrcs: ["; - for (size_t i = 0; i < ssrcs.size(); ++i) { - ss << ssrcs[i]; - if (i != ssrcs.size() - 1) - ss << ", "; - } - ss << ']'; - - ss << ", payload_type: " << payload_type; - ss << '}'; - return ss.str(); -} - -VideoSendStream::Config::Rtcp::Rtcp() = default; -VideoSendStream::Config::Rtcp::Rtcp(const Rtcp&) = default; -VideoSendStream::Config::Rtcp::~Rtcp() = default; - -std::string VideoSendStream::Config::Rtcp::ToString() const { - char buf[1024]; - rtc::SimpleStringBuilder ss(buf); - ss << "{video_report_interval_ms: " << video_report_interval_ms; - ss << ", audio_report_interval_ms: " << audio_report_interval_ms; - ss << '}'; - return ss.str(); -} - } // namespace webrtc diff --git a/call/video_send_stream.h b/call/video_send_stream.h index b5bd199410..eada8fef2c 100644 --- a/call/video_send_stream.h +++ b/call/video_send_stream.h @@ -118,92 +118,9 @@ class VideoSendStream { VideoEncoderFactory* encoder_factory = nullptr; } encoder_settings; - static const size_t kDefaultMaxPacketSize = 1500 - 40; // TCP over IPv4. - struct Rtp { - Rtp(); - Rtp(const Rtp&); - ~Rtp(); - std::string ToString() const; + RtpConfig rtp; - std::vector ssrcs; - - // The value to send in the MID RTP header extension if the extension is - // included in the list of extensions. - std::string mid; - - // See RtcpMode for description. - RtcpMode rtcp_mode = RtcpMode::kCompound; - - // Max RTP packet size delivered to send transport from VideoEngine. - size_t max_packet_size = kDefaultMaxPacketSize; - - // RTP header extensions to use for this send stream. - std::vector extensions; - - // TODO(nisse): For now, these are fixed, but we'd like to support - // changing codec without recreating the VideoSendStream. Then these - // fields must be removed, and association between payload type and codec - // must move above the per-stream level. Ownership could be with - // RtpTransportControllerSend, with a reference from PayloadRouter, where - // the latter would be responsible for mapping the codec type of encoded - // images to the right payload type. - std::string payload_name; - int payload_type = -1; - - // See NackConfig for description. - NackConfig nack; - - // See UlpfecConfig for description. - UlpfecConfig ulpfec; - - struct Flexfec { - Flexfec(); - Flexfec(const Flexfec&); - ~Flexfec(); - // Payload type of FlexFEC. Set to -1 to disable sending FlexFEC. - int payload_type = -1; - - // SSRC of FlexFEC stream. - uint32_t ssrc = 0; - - // Vector containing a single element, corresponding to the SSRC of the - // media stream being protected by this FlexFEC stream. - // The vector MUST have size 1. - // - // TODO(brandtr): Update comment above when we support - // multistream protection. - std::vector protected_media_ssrcs; - } flexfec; - - // Settings for RTP retransmission payload format, see RFC 4588 for - // details. - struct Rtx { - Rtx(); - Rtx(const Rtx&); - ~Rtx(); - std::string ToString() const; - // SSRCs to use for the RTX streams. - std::vector ssrcs; - - // Payload type to use for the RTX stream. - int payload_type = -1; - } rtx; - - // RTCP CNAME, see RFC 3550. - std::string c_name; - } rtp; - - struct Rtcp { - Rtcp(); - Rtcp(const Rtcp&); - ~Rtcp(); - std::string ToString() const; - - // Time interval between RTCP report for video - int64_t video_report_interval_ms = 1000; - // Time interval between RTCP report for audio - int64_t audio_report_interval_ms = 5000; - } rtcp; + RtcpConfig rtcp; // Transport for outgoing packets. Transport* send_transport = nullptr; diff --git a/modules/video_coding/decoder_database.cc b/modules/video_coding/decoder_database.cc index 908a94a75e..9cb7823256 100644 --- a/modules/video_coding/decoder_database.cc +++ b/modules/video_coding/decoder_database.cc @@ -29,6 +29,8 @@ VCMExtDecoderMapItem::VCMExtDecoderMapItem( : payload_type(payload_type), external_decoder_instance(external_decoder_instance) {} +VCMDecoderMapItem::~VCMDecoderMapItem() {} + VCMDecoderDataBase::VCMDecoderDataBase() : receive_codec_(), dec_map_(), dec_external_map_() {} diff --git a/modules/video_coding/decoder_database.h b/modules/video_coding/decoder_database.h index c3779c5362..8c96b41efd 100644 --- a/modules/video_coding/decoder_database.h +++ b/modules/video_coding/decoder_database.h @@ -23,6 +23,7 @@ struct VCMDecoderMapItem { VCMDecoderMapItem(VideoCodec* settings, int number_of_cores, bool require_key_frame); + ~VCMDecoderMapItem(); std::unique_ptr settings; int number_of_cores; diff --git a/modules/video_coding/generic_encoder.cc b/modules/video_coding/generic_encoder.cc index 7eb35e78da..7d8bb6a784 100644 --- a/modules/video_coding/generic_encoder.cc +++ b/modules/video_coding/generic_encoder.cc @@ -31,6 +31,9 @@ const int kMessagesThrottlingThreshold = 2; const int kThrottleRatio = 100000; } // namespace +VCMEncodedFrameCallback::TimingFramesLayerInfo::TimingFramesLayerInfo() {} +VCMEncodedFrameCallback::TimingFramesLayerInfo::~TimingFramesLayerInfo() {} + VCMGenericEncoder::VCMGenericEncoder( VideoEncoder* encoder, VCMEncodedFrameCallback* encoded_frame_callback, diff --git a/modules/video_coding/generic_encoder.h b/modules/video_coding/generic_encoder.h index 0759f55053..151e93e497 100644 --- a/modules/video_coding/generic_encoder.h +++ b/modules/video_coding/generic_encoder.h @@ -38,7 +38,7 @@ class VCMEncodedFrameCallback : public EncodedImageCallback { public: VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback, media_optimization::MediaOptimization* media_opt); - virtual ~VCMEncodedFrameCallback(); + ~VCMEncodedFrameCallback() override; // Implements EncodedImageCallback. EncodedImageCallback::Result OnEncodedImage( @@ -102,6 +102,8 @@ class VCMEncodedFrameCallback : public EncodedImageCallback { int64_t encode_start_time_ms; }; struct TimingFramesLayerInfo { + TimingFramesLayerInfo(); + ~TimingFramesLayerInfo(); size_t target_bitrate_bytes_per_sec = 0; std::list encode_start_list; }; diff --git a/modules/video_coding/include/video_coding.h b/modules/video_coding/include/video_coding.h index e5c30ebf2d..8ef046aa23 100644 --- a/modules/video_coding/include/video_coding.h +++ b/modules/video_coding/include/video_coding.h @@ -46,9 +46,9 @@ class EventFactory { class EventFactoryImpl : public EventFactory { public: - virtual ~EventFactoryImpl() {} + ~EventFactoryImpl() override {} - virtual EventWrapper* CreateEvent() { return EventWrapper::Create(); } + EventWrapper* CreateEvent() override; }; // Used to indicate which decode with errors mode should be used. diff --git a/modules/video_coding/jitter_buffer.cc b/modules/video_coding/jitter_buffer.cc index b98fd922b4..83f90e334f 100644 --- a/modules/video_coding/jitter_buffer.cc +++ b/modules/video_coding/jitter_buffer.cc @@ -123,6 +123,9 @@ void FrameList::Reset(UnorderedFrameList* free_frames) { } } +Vp9SsMap::Vp9SsMap() {} +Vp9SsMap::~Vp9SsMap() {} + bool Vp9SsMap::Insert(const VCMPacket& packet) { if (!packet.video_header.vp9().ss_data_available) return false; diff --git a/modules/video_coding/jitter_buffer.h b/modules/video_coding/jitter_buffer.h index 49080808f7..e1414aa4ee 100644 --- a/modules/video_coding/jitter_buffer.h +++ b/modules/video_coding/jitter_buffer.h @@ -75,6 +75,9 @@ class FrameList class Vp9SsMap { public: typedef std::map SsMap; + Vp9SsMap(); + ~Vp9SsMap(); + bool Insert(const VCMPacket& packet); void Reset(); diff --git a/modules/video_coding/media_opt_util.cc b/modules/video_coding/media_opt_util.cc index ca9620f85e..4afe47dd8d 100644 --- a/modules/video_coding/media_opt_util.cc +++ b/modules/video_coding/media_opt_util.cc @@ -29,6 +29,20 @@ static const int kPacketLossMax = 129; namespace media_optimization { +VCMProtectionParameters::VCMProtectionParameters() + : rtt(0), + lossPr(0.0f), + bitRate(0.0f), + packetsPerFrame(0.0f), + packetsPerFrameKey(0.0f), + frameRate(0.0f), + keyFrameSize(0.0f), + fecRateDelta(0), + fecRateKey(0), + codecWidth(0), + codecHeight(0), + numLayers(1) {} + VCMProtectionMethod::VCMProtectionMethod() : _effectivePacketLoss(0), _protectionFactorK(0), @@ -40,6 +54,34 @@ VCMProtectionMethod::VCMProtectionMethod() VCMProtectionMethod::~VCMProtectionMethod() {} +enum VCMProtectionMethodEnum VCMProtectionMethod::Type() const { + return _type; +} + +uint8_t VCMProtectionMethod::RequiredPacketLossER() { + return _effectivePacketLoss; +} + +uint8_t VCMProtectionMethod::RequiredProtectionFactorK() { + return _protectionFactorK; +} + +uint8_t VCMProtectionMethod::RequiredProtectionFactorD() { + return _protectionFactorD; +} + +bool VCMProtectionMethod::RequiredUepProtectionK() { + return _useUepProtectionK; +} + +bool VCMProtectionMethod::RequiredUepProtectionD() { + return _useUepProtectionD; +} + +int VCMProtectionMethod::MaxFramesFec() const { + return 1; +} + VCMNackFecMethod::VCMNackFecMethod(int64_t lowRttNackThresholdMs, int64_t highRttNackThresholdMs) : VCMFecMethod(), diff --git a/modules/video_coding/media_opt_util.h b/modules/video_coding/media_opt_util.h index c91ab2bbe7..9cc8d6db3e 100644 --- a/modules/video_coding/media_opt_util.h +++ b/modules/video_coding/media_opt_util.h @@ -48,19 +48,7 @@ const int64_t kLowRttNackMs = 20; const int kMaxRttDelayThreshold = 500; struct VCMProtectionParameters { - VCMProtectionParameters() - : rtt(0), - lossPr(0.0f), - bitRate(0.0f), - packetsPerFrame(0.0f), - packetsPerFrameKey(0.0f), - frameRate(0.0f), - keyFrameSize(0.0f), - fecRateDelta(0), - fecRateKey(0), - codecWidth(0), - codecHeight(0), - numLayers(1) {} + VCMProtectionParameters(); int64_t rtt; float lossPr; @@ -107,38 +95,38 @@ class VCMProtectionMethod { // Returns the protection type // // Return value : The protection type - enum VCMProtectionMethodEnum Type() const { return _type; } + VCMProtectionMethodEnum Type() const; // Returns the effective packet loss for ER, required by this protection // method // // Return value : Required effective packet loss - virtual uint8_t RequiredPacketLossER() { return _effectivePacketLoss; } + virtual uint8_t RequiredPacketLossER(); // Extracts the FEC protection factor for Key frame, required by this // protection method // // Return value : Required protectionFactor for Key frame - virtual uint8_t RequiredProtectionFactorK() { return _protectionFactorK; } + virtual uint8_t RequiredProtectionFactorK(); // Extracts the FEC protection factor for Delta frame, required by this // protection method // // Return value : Required protectionFactor for delta frame - virtual uint8_t RequiredProtectionFactorD() { return _protectionFactorD; } + virtual uint8_t RequiredProtectionFactorD(); // Extracts whether the FEC Unequal protection (UEP) is used for Key frame. // // Return value : Required Unequal protection on/off state. - virtual bool RequiredUepProtectionK() { return _useUepProtectionK; } + virtual bool RequiredUepProtectionK(); // Extracts whether the the FEC Unequal protection (UEP) is used for Delta // frame. // // Return value : Required Unequal protection on/off state. - virtual bool RequiredUepProtectionD() { return _useUepProtectionD; } + virtual bool RequiredUepProtectionD(); - virtual int MaxFramesFec() const { return 1; } + virtual int MaxFramesFec() const; protected: uint8_t _effectivePacketLoss; @@ -151,14 +139,14 @@ class VCMProtectionMethod { bool _useUepProtectionK; bool _useUepProtectionD; float _corrFecCost; - enum VCMProtectionMethodEnum _type; + VCMProtectionMethodEnum _type; }; class VCMNackMethod : public VCMProtectionMethod { public: VCMNackMethod(); - virtual ~VCMNackMethod(); - virtual bool UpdateParameters(const VCMProtectionParameters* parameters); + ~VCMNackMethod() override; + bool UpdateParameters(const VCMProtectionParameters* parameters) override; // Get the effective packet loss bool EffectivePacketLoss(const VCMProtectionParameters* parameter); }; @@ -166,8 +154,8 @@ class VCMNackMethod : public VCMProtectionMethod { class VCMFecMethod : public VCMProtectionMethod { public: VCMFecMethod(); - virtual ~VCMFecMethod(); - virtual bool UpdateParameters(const VCMProtectionParameters* parameters); + ~VCMFecMethod() override; + bool UpdateParameters(const VCMProtectionParameters* parameters) override; // Get the effective packet loss for ER bool EffectivePacketLoss(const VCMProtectionParameters* parameters); // Get the FEC protection factors @@ -202,14 +190,14 @@ class VCMNackFecMethod : public VCMFecMethod { public: VCMNackFecMethod(int64_t lowRttNackThresholdMs, int64_t highRttNackThresholdMs); - virtual ~VCMNackFecMethod(); - virtual bool UpdateParameters(const VCMProtectionParameters* parameters); + ~VCMNackFecMethod() override; + bool UpdateParameters(const VCMProtectionParameters* parameters) override; // Get the effective packet loss for ER bool EffectivePacketLoss(const VCMProtectionParameters* parameters); // Get the protection factors bool ProtectionFactor(const VCMProtectionParameters* parameters); // Get the max number of frames the FEC is allowed to be based on. - int MaxFramesFec() const; + int MaxFramesFec() const override; // Turn off the FEC based on low bitrate and other factors. bool BitRateTooLowForFec(const VCMProtectionParameters* parameters); diff --git a/modules/video_coding/session_info.cc b/modules/video_coding/session_info.cc index 1b6f7329c8..834684ecf0 100644 --- a/modules/video_coding/session_info.cc +++ b/modules/video_coding/session_info.cc @@ -33,6 +33,8 @@ VCMSessionInfo::VCMSessionInfo() first_packet_seq_num_(-1), last_packet_seq_num_(-1) {} +VCMSessionInfo::~VCMSessionInfo() {} + void VCMSessionInfo::UpdateDataPointers(const uint8_t* old_base_ptr, const uint8_t* new_base_ptr) { for (PacketIterator it = packets_.begin(); it != packets_.end(); ++it) diff --git a/modules/video_coding/session_info.h b/modules/video_coding/session_info.h index 0b8fd69092..b845ffb546 100644 --- a/modules/video_coding/session_info.h +++ b/modules/video_coding/session_info.h @@ -30,6 +30,7 @@ struct FrameData { class VCMSessionInfo { public: VCMSessionInfo(); + ~VCMSessionInfo(); void UpdateDataPointers(const uint8_t* old_base_ptr, const uint8_t* new_base_ptr); diff --git a/modules/video_coding/video_coding_impl.cc b/modules/video_coding/video_coding_impl.cc index aa9a0d5bd5..77bd2889b4 100644 --- a/modules/video_coding/video_coding_impl.cc +++ b/modules/video_coding/video_coding_impl.cc @@ -27,6 +27,10 @@ #include "system_wrappers/include/clock.h" namespace webrtc { +EventWrapper* EventFactoryImpl::CreateEvent() { + return EventWrapper::Create(); +} + namespace vcm { int64_t VCMProcessTimer::Period() const { diff --git a/video/BUILD.gn b/video/BUILD.gn index 7bf7404ae5..a135696692 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -77,6 +77,7 @@ rtc_static_library("video") { "../modules/video_coding:packet", "../modules/video_coding:video_codec_interface", "../rtc_base:checks", + "../rtc_base:rate_limiter", "../rtc_base:stringutils", "../rtc_base/experiments:alr_experiment", "../rtc_base/experiments:quality_scaling_experiment", diff --git a/video/call_stats.h b/video/call_stats.h index 00feb538ee..930c3efffd 100644 --- a/video/call_stats.h +++ b/video/call_stats.h @@ -32,7 +32,7 @@ class CallStats : public Module, public RtcpRttStats { static constexpr int64_t kUpdateIntervalMs = 1000; CallStats(Clock* clock, ProcessThread* process_thread); - ~CallStats(); + ~CallStats() override; // Registers/deregisters a new observer to receive statistics updates. // Must be called from the construction thread. diff --git a/video/end_to_end_tests/config_tests.cc b/video/end_to_end_tests/config_tests.cc index b5724a317e..d32b1118b7 100644 --- a/video/end_to_end_tests/config_tests.cc +++ b/video/end_to_end_tests/config_tests.cc @@ -31,8 +31,7 @@ void VerifyEmptyUlpfecConfig(const UlpfecConfig& config) { << "Enabling RTX in ULPFEC requires rtpmap: rtx negotiation."; } -void VerifyEmptyFlexfecConfig( - const VideoSendStream::Config::Rtp::Flexfec& config) { +void VerifyEmptyFlexfecConfig(const RtpConfig::Flexfec& config) { EXPECT_EQ(-1, config.payload_type) << "Enabling FlexFEC requires rtpmap: flexfec negotiation."; EXPECT_EQ(0U, config.ssrc) diff --git a/video/report_block_stats.cc b/video/report_block_stats.cc index 42cd2ca4e8..e11568eeaf 100644 --- a/video/report_block_stats.cc +++ b/video/report_block_stats.cc @@ -27,6 +27,8 @@ int FractionLost(uint32_t num_lost_sequence_numbers, ReportBlockStats::ReportBlockStats() : num_sequence_numbers_(0), num_lost_sequence_numbers_(0) {} +ReportBlockStats::~ReportBlockStats() {} + void ReportBlockStats::Store(const RtcpStatistics& rtcp_stats, uint32_t remote_ssrc, uint32_t source_ssrc) { diff --git a/video/report_block_stats.h b/video/report_block_stats.h index b3c7cf259b..241fec76a6 100644 --- a/video/report_block_stats.h +++ b/video/report_block_stats.h @@ -25,7 +25,7 @@ class ReportBlockStats { typedef std::map ReportBlockMap; typedef std::vector ReportBlockVector; ReportBlockStats(); - ~ReportBlockStats() {} + ~ReportBlockStats(); // Updates stats and stores report blocks. // Returns an aggregate of the |report_blocks|. diff --git a/video/send_delay_stats.h b/video/send_delay_stats.h index 9b9e9214d9..81442bcbd1 100644 --- a/video/send_delay_stats.h +++ b/video/send_delay_stats.h @@ -28,7 +28,7 @@ namespace webrtc { class SendDelayStats : public SendPacketObserver { public: explicit SendDelayStats(Clock* clock); - virtual ~SendDelayStats(); + ~SendDelayStats() override; // Adds the configured ssrcs for the rtp streams. // Stats will be calculated for these streams. diff --git a/video/send_statistics_proxy.cc b/video/send_statistics_proxy.cc index 44cbe010b5..b446d193dd 100644 --- a/video/send_statistics_proxy.cc +++ b/video/send_statistics_proxy.cc @@ -262,7 +262,7 @@ bool SendStatisticsProxy::UmaSamplesContainer::InsertEncodedFrame( } void SendStatisticsProxy::UmaSamplesContainer::UpdateHistograms( - const VideoSendStream::Config::Rtp& rtp_config, + const RtpConfig& rtp_config, const VideoSendStream::Stats& current_stats) { RTC_DCHECK(uma_prefix_ == kRealtimePrefix || uma_prefix_ == kScreenPrefix); const int kIndex = uma_prefix_ == kScreenPrefix ? 1 : 0; diff --git a/video/send_statistics_proxy.h b/video/send_statistics_proxy.h index a36e9a8c9e..5bc6c90cdd 100644 --- a/video/send_statistics_proxy.h +++ b/video/send_statistics_proxy.h @@ -48,12 +48,12 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver, SendStatisticsProxy(Clock* clock, const VideoSendStream::Config& config, VideoEncoderConfig::ContentType content_type); - virtual ~SendStatisticsProxy(); + ~SendStatisticsProxy() override; virtual VideoSendStream::Stats GetStats(); - virtual void OnSendEncodedImage(const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_info); + void OnSendEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_info); // Used to update incoming frame rate. void OnIncomingFrame(int width, int height); @@ -158,6 +158,7 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver, int64_t last_ms; }; struct FallbackEncoderInfo { + FallbackEncoderInfo() = default; bool is_possible = true; bool is_active = false; int on_off_events = 0; @@ -234,7 +235,7 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver, Clock* const clock_; const std::string payload_name_; - const VideoSendStream::Config::Rtp rtp_config_; + const RtpConfig rtp_config_; const absl::optional fallback_max_pixels_; const absl::optional fallback_max_pixels_disabled_; rtc::CriticalSection crit_; @@ -259,7 +260,7 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver, Clock* clock); ~UmaSamplesContainer(); - void UpdateHistograms(const VideoSendStream::Config::Rtp& rtp_config, + void UpdateHistograms(const RtpConfig& rtp_config, const VideoSendStream::Stats& current_stats); void InitializeBitrateCounters(const VideoSendStream::Stats& stats); diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index 4885fc3075..f65c8c58ae 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -12,13 +12,14 @@ #include #include "modules/rtp_rtcp/source/rtp_sender.h" +#include "rtc_base/logging.h" #include "video/video_send_stream_impl.h" namespace webrtc { namespace { -size_t CalculateMaxHeaderSize(const VideoSendStream::Config::Rtp& config) { +size_t CalculateMaxHeaderSize(const RtpConfig& config) { size_t header_size = kRtpHeaderSize; size_t extensions_size = 0; size_t fec_extensions_size = 0; @@ -66,8 +67,7 @@ VideoSendStream::VideoSendStream( VideoEncoderConfig encoder_config, const std::map& suspended_ssrcs, const std::map& suspended_payload_states, - std::unique_ptr fec_controller, - RateLimiter* retransmission_limiter) + std::unique_ptr fec_controller) : worker_queue_(worker_queue), thread_sync_event_(false /* manual_reset */, false), stats_proxy_(Clock::GetRealTimeClock(), @@ -87,14 +87,14 @@ VideoSendStream::VideoSendStream( worker_queue_->PostTask(rtc::NewClosure( [this, call_stats, transport, bitrate_allocator, send_delay_stats, event_log, &suspended_ssrcs, &encoder_config, &suspended_payload_states, - &fec_controller, retransmission_limiter]() { + &fec_controller]() { send_stream_.reset(new VideoSendStreamImpl( &stats_proxy_, worker_queue_, call_stats, transport, bitrate_allocator, send_delay_stats, video_stream_encoder_.get(), event_log, &config_, encoder_config.max_bitrate_bps, encoder_config.bitrate_priority, suspended_ssrcs, suspended_payload_states, encoder_config.content_type, - std::move(fec_controller), retransmission_limiter)); + std::move(fec_controller))); }, [this]() { thread_sync_event_.Set(); })); @@ -180,13 +180,6 @@ absl::optional VideoSendStream::GetPacingFactorOverride() const { return send_stream_->configured_pacing_factor_; } -void VideoSendStream::SignalNetworkState(NetworkState state) { - RTC_DCHECK_RUN_ON(&thread_checker_); - VideoSendStreamImpl* send_stream = send_stream_.get(); - worker_queue_->PostTask( - [send_stream, state] { send_stream->SignalNetworkState(state); }); -} - void VideoSendStream::StopPermanentlyAndGetRtpStates( VideoSendStream::RtpStateMap* rtp_state_map, VideoSendStream::RtpPayloadStateMap* payload_state_map) { diff --git a/video/video_send_stream.h b/video/video_send_stream.h index fec53ac217..b0e4071906 100644 --- a/video/video_send_stream.h +++ b/video/video_send_stream.h @@ -51,6 +51,9 @@ class VideoSendStreamImpl; // |worker_queue|. class VideoSendStream : public webrtc::VideoSendStream { public: + using RtpStateMap = std::map; + using RtpPayloadStateMap = std::map; + VideoSendStream( int num_cpu_cores, ProcessThread* module_process_thread, @@ -64,12 +67,10 @@ class VideoSendStream : public webrtc::VideoSendStream { VideoEncoderConfig encoder_config, const std::map& suspended_ssrcs, const std::map& suspended_payload_states, - std::unique_ptr fec_controller, - RateLimiter* retransmission_limiter); + std::unique_ptr fec_controller); ~VideoSendStream() override; - void SignalNetworkState(NetworkState state); bool DeliverRtcp(const uint8_t* packet, size_t length); // webrtc::VideoSendStream implementation. @@ -84,9 +85,6 @@ class VideoSendStream : public webrtc::VideoSendStream { void ReconfigureVideoEncoder(VideoEncoderConfig) override; Stats GetStats() override; - typedef std::map RtpStateMap; - typedef std::map RtpPayloadStateMap; - // Takes ownership of each file, is responsible for closing them later. // Calling this method will close and finalize any current logs. // Giving rtc::kInvalidPlatformFileValue in any position disables logging diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index 3ca1da3bd9..13461c6783 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -15,7 +15,6 @@ #include "call/rtp_transport_controller_send_interface.h" #include "modules/pacing/packet_router.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/source/rtp_sender.h" #include "rtc_base/checks.h" #include "rtc_base/experiments/alr_experiment.h" @@ -29,8 +28,6 @@ namespace webrtc { namespace internal { namespace { -static const int kMinSendSidePacketHistorySize = 600; - // Assume an average video stream has around 3 packets per frame (1 mbps / 30 // fps / 1400B) A sequence number set with size 5500 will be able to store // packet sequence number for at least last 60 seconds. @@ -39,107 +36,6 @@ static const int kSendSideSeqNumSetMaxSize = 5500; // We don't do MTU discovery, so assume that we have the standard ethernet MTU. const size_t kPathMTU = 1500; -std::vector CreateRtpRtcpModules( - const VideoSendStream::Config& config, - RtcpIntraFrameObserver* intra_frame_callback, - RtcpBandwidthObserver* bandwidth_callback, - RtpTransportControllerSendInterface* transport, - RtcpRttStats* rtt_stats, - FlexfecSender* flexfec_sender, - SendStatisticsProxy* stats_proxy, - SendDelayStats* send_delay_stats, - RtcEventLog* event_log, - RateLimiter* retransmission_rate_limiter, - OverheadObserver* overhead_observer, - RtpKeepAliveConfig keepalive_config) { - RTC_DCHECK_GT(config.rtp.ssrcs.size(), 0); - RtpRtcp::Configuration configuration; - configuration.audio = false; - configuration.receiver_only = false; - configuration.outgoing_transport = config.send_transport; - configuration.intra_frame_callback = intra_frame_callback; - configuration.bandwidth_callback = bandwidth_callback; - configuration.transport_feedback_callback = - transport->transport_feedback_observer(); - configuration.rtt_stats = rtt_stats; - configuration.rtcp_packet_type_counter_observer = stats_proxy; - configuration.paced_sender = transport->packet_sender(); - configuration.transport_sequence_number_allocator = - transport->packet_router(); - configuration.send_bitrate_observer = stats_proxy; - configuration.send_frame_count_observer = stats_proxy; - configuration.send_side_delay_observer = stats_proxy; - configuration.send_packet_observer = send_delay_stats; - configuration.event_log = event_log; - configuration.retransmission_rate_limiter = retransmission_rate_limiter; - configuration.overhead_observer = overhead_observer; - configuration.keepalive_config = keepalive_config; - configuration.rtcp_interval_config.video_interval_ms = - config.rtcp.video_report_interval_ms; - configuration.rtcp_interval_config.audio_interval_ms = - config.rtcp.audio_report_interval_ms; - std::vector modules; - const std::vector& flexfec_protected_ssrcs = - config.rtp.flexfec.protected_media_ssrcs; - for (uint32_t ssrc : config.rtp.ssrcs) { - bool enable_flexfec = flexfec_sender != nullptr && - std::find(flexfec_protected_ssrcs.begin(), - flexfec_protected_ssrcs.end(), - ssrc) != flexfec_protected_ssrcs.end(); - configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr; - RtpRtcp* rtp_rtcp = RtpRtcp::CreateRtpRtcp(configuration); - rtp_rtcp->SetSendingStatus(false); - rtp_rtcp->SetSendingMediaStatus(false); - rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); - modules.push_back(rtp_rtcp); - } - return modules; -} - -// TODO(brandtr): Update this function when we support multistream protection. -std::unique_ptr MaybeCreateFlexfecSender( - const VideoSendStream::Config& config, - const std::map& suspended_ssrcs) { - if (config.rtp.flexfec.payload_type < 0) { - return nullptr; - } - RTC_DCHECK_GE(config.rtp.flexfec.payload_type, 0); - RTC_DCHECK_LE(config.rtp.flexfec.payload_type, 127); - if (config.rtp.flexfec.ssrc == 0) { - RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " - "Therefore disabling FlexFEC."; - return nullptr; - } - if (config.rtp.flexfec.protected_media_ssrcs.empty()) { - RTC_LOG(LS_WARNING) - << "FlexFEC is enabled, but no protected media SSRC given. " - "Therefore disabling FlexFEC."; - return nullptr; - } - - if (config.rtp.flexfec.protected_media_ssrcs.size() > 1) { - RTC_LOG(LS_WARNING) - << "The supplied FlexfecConfig contained multiple protected " - "media streams, but our implementation currently only " - "supports protecting a single media stream. " - "To avoid confusion, disabling FlexFEC completely."; - return nullptr; - } - - const RtpState* rtp_state = nullptr; - auto it = suspended_ssrcs.find(config.rtp.flexfec.ssrc); - if (it != suspended_ssrcs.end()) { - rtp_state = &it->second; - } - - RTC_DCHECK_EQ(1U, config.rtp.flexfec.protected_media_ssrcs.size()); - return absl::make_unique( - config.rtp.flexfec.payload_type, config.rtp.flexfec.ssrc, - config.rtp.flexfec.protected_media_ssrcs[0], config.rtp.mid, - config.rtp.extensions, RTPSender::FecExtensionSizes(), rtp_state, - Clock::GetRealTimeClock()); -} - bool TransportSeqNumExtensionConfigured(const VideoSendStream::Config& config) { const std::vector& extensions = config.rtp.extensions; return std::find_if( @@ -180,14 +76,6 @@ int GetEncoderMinBitrateBps() { kDefaultEncoderMinBitrateBps); } -bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) { - const VideoCodecType codecType = PayloadStringToCodecType(payload_name); - if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) { - return true; - } - return false; -} - int CalculateMaxPadBitrateBps(std::vector streams, int min_transmit_bitrate_bps, bool pad_to_min_bitrate) { @@ -223,7 +111,34 @@ int CalculatePacketRate(uint32_t bitrate_bps, size_t packet_size_bytes) { return static_cast((bitrate_bps + packet_size_bits - 1) / packet_size_bits); } - +// call_stats, +// &encoder_feedback_, +// stats_proxy_, +// stats_proxy_, +// stats_proxy_, +// stats_proxy_, +// stats_proxy_, +// stats_proxy_, +// send_delay_stats, +// this +RtpSenderObservers CreateObservers(CallStats* call_stats, + EncoderRtcpFeedback* encoder_feedback, + SendStatisticsProxy* stats_proxy, + SendDelayStats* send_delay_stats, + OverheadObserver* overhead_observer) { + RtpSenderObservers observers; + observers.rtcp_rtt_stats = call_stats; + observers.intra_frame_callback = encoder_feedback; + observers.rtcp_stats = stats_proxy; + observers.rtp_stats = stats_proxy; + observers.bitrate_observer = stats_proxy; + observers.frame_count_observer = stats_proxy; + observers.rtcp_type_observer = stats_proxy; + observers.send_delay_observer = stats_proxy; + observers.send_packet_observer = send_delay_stats; + observers.overhead_observer = overhead_observer; + return observers; +} } // namespace // CheckEncoderActivityTask is used for tracking when the encoder last produced @@ -293,21 +208,17 @@ VideoSendStreamImpl::VideoSendStreamImpl( std::map suspended_ssrcs, std::map suspended_payload_states, VideoEncoderConfig::ContentType content_type, - std::unique_ptr fec_controller, - RateLimiter* retransmission_limiter) + std::unique_ptr fec_controller) : send_side_bwe_with_overhead_( webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), stats_proxy_(stats_proxy), config_(config), - suspended_ssrcs_(std::move(suspended_ssrcs)), fec_controller_(std::move(fec_controller)), - module_process_thread_(nullptr), worker_queue_(worker_queue), check_encoder_activity_task_(nullptr), call_stats_(call_stats), transport_(transport), bitrate_allocator_(bitrate_allocator), - flexfec_sender_(MaybeCreateFlexfecSender(*config_, suspended_ssrcs_)), max_padding_bitrate_(0), encoder_min_bitrate_bps_(0), encoder_target_rate_bps_(0), @@ -318,29 +229,25 @@ VideoSendStreamImpl::VideoSendStreamImpl( config_->rtp.ssrcs, video_stream_encoder), bandwidth_observer_(transport->GetBandwidthObserver()), - rtp_rtcp_modules_(CreateRtpRtcpModules(*config_, - &encoder_feedback_, - bandwidth_observer_, - transport, - call_stats, - flexfec_sender_.get(), - stats_proxy_, - send_delay_stats, - event_log, - retransmission_limiter, - this, - transport->keepalive_config())), - payload_router_(rtp_rtcp_modules_, - config_->rtp.ssrcs, - config_->rtp.payload_type, - suspended_payload_states), + payload_router_( + transport_->CreateVideoRtpSender(config_->rtp.ssrcs, + suspended_ssrcs, + suspended_payload_states, + config_->rtp, + config_->rtcp, + config_->send_transport, + CreateObservers(call_stats, + &encoder_feedback_, + stats_proxy_, + send_delay_stats, + this), + event_log)), weak_ptr_factory_(this), overhead_bytes_per_packet_(0), transport_overhead_bytes_per_packet_(0) { RTC_DCHECK_RUN_ON(worker_queue_); RTC_LOG(LS_INFO) << "VideoSendStreamInternal: " << config_->ToString(); weak_ptr_ = weak_ptr_factory_.GetWeakPtr(); - module_process_thread_checker_.DetachFromThread(); RTC_DCHECK(!config_->rtp.ssrcs.empty()); RTC_DCHECK(call_stats_); @@ -395,48 +302,10 @@ VideoSendStreamImpl::VideoSendStreamImpl( transport->EnablePeriodicAlrProbing(true); } - // RTP/RTCP initialization. - - // We add the highest spatial layer first to ensure it'll be prioritized - // when sending padding, with the hope that the packet rate will be smaller, - // and that it's more important to protect than the lower layers. - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - constexpr bool remb_candidate = true; - transport->packet_router()->AddSendRtpModule(rtp_rtcp, remb_candidate); - } - - for (size_t i = 0; i < config_->rtp.extensions.size(); ++i) { - const std::string& extension = config_->rtp.extensions[i].uri; - int id = config_->rtp.extensions[i].id; - // One-byte-extension local identifiers are in the range 1-14 inclusive. - RTC_DCHECK_GE(id, 1); - RTC_DCHECK_LE(id, 14); - RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension)); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - RTC_CHECK_EQ(0, rtp_rtcp->RegisterSendRtpHeaderExtension( - StringToRtpExtensionType(extension), id)); - } - } - - ConfigureProtection(); - ConfigureSsrcs(); - - if (!config_->rtp.mid.empty()) { - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetMid(config_->rtp.mid); - } - } - - // TODO(pbos): Should we set CNAME on all RTP modules? - rtp_rtcp_modules_.front()->SetCNAME(config_->rtp.c_name.c_str()); - - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->RegisterRtcpStatisticsCallback(stats_proxy_); - rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(stats_proxy_); - rtp_rtcp->SetMaxRtpPacketSize(config_->rtp.max_packet_size); - rtp_rtcp->RegisterVideoSendPayload(config_->rtp.payload_type, - config_->rtp.payload_name.c_str()); - } + // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, + // so enable that logic if either of those FEC schemes are enabled. + fec_controller_->SetProtectionMethod(payload_router_->FecEnabled(), + payload_router_->NackEnabled()); fec_controller_->SetProtectionCallback(this); // Signal congestion controller this object is ready for OnPacket* callbacks. @@ -464,55 +333,42 @@ VideoSendStreamImpl::VideoSendStreamImpl( video_stream_encoder_->SetSink(this, rotation_applied); } -void VideoSendStreamImpl::RegisterProcessThread( - ProcessThread* module_process_thread) { - RTC_DCHECK_RUN_ON(&module_process_thread_checker_); - RTC_DCHECK(!module_process_thread_); - module_process_thread_ = module_process_thread; - - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) - module_process_thread_->RegisterModule(rtp_rtcp, RTC_FROM_HERE); -} - -void VideoSendStreamImpl::DeRegisterProcessThread() { - RTC_DCHECK_RUN_ON(&module_process_thread_checker_); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) - module_process_thread_->DeRegisterModule(rtp_rtcp); -} - VideoSendStreamImpl::~VideoSendStreamImpl() { RTC_DCHECK_RUN_ON(worker_queue_); - RTC_DCHECK(!payload_router_.IsActive()) + RTC_DCHECK(!payload_router_->IsActive()) << "VideoSendStreamImpl::Stop not called"; RTC_LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString(); if (fec_controller_->UseLossVectorMask()) { transport_->DeRegisterPacketFeedbackObserver(this); } - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp); - delete rtp_rtcp; - } +} + +void VideoSendStreamImpl::RegisterProcessThread( + ProcessThread* module_process_thread) { + payload_router_->RegisterProcessThread(module_process_thread); +} + +void VideoSendStreamImpl::DeRegisterProcessThread() { + payload_router_->DeRegisterProcessThread(); } bool VideoSendStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) { // Runs on a network thread. RTC_DCHECK(!worker_queue_->IsCurrent()); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) - rtp_rtcp->IncomingRtcpPacket(packet, length); + payload_router_->DeliverRtcp(packet, length); return true; } void VideoSendStreamImpl::UpdateActiveSimulcastLayers( const std::vector active_layers) { RTC_DCHECK_RUN_ON(worker_queue_); - RTC_DCHECK_EQ(rtp_rtcp_modules_.size(), active_layers.size()); RTC_LOG(LS_INFO) << "VideoSendStream::UpdateActiveSimulcastLayers"; - bool previously_active = payload_router_.IsActive(); - payload_router_.SetActiveModules(active_layers); - if (!payload_router_.IsActive() && previously_active) { + bool previously_active = payload_router_->IsActive(); + payload_router_->SetActiveModules(active_layers); + if (!payload_router_->IsActive() && previously_active) { // Payload router switched from active to inactive. StopVideoSendStream(); - } else if (payload_router_.IsActive() && !previously_active) { + } else if (payload_router_->IsActive() && !previously_active) { // Payload router switched from inactive to active. StartupVideoSendStream(); } @@ -521,10 +377,10 @@ void VideoSendStreamImpl::UpdateActiveSimulcastLayers( void VideoSendStreamImpl::Start() { RTC_DCHECK_RUN_ON(worker_queue_); RTC_LOG(LS_INFO) << "VideoSendStream::Start"; - if (payload_router_.IsActive()) + if (payload_router_->IsActive()) return; TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Start"); - payload_router_.SetActive(true); + payload_router_->SetActive(true); StartupVideoSendStream(); } @@ -553,10 +409,10 @@ void VideoSendStreamImpl::StartupVideoSendStream() { void VideoSendStreamImpl::Stop() { RTC_DCHECK_RUN_ON(worker_queue_); RTC_LOG(LS_INFO) << "VideoSendStream::Stop"; - if (!payload_router_.IsActive()) + if (!payload_router_->IsActive()) return; TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Stop"); - payload_router_.SetActive(false); + payload_router_->SetActive(false); StopVideoSendStream(); } @@ -584,7 +440,7 @@ void VideoSendStreamImpl::SignalEncoderTimedOut() { void VideoSendStreamImpl::OnBitrateAllocationUpdated( const VideoBitrateAllocation& allocation) { - payload_router_.OnBitrateAllocationUpdated(allocation); + payload_router_->OnBitrateAllocationUpdated(allocation); } void VideoSendStreamImpl::SignalEncoderActive() { @@ -654,7 +510,7 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( num_temporal_layers, config_->rtp.max_packet_size); - if (payload_router_.IsActive()) { + if (payload_router_->IsActive()) { // The send stream is started already. Update the allocator with new bitrate // limits. bitrate_allocator_->AddObserver( @@ -691,7 +547,7 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( fec_controller_->UpdateWithEncodedData(encoded_image._length, encoded_image._frameType); - EncodedImageCallback::Result result = payload_router_.OnEncodedImage( + EncodedImageCallback::Result result = payload_router_->OnEncodedImage( encoded_image, codec_specific_info, fragmentation); RTC_DCHECK(codec_specific_info); @@ -711,152 +567,13 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage( return result; } -void VideoSendStreamImpl::ConfigureProtection() { - RTC_DCHECK_RUN_ON(worker_queue_); - - // Consistency of FlexFEC parameters is checked in MaybeCreateFlexfecSender. - const bool flexfec_enabled = (flexfec_sender_ != nullptr); - - // Consistency of NACK and RED+ULPFEC parameters is checked in this function. - const bool nack_enabled = config_->rtp.nack.rtp_history_ms > 0; - int red_payload_type = config_->rtp.ulpfec.red_payload_type; - int ulpfec_payload_type = config_->rtp.ulpfec.ulpfec_payload_type; - - // Shorthands. - auto IsRedEnabled = [&]() { return red_payload_type >= 0; }; - auto IsUlpfecEnabled = [&]() { return ulpfec_payload_type >= 0; }; - auto DisableRedAndUlpfec = [&]() { - red_payload_type = -1; - ulpfec_payload_type = -1; - }; - - if (webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment")) { - RTC_LOG(LS_INFO) << "Experiment to disable sending ULPFEC is enabled."; - DisableRedAndUlpfec(); - } - - // If enabled, FlexFEC takes priority over RED+ULPFEC. - if (flexfec_enabled) { - if (IsUlpfecEnabled()) { - RTC_LOG(LS_INFO) - << "Both FlexFEC and ULPFEC are configured. Disabling ULPFEC."; - } - DisableRedAndUlpfec(); - } - - // Payload types without picture ID cannot determine that a stream is complete - // without retransmitting FEC, so using ULPFEC + NACK for H.264 (for instance) - // is a waste of bandwidth since FEC packets still have to be transmitted. - // Note that this is not the case with FlexFEC. - if (nack_enabled && IsUlpfecEnabled() && - !PayloadTypeSupportsSkippingFecPackets(config_->rtp.payload_name)) { - RTC_LOG(LS_WARNING) - << "Transmitting payload type without picture ID using " - "NACK+ULPFEC is a waste of bandwidth since ULPFEC packets " - "also have to be retransmitted. Disabling ULPFEC."; - DisableRedAndUlpfec(); - } - - // Verify payload types. - if (IsUlpfecEnabled() ^ IsRedEnabled()) { - RTC_LOG(LS_WARNING) - << "Only RED or only ULPFEC enabled, but not both. Disabling both."; - DisableRedAndUlpfec(); - } - - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - // Set NACK. - rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize); - // Set RED/ULPFEC information. - rtp_rtcp->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); - } - - // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, - // so enable that logic if either of those FEC schemes are enabled. - fec_controller_->SetProtectionMethod(flexfec_enabled || IsUlpfecEnabled(), - nack_enabled); -} - -void VideoSendStreamImpl::ConfigureSsrcs() { - RTC_DCHECK_RUN_ON(worker_queue_); - // Configure regular SSRCs. - for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) { - uint32_t ssrc = config_->rtp.ssrcs[i]; - RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i]; - rtp_rtcp->SetSSRC(ssrc); - - // Restore RTP state if previous existed. - VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc); - if (it != suspended_ssrcs_.end()) - rtp_rtcp->SetRtpState(it->second); - } - - // Set up RTX if available. - if (config_->rtp.rtx.ssrcs.empty()) - return; - - // Configure RTX SSRCs. - RTC_DCHECK_EQ(config_->rtp.rtx.ssrcs.size(), config_->rtp.ssrcs.size()); - for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) { - uint32_t ssrc = config_->rtp.rtx.ssrcs[i]; - RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i]; - rtp_rtcp->SetRtxSsrc(ssrc); - VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc); - if (it != suspended_ssrcs_.end()) - rtp_rtcp->SetRtxState(it->second); - } - - // Configure RTX payload types. - RTC_DCHECK_GE(config_->rtp.rtx.payload_type, 0); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetRtxSendPayloadType(config_->rtp.rtx.payload_type, - config_->rtp.payload_type); - rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads); - } - if (config_->rtp.ulpfec.red_payload_type != -1 && - config_->rtp.ulpfec.red_rtx_payload_type != -1) { - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetRtxSendPayloadType(config_->rtp.ulpfec.red_rtx_payload_type, - config_->rtp.ulpfec.red_payload_type); - } - } -} - std::map VideoSendStreamImpl::GetRtpStates() const { - RTC_DCHECK_RUN_ON(worker_queue_); - std::map rtp_states; - - for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) { - uint32_t ssrc = config_->rtp.ssrcs[i]; - RTC_DCHECK_EQ(ssrc, rtp_rtcp_modules_[i]->SSRC()); - rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtpState(); - } - - for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) { - uint32_t ssrc = config_->rtp.rtx.ssrcs[i]; - rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtxState(); - } - - if (flexfec_sender_) { - uint32_t ssrc = config_->rtp.flexfec.ssrc; - rtp_states[ssrc] = flexfec_sender_->GetRtpState(); - } - - return rtp_states; + return payload_router_->GetRtpStates(); } std::map VideoSendStreamImpl::GetRtpPayloadStates() const { - RTC_DCHECK_RUN_ON(worker_queue_); - return payload_router_.GetRtpPayloadStates(); -} - -void VideoSendStreamImpl::SignalNetworkState(NetworkState state) { - RTC_DCHECK_RUN_ON(worker_queue_); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetRTCPStatus(state == kNetworkUp ? config_->rtp.rtcp_mode - : RtcpMode::kOff); - } + return payload_router_->GetRtpPayloadStates(); } uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps, @@ -864,7 +581,7 @@ uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps, int64_t rtt, int64_t probing_interval_ms) { RTC_DCHECK_RUN_ON(worker_queue_); - RTC_DCHECK(payload_router_.IsActive()) + RTC_DCHECK(payload_router_->IsActive()) << "VideoSendStream::Start has not been called."; // Substract overhead from bitrate. @@ -939,21 +656,9 @@ int VideoSendStreamImpl::ProtectionRequest( uint32_t* sent_nack_rate_bps, uint32_t* sent_fec_rate_bps) { RTC_DCHECK_RUN_ON(worker_queue_); - *sent_video_rate_bps = 0; - *sent_nack_rate_bps = 0; - *sent_fec_rate_bps = 0; - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - uint32_t not_used = 0; - uint32_t module_video_rate = 0; - uint32_t module_fec_rate = 0; - uint32_t module_nack_rate = 0; - rtp_rtcp->SetFecParameters(*delta_params, *key_params); - rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate, - &module_nack_rate); - *sent_video_rate_bps += module_video_rate; - *sent_nack_rate_bps += module_nack_rate; - *sent_fec_rate_bps += module_fec_rate; - } + payload_router_->ProtectionRequest(delta_params, key_params, + sent_video_rate_bps, sent_nack_rate_bps, + sent_fec_rate_bps); return 0; } @@ -975,9 +680,7 @@ void VideoSendStreamImpl::SetTransportOverhead( std::min(config_->rtp.max_packet_size, kPathMTU - transport_overhead_bytes_per_packet_); - for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { - rtp_rtcp->SetMaxRtpPacketSize(rtp_packet_size); - } + payload_router_->SetMaxRtpPacketSize(rtp_packet_size); } void VideoSendStreamImpl::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) { diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h index a4a9078dc9..ae2e4f4624 100644 --- a/video/video_send_stream_impl.h +++ b/video/video_send_stream_impl.h @@ -19,7 +19,6 @@ #include "call/payload_router.h" #include "common_types.h" // NOLINT(build/include) #include "common_video/include/video_bitrate_allocator.h" -#include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/utility/include/process_thread.h" #include "modules/video_coding/utility/ivf_file_writer.h" #include "rtc_base/weak_ptr.h" @@ -62,8 +61,7 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, std::map suspended_ssrcs, std::map suspended_payload_states, VideoEncoderConfig::ContentType content_type, - std::unique_ptr fec_controller, - RateLimiter* retransmission_limiter); + std::unique_ptr fec_controller); ~VideoSendStreamImpl() override; // RegisterProcessThread register |module_process_thread| with those objects @@ -74,14 +72,15 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, void RegisterProcessThread(ProcessThread* module_process_thread); void DeRegisterProcessThread(); - void SignalNetworkState(NetworkState state); bool DeliverRtcp(const uint8_t* packet, size_t length); void UpdateActiveSimulcastLayers(const std::vector active_layers); void Start(); void Stop(); - VideoSendStream::RtpStateMap GetRtpStates() const; - VideoSendStream::RtpPayloadStateMap GetRtpPayloadStates() const; + // TODO(holmer): Move these to RtpTransportControllerSend. + std::map GetRtpStates() const; + + std::map GetRtpPayloadStates() const; void EnableEncodedFrameRecording(const std::vector& files, size_t byte_limit); @@ -144,11 +143,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, SendStatisticsProxy* const stats_proxy_; const VideoSendStream::Config* const config_; - std::map suspended_ssrcs_; std::unique_ptr fec_controller_; - ProcessThread* module_process_thread_; - rtc::ThreadChecker module_process_thread_checker_; rtc::TaskQueue* const worker_queue_; rtc::CriticalSection encoder_activity_crit_sect_; @@ -159,9 +155,6 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, RtpTransportControllerSendInterface* const transport_; BitrateAllocatorInterface* const bitrate_allocator_; - // TODO(brandtr): Move ownership to PayloadRouter. - std::unique_ptr flexfec_sender_; - rtc::CriticalSection ivf_writers_crit_; std::unique_ptr file_writers_[kMaxSimulcastStreams] RTC_GUARDED_BY(ivf_writers_crit_); @@ -177,9 +170,7 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, EncoderRtcpFeedback encoder_feedback_; RtcpBandwidthObserver* const bandwidth_observer_; - // RtpRtcp modules, declared here as they use other members on construction. - const std::vector rtp_rtcp_modules_; - PayloadRouter payload_router_; + VideoRtpSenderInterface* const payload_router_; // |weak_ptr_| to our self. This is used since we can not call // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy diff --git a/video/video_send_stream_impl_unittest.cc b/video/video_send_stream_impl_unittest.cc index b50dfecf06..66deb680d5 100644 --- a/video/video_send_stream_impl_unittest.cc +++ b/video/video_send_stream_impl_unittest.cc @@ -10,9 +10,11 @@ #include +#include "call/payload_router.h" #include "call/test/mock_bitrate_allocator.h" #include "call/test/mock_rtp_transport_controller_send.h" #include "logging/rtc_event_log/rtc_event_log.h" +#include "modules/utility/include/process_thread.h" #include "modules/video_coding/fec_controller_default.h" #include "rtc_base/experiments/alr_experiment.h" #include "rtc_base/task_queue_for_test.h" @@ -42,7 +44,33 @@ std::string GetAlrProbingExperimentString() { AlrExperimentSettings::kScreenshareProbingBweExperimentName) + "/1.0,2875,80,40,-60,3/"; } - +class MockPayloadRouter : public VideoRtpSenderInterface { + public: + MOCK_METHOD1(RegisterProcessThread, void(ProcessThread*)); + MOCK_METHOD0(DeRegisterProcessThread, void()); + MOCK_METHOD1(SetActive, void(bool)); + MOCK_METHOD1(SetActiveModules, void(const std::vector)); + MOCK_METHOD0(IsActive, bool()); + MOCK_METHOD1(OnNetworkAvailability, void(bool)); + MOCK_CONST_METHOD0(GetRtpStates, std::map()); + MOCK_CONST_METHOD0(GetRtpPayloadStates, + std::map()); + MOCK_CONST_METHOD0(FecEnabled, bool()); + MOCK_CONST_METHOD0(NackEnabled, bool()); + MOCK_METHOD2(DeliverRtcp, void(const uint8_t*, size_t)); + MOCK_METHOD5(ProtectionRequest, + void(const FecProtectionParams*, + const FecProtectionParams*, + uint32_t*, + uint32_t*, + uint32_t*)); + MOCK_METHOD1(SetMaxRtpPacketSize, void(size_t)); + MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&)); + MOCK_METHOD3(OnEncodedImage, + EncodedImageCallback::Result(const EncodedImage&, + const CodecSpecificInfo*, + const RTPFragmentationHeader*)); +}; } // namespace class VideoSendStreamImplTest : public ::testing::Test { @@ -51,7 +79,6 @@ class VideoSendStreamImplTest : public ::testing::Test { : clock_(1000 * 1000 * 1000), config_(&transport_), send_delay_stats_(&clock_), - retransmission_limiter_(&clock_, 1000), test_queue_("test_queue"), process_thread_(ProcessThread::Create("test_thread")), call_stats_(&clock_, process_thread_.get()), @@ -65,6 +92,15 @@ class VideoSendStreamImplTest : public ::testing::Test { .WillRepeatedly(ReturnRef(keepalive_config_)); EXPECT_CALL(transport_controller_, packet_router()) .WillRepeatedly(Return(&packet_router_)); + EXPECT_CALL(transport_controller_, + CreateVideoRtpSender(_, _, _, _, _, _, _, _)) + .WillRepeatedly(Return(&payload_router_)); + EXPECT_CALL(payload_router_, SetActive(_)) + .WillRepeatedly(testing::Invoke( + [&](bool active) { payload_router_active_ = active; })); + EXPECT_CALL(payload_router_, IsActive()) + .WillRepeatedly( + testing::Invoke([&]() { return payload_router_active_; })); } ~VideoSendStreamImplTest() {} @@ -82,8 +118,7 @@ class VideoSendStreamImplTest : public ::testing::Test { &event_log_, &config_, initial_encoder_max_bitrate, initial_encoder_bitrate_priority, suspended_ssrcs, suspended_payload_states, content_type, - absl::make_unique(&clock_), - &retransmission_limiter_); + absl::make_unique(&clock_)); } protected: @@ -91,12 +126,13 @@ class VideoSendStreamImplTest : public ::testing::Test { NiceMock transport_controller_; NiceMock bitrate_allocator_; NiceMock video_stream_encoder_; + NiceMock payload_router_; + bool payload_router_active_ = false; SimulatedClock clock_; RtcEventLogNullImpl event_log_; VideoSendStream::Config config_; SendDelayStats send_delay_stats_; - RateLimiter retransmission_limiter_; rtc::test::TaskQueueForTest test_queue_; std::unique_ptr process_thread_; CallStats call_stats_; diff --git a/video/video_stream_decoder.cc b/video/video_stream_decoder.cc index 87ce2ae71d..10af016424 100644 --- a/video/video_stream_decoder.cc +++ b/video/video_stream_decoder.cc @@ -14,7 +14,7 @@ #include #include -#include "common_video/include/frame_callback.h" +#include "call/payload_router.h" #include "modules/video_coding/video_coding_impl.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 76ae07e32d..ea4c6e288c 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -21,7 +21,6 @@ #include "api/video/video_sink_interface.h" #include "api/video/video_stream_encoder_interface.h" #include "api/video_codecs/video_encoder.h" -#include "call/call.h" #include "call/video_send_stream.h" #include "common_types.h" // NOLINT(build/include) #include "common_video/include/video_bitrate_allocator.h" @@ -63,7 +62,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, const VideoSendStream::Config::EncoderSettings& settings, rtc::VideoSinkInterface* pre_encode_callback, std::unique_ptr overuse_detector); - ~VideoStreamEncoder(); + ~VideoStreamEncoder() override; void SetSource(rtc::VideoSourceInterface* source, const DegradationPreference& degradation_preference) override; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 7324b80058..8ce5071422 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -19,6 +19,7 @@ #include "modules/video_coding/utility/default_video_bitrate_allocator.h" #include "rtc_base/fakeclock.h" #include "rtc_base/logging.h" +#include "rtc_base/refcountedobject.h" #include "system_wrappers/include/metrics_default.h" #include "system_wrappers/include/sleep.h" #include "test/encoder_proxy_factory.h"