From 9fdcfe90f1c4d93f3daed5a085a38fbef7fbaa28 Mon Sep 17 00:00:00 2001 From: Danil Chapovalov Date: Mon, 30 Aug 2021 14:42:58 +0200 Subject: [PATCH] In RtcpTransceiver add support for receiving network generic messages These message suppose to extract all information NetworkControllerInterface may need from rtcp. Bug: webrtc:8239 Change-Id: I21d9081ad147ca8abe1ae05ca7201568c6ff77d1 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/230421 Commit-Queue: Danil Chapovalov Reviewed-by: Per Kjellander Cr-Commit-Position: refs/heads/main@{#34876} --- modules/rtp_rtcp/BUILD.gn | 3 + .../rtp_rtcp/source/rtcp_transceiver_config.h | 37 ++++- .../rtp_rtcp/source/rtcp_transceiver_impl.cc | 120 ++++++++++++-- .../rtp_rtcp/source/rtcp_transceiver_impl.h | 16 +- .../source/rtcp_transceiver_impl_unittest.cc | 153 ++++++++++++++++-- 5 files changed, 299 insertions(+), 30 deletions(-) diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index d11a2196a7..f9bc1ffc01 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -370,6 +370,8 @@ rtc_library("rtcp_transceiver") { "../../api:rtp_headers", "../../api:transport_api", "../../api/task_queue", + "../../api/units:data_rate", + "../../api/units:time_delta", "../../api/units:timestamp", "../../api/video:video_bitrate_allocation", "../../rtc_base:checks", @@ -583,6 +585,7 @@ if (rtc_include_tests) { "../../api/rtc_event_log", "../../api/transport:field_trial_based_config", "../../api/transport/rtp:dependency_descriptor", + "../../api/units:data_rate", "../../api/units:data_size", "../../api/units:time_delta", "../../api/units:timestamp", diff --git a/modules/rtp_rtcp/source/rtcp_transceiver_config.h b/modules/rtp_rtcp/source/rtcp_transceiver_config.h index 5d55990668..f24a23d75a 100644 --- a/modules/rtp_rtcp/source/rtcp_transceiver_config.h +++ b/modules/rtp_rtcp/source/rtcp_transceiver_config.h @@ -13,10 +13,16 @@ #include +#include "api/array_view.h" #include "api/rtp_headers.h" #include "api/task_queue/task_queue_base.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" #include "api/video/video_bitrate_allocation.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtcp_packet/report_block.h" +#include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" #include "system_wrappers/include/clock.h" #include "system_wrappers/include/ntp_time.h" @@ -24,13 +30,32 @@ namespace webrtc { class ReceiveStatisticsProvider; class Transport; +// Interface to watch incoming rtcp packets related to the link in general. +// All message handlers have default empty implementation. This way users only +// need to implement the ones they are interested in. +// All message handles pass `receive_time` parameter, which is receive time +// of the rtcp packet that triggered the update. +class NetworkLinkRtcpObserver { + public: + virtual ~NetworkLinkRtcpObserver() = default; + + virtual void OnTransportFeedback(Timestamp receive_time, + const rtcp::TransportFeedback& feedback) {} + virtual void OnReceiverEstimatedMaxBitrate(Timestamp receive_time, + DataRate bitrate) {} + virtual void OnReportBlocks( + Timestamp receive_time, + rtc::ArrayView report_blocks) {} + virtual void OnRttUpdate(Timestamp receive_time, TimeDelta rtt) {} +}; + // Interface to watch incoming rtcp packets by media (rtp) receiver. +// All message handlers have default empty implementation. This way users only +// need to implement the ones they are interested in. class MediaReceiverRtcpObserver { public: virtual ~MediaReceiverRtcpObserver() = default; - // All message handlers have default empty implementation. This way users only - // need to implement the ones they are interested in. virtual void OnSenderReport(uint32_t sender_ssrc, NtpTime ntp_time, uint32_t rtp_time) {} @@ -75,9 +100,15 @@ struct RtcpTransceiverConfig { ReceiveStatisticsProvider* receive_statistics = nullptr; // Callback to pass result of rtt calculation. Should outlive RtcpTransceiver. - // Callbacks will be invoked on the task_queue. + // Callbacks will be invoked on the `task_queue`. + // Deprecated, rtt_observer will be deleted in favor of more generic + // `network_link_observer` RtcpRttStats* rtt_observer = nullptr; + // Should outlive RtcpTransceiver. + // Callbacks will be invoked on the `task_queue`. + NetworkLinkRtcpObserver* network_link_observer = nullptr; + // Configures if sending should // enforce compound packets: https://tools.ietf.org/html/rfc4585#section-3.1 // or allow reduced size packets: https://tools.ietf.org/html/rfc5506 diff --git a/modules/rtp_rtcp/source/rtcp_transceiver_impl.cc b/modules/rtp_rtcp/source/rtcp_transceiver_impl.cc index 0f29b4dcd0..c56515e27e 100644 --- a/modules/rtp_rtcp/source/rtcp_transceiver_impl.cc +++ b/modules/rtp_rtcp/source/rtcp_transceiver_impl.cc @@ -132,16 +132,22 @@ void RtcpTransceiverImpl::SetReadyToSend(bool ready) { void RtcpTransceiverImpl::ReceivePacket(rtc::ArrayView packet, Timestamp now) { + // Report blocks may be spread across multiple sender and receiver reports. + std::vector report_blocks; + while (!packet.empty()) { rtcp::CommonHeader rtcp_block; if (!rtcp_block.Parse(packet.data(), packet.size())) - return; + break; - HandleReceivedPacket(rtcp_block, now); + HandleReceivedPacket(rtcp_block, now, report_blocks); - // TODO(danilchap): Use packet.remove_prefix() when that function exists. packet = packet.subview(rtcp_block.packet_size()); } + + if (!report_blocks.empty()) { + ProcessReportBlocks(now, report_blocks); + } } void RtcpTransceiverImpl::SendCompoundPacket() { @@ -226,17 +232,27 @@ void RtcpTransceiverImpl::SendFullIntraRequest( void RtcpTransceiverImpl::HandleReceivedPacket( const rtcp::CommonHeader& rtcp_packet_header, - Timestamp now) { + Timestamp now, + std::vector& report_blocks) { switch (rtcp_packet_header.type()) { case rtcp::Bye::kPacketType: HandleBye(rtcp_packet_header); break; case rtcp::SenderReport::kPacketType: - HandleSenderReport(rtcp_packet_header, now); + HandleSenderReport(rtcp_packet_header, now, report_blocks); + break; + case rtcp::ReceiverReport::kPacketType: + HandleReceiverReport(rtcp_packet_header, report_blocks); break; case rtcp::ExtendedReports::kPacketType: HandleExtendedReports(rtcp_packet_header, now); break; + case rtcp::Psfb::kPacketType: + HandlePayloadSpecificFeedback(rtcp_packet_header, now); + break; + case rtcp::Rtpfb::kPacketType: + HandleRtpFeedback(rtcp_packet_header, now); + break; } } @@ -254,20 +270,65 @@ void RtcpTransceiverImpl::HandleBye( void RtcpTransceiverImpl::HandleSenderReport( const rtcp::CommonHeader& rtcp_packet_header, - Timestamp now) { + Timestamp now, + std::vector& report_blocks) { rtcp::SenderReport sender_report; if (!sender_report.Parse(rtcp_packet_header)) return; RemoteSenderState& remote_sender = remote_senders_[sender_report.sender_ssrc()]; - remote_sender.last_received_sender_report = - absl::optional({now, sender_report.ntp()}); + remote_sender.last_received_sender_report = {{now, sender_report.ntp()}}; + const auto& received_report_blocks = sender_report.report_blocks(); + report_blocks.insert(report_blocks.end(), received_report_blocks.begin(), + received_report_blocks.end()); for (MediaReceiverRtcpObserver* observer : remote_sender.observers) observer->OnSenderReport(sender_report.sender_ssrc(), sender_report.ntp(), sender_report.rtp_timestamp()); } +void RtcpTransceiverImpl::HandleReceiverReport( + const rtcp::CommonHeader& rtcp_packet_header, + std::vector& report_blocks) { + rtcp::ReceiverReport receiver_report; + if (!receiver_report.Parse(rtcp_packet_header)) { + return; + } + const auto& received_report_blocks = receiver_report.report_blocks(); + report_blocks.insert(report_blocks.end(), received_report_blocks.begin(), + received_report_blocks.end()); +} + +void RtcpTransceiverImpl::HandlePayloadSpecificFeedback( + const rtcp::CommonHeader& rtcp_packet_header, + Timestamp now) { + // Remb is the only payload specific message handled right now. + if (rtcp_packet_header.fmt() != rtcp::Psfb::kAfbMessageType || + config_.network_link_observer == nullptr) { + return; + } + rtcp::Remb remb; + if (remb.Parse(rtcp_packet_header)) { + config_.network_link_observer->OnReceiverEstimatedMaxBitrate( + now, DataRate::BitsPerSec(remb.bitrate_bps())); + } +} + +void RtcpTransceiverImpl::HandleRtpFeedback( + const rtcp::CommonHeader& rtcp_packet_header, + Timestamp now) { + // Transport feedback is the only message handled right now. + if (rtcp_packet_header.fmt() != + rtcp::TransportFeedback::kFeedbackMessageType || + config_.network_link_observer == nullptr) { + return; + } + rtcp::TransportFeedback feedback; + if (feedback.Parse(rtcp_packet_header)) { + config_.network_link_observer->OnTransportFeedback(now, feedback); + } +} + void RtcpTransceiverImpl::HandleExtendedReports( const rtcp::CommonHeader& rtcp_packet_header, Timestamp now) { @@ -284,8 +345,9 @@ void RtcpTransceiverImpl::HandleExtendedReports( } void RtcpTransceiverImpl::HandleDlrr(const rtcp::Dlrr& dlrr, Timestamp now) { - if (!config_.non_sender_rtt_measurement || config_.rtt_observer == nullptr) + if (!config_.non_sender_rtt_measurement) { return; + } // Delay and last_rr are transferred using 32bit compact ntp resolution. // Convert packet arrival time to same format through 64bit ntp format. @@ -296,10 +358,48 @@ void RtcpTransceiverImpl::HandleDlrr(const rtcp::Dlrr& dlrr, Timestamp now) { continue; uint32_t rtt_ntp = receive_time_ntp - rti.delay_since_last_rr - rti.last_rr; int64_t rtt_ms = CompactNtpRttToMs(rtt_ntp); - config_.rtt_observer->OnRttUpdate(rtt_ms); + if (config_.rtt_observer != nullptr) { + config_.rtt_observer->OnRttUpdate(rtt_ms); + } + if (config_.network_link_observer != nullptr) { + config_.network_link_observer->OnRttUpdate(now, + TimeDelta::Millis(rtt_ms)); + } } } +void RtcpTransceiverImpl::ProcessReportBlocks( + Timestamp now, + rtc::ArrayView report_blocks) { + RTC_DCHECK(!report_blocks.empty()); + if (config_.network_link_observer == nullptr) { + return; + } + // Round trip time calculated from different report blocks suppose to be about + // the same, as those blocks should be generated by the same remote sender. + // To avoid too many callbacks, this code accumulate multiple rtts into one. + TimeDelta rtt_sum = TimeDelta::Zero(); + size_t num_rtts = 0; + uint32_t receive_time_ntp = + CompactNtp(config_.clock->ConvertTimestampToNtpTime(now)); + for (const rtcp::ReportBlock& report_block : report_blocks) { + if (report_block.last_sr() == 0) { + continue; + } + + uint32_t rtt_ntp = receive_time_ntp - report_block.delay_since_last_sr() - + report_block.last_sr(); + rtt_sum += TimeDelta::Millis(CompactNtpRttToMs(rtt_ntp)); + ++num_rtts; + } + // For backward compatibility, do not report rtt based on report blocks to the + // `config_.rtt_observer` + if (num_rtts > 0) { + config_.network_link_observer->OnRttUpdate(now, rtt_sum / num_rtts); + } + config_.network_link_observer->OnReportBlocks(now, report_blocks); +} + void RtcpTransceiverImpl::HandleTargetBitrate( const rtcp::TargetBitrate& target_bitrate, uint32_t remote_ssrc) { diff --git a/modules/rtp_rtcp/source/rtcp_transceiver_impl.h b/modules/rtp_rtcp/source/rtcp_transceiver_impl.h index bcdee83e56..91dc496faf 100644 --- a/modules/rtp_rtcp/source/rtcp_transceiver_impl.h +++ b/modules/rtp_rtcp/source/rtcp_transceiver_impl.h @@ -77,17 +77,29 @@ class RtcpTransceiverImpl { struct RemoteSenderState; void HandleReceivedPacket(const rtcp::CommonHeader& rtcp_packet_header, - Timestamp now); + Timestamp now, + std::vector& report_blocks); // Individual rtcp packet handlers. void HandleBye(const rtcp::CommonHeader& rtcp_packet_header); void HandleSenderReport(const rtcp::CommonHeader& rtcp_packet_header, - Timestamp now); + Timestamp now, + std::vector& report_blocks); + void HandleReceiverReport(const rtcp::CommonHeader& rtcp_packet_header, + std::vector& report_blocks); + void HandlePayloadSpecificFeedback( + const rtcp::CommonHeader& rtcp_packet_header, + Timestamp now); + void HandleRtpFeedback(const rtcp::CommonHeader& rtcp_packet_header, + Timestamp now); void HandleExtendedReports(const rtcp::CommonHeader& rtcp_packet_header, Timestamp now); // Extended Reports blocks handlers. void HandleDlrr(const rtcp::Dlrr& dlrr, Timestamp now); void HandleTargetBitrate(const rtcp::TargetBitrate& target_bitrate, uint32_t remote_ssrc); + void ProcessReportBlocks( + Timestamp now, + rtc::ArrayView report_blocks); void ReschedulePeriodicCompoundPackets(); void SchedulePeriodicCompoundPackets(int64_t delay_ms); diff --git a/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc b/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc index 06e1083aa8..a76f36dd2f 100644 --- a/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc +++ b/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc @@ -16,6 +16,7 @@ #include "absl/memory/memory.h" #include "api/rtp_headers.h" +#include "api/units/data_rate.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/video_bitrate_allocation.h" @@ -34,6 +35,7 @@ #include "test/mock_transport.h" #include "test/rtcp_packet_parser.h" +namespace webrtc { namespace { using ::testing::_; @@ -42,31 +44,19 @@ using ::testing::NiceMock; using ::testing::Return; using ::testing::SizeIs; using ::testing::StrictMock; -using ::webrtc::CompactNtp; -using ::webrtc::CompactNtpRttToMs; -using ::webrtc::MockRtcpRttStats; -using ::webrtc::MockTransport; -using ::webrtc::NtpTime; -using ::webrtc::RtcpTransceiverConfig; -using ::webrtc::RtcpTransceiverImpl; -using ::webrtc::SaturatedUsToCompactNtp; -using ::webrtc::SimulatedClock; -using ::webrtc::TaskQueueForTest; -using ::webrtc::TimeDelta; -using ::webrtc::Timestamp; -using ::webrtc::VideoBitrateAllocation; +using ::testing::WithArg; using ::webrtc::rtcp::Bye; using ::webrtc::rtcp::CompoundPacket; using ::webrtc::rtcp::ReportBlock; using ::webrtc::rtcp::SenderReport; using ::webrtc::test::RtcpPacketParser; -class MockReceiveStatisticsProvider : public webrtc::ReceiveStatisticsProvider { +class MockReceiveStatisticsProvider : public ReceiveStatisticsProvider { public: MOCK_METHOD(std::vector, RtcpReportBlocks, (size_t), (override)); }; -class MockMediaReceiverRtcpObserver : public webrtc::MediaReceiverRtcpObserver { +class MockMediaReceiverRtcpObserver : public MediaReceiverRtcpObserver { public: MOCK_METHOD(void, OnSenderReport, (uint32_t, NtpTime, uint32_t), (override)); MOCK_METHOD(void, OnBye, (uint32_t), (override)); @@ -76,6 +66,27 @@ class MockMediaReceiverRtcpObserver : public webrtc::MediaReceiverRtcpObserver { (override)); }; +class MockNetworkLinkRtcpObserver : public NetworkLinkRtcpObserver { + public: + MOCK_METHOD(void, + OnRttUpdate, + (Timestamp receive_time, TimeDelta rtt), + (override)); + MOCK_METHOD(void, + OnTransportFeedback, + (Timestamp receive_time, const rtcp::TransportFeedback& feedback), + (override)); + MOCK_METHOD(void, + OnReceiverEstimatedMaxBitrate, + (Timestamp receive_time, DataRate bitrate), + (override)); + MOCK_METHOD(void, + OnReportBlocks, + (Timestamp receive_time, + rtc::ArrayView report_blocks), + (override)); +}; + // Since some tests will need to wait for this period, make it small to avoid // slowing tests too much. As long as there are test bots with high scheduler // granularity, small period should be ok. @@ -139,7 +150,9 @@ RtcpTransceiverConfig DefaultTestConfig() { // Test doesn't need to support all key features: Default test config returns // valid config with all features turned off. static MockTransport null_transport; + static SimulatedClock null_clock(0); RtcpTransceiverConfig config; + config.clock = &null_clock; config.outgoing_transport = &null_transport; config.schedule_periodic_compound_packets = false; config.initial_report_delay_ms = 10; @@ -1166,6 +1179,53 @@ TEST(RtcpTransceiverImplTest, CalculatesRoundTripTimeOnDlrr) { rtcp_transceiver.ReceivePacket(raw_packet, time + TimeDelta::Millis(110)); } +TEST(RtcpTransceiverImplTest, PassRttFromDlrrToLinkObserver) { + const uint32_t kSenderSsrc = 4321; + MockNetworkLinkRtcpObserver link_observer; + RtcpTransceiverConfig config = DefaultTestConfig(); + config.feedback_ssrc = kSenderSsrc; + config.network_link_observer = &link_observer; + config.non_sender_rtt_measurement = true; + RtcpTransceiverImpl rtcp_transceiver(config); + + Timestamp send_time = Timestamp::Seconds(5678); + Timestamp receive_time = send_time + TimeDelta::Millis(110); + rtcp::ReceiveTimeInfo rti; + rti.ssrc = kSenderSsrc; + rti.last_rr = CompactNtp(config.clock->ConvertTimestampToNtpTime(send_time)); + rti.delay_since_last_rr = SaturatedUsToCompactNtp(10'000); // 10ms + rtcp::ExtendedReports xr; + xr.AddDlrrItem(rti); + + EXPECT_CALL(link_observer, OnRttUpdate(receive_time, TimeDelta::Millis(100))); + rtcp_transceiver.ReceivePacket(xr.Build(), receive_time); +} + +TEST(RtcpTransceiverImplTest, CalculatesRoundTripTimeFromReportBlocks) { + MockNetworkLinkRtcpObserver link_observer; + RtcpTransceiverConfig config = DefaultTestConfig(); + config.network_link_observer = &link_observer; + RtcpTransceiverImpl rtcp_transceiver(config); + + TimeDelta rtt = TimeDelta::Millis(100); + Timestamp send_time = Timestamp::Seconds(5678); + Timestamp receive_time = send_time + TimeDelta::Millis(110); + rtcp::ReceiverReport rr; + rtcp::ReportBlock rb1; + rb1.SetLastSr(CompactNtp(config.clock->ConvertTimestampToNtpTime( + receive_time - rtt - TimeDelta::Millis(10)))); + rb1.SetDelayLastSr(SaturatedUsToCompactNtp(10'000)); // 10ms + rr.AddReportBlock(rb1); + rtcp::ReportBlock rb2; + rb2.SetLastSr(CompactNtp(config.clock->ConvertTimestampToNtpTime( + receive_time - rtt - TimeDelta::Millis(20)))); + rb2.SetDelayLastSr(SaturatedUsToCompactNtp(20'000)); // 20ms + rr.AddReportBlock(rb2); + + EXPECT_CALL(link_observer, OnRttUpdate(receive_time, rtt)); + rtcp_transceiver.ReceivePacket(rr.Build(), receive_time); +} + TEST(RtcpTransceiverImplTest, IgnoresUnknownSsrcInDlrr) { const uint32_t kSenderSsrc = 4321; const uint32_t kUnknownSsrc = 4322; @@ -1193,4 +1253,67 @@ TEST(RtcpTransceiverImplTest, IgnoresUnknownSsrcInDlrr) { rtcp_transceiver.ReceivePacket(raw_packet, time + TimeDelta::Millis(100)); } +TEST(RtcpTransceiverImplTest, ParsesTransportFeedback) { + MockNetworkLinkRtcpObserver link_observer; + RtcpTransceiverConfig config = DefaultTestConfig(); + config.network_link_observer = &link_observer; + Timestamp receive_time = Timestamp::Seconds(5678); + RtcpTransceiverImpl rtcp_transceiver(config); + + EXPECT_CALL(link_observer, OnTransportFeedback(receive_time, _)) + .WillOnce(WithArg<1>([](const rtcp::TransportFeedback& message) { + EXPECT_EQ(message.GetBaseSequence(), 321); + EXPECT_THAT(message.GetReceivedPackets(), SizeIs(2)); + })); + + rtcp::TransportFeedback tb; + tb.SetBase(/*base_sequence=*/321, /*ref_timestamp_us=*/15); + tb.AddReceivedPacket(/*base_sequence=*/321, /*timestamp_us=*/15); + tb.AddReceivedPacket(/*base_sequence=*/322, /*timestamp_us=*/17); + rtcp_transceiver.ReceivePacket(tb.Build(), receive_time); +} + +TEST(RtcpTransceiverImplTest, ParsesRemb) { + MockNetworkLinkRtcpObserver link_observer; + RtcpTransceiverConfig config = DefaultTestConfig(); + config.network_link_observer = &link_observer; + Timestamp receive_time = Timestamp::Seconds(5678); + RtcpTransceiverImpl rtcp_transceiver(config); + + EXPECT_CALL(link_observer, + OnReceiverEstimatedMaxBitrate(receive_time, + DataRate::BitsPerSec(1'234'000))); + + rtcp::Remb remb; + remb.SetBitrateBps(1'234'000); + rtcp_transceiver.ReceivePacket(remb.Build(), receive_time); +} + +TEST(RtcpTransceiverImplTest, + CombinesReportBlocksFromSenderAndRecieverReports) { + MockNetworkLinkRtcpObserver link_observer; + RtcpTransceiverConfig config = DefaultTestConfig(); + config.network_link_observer = &link_observer; + Timestamp receive_time = Timestamp::Seconds(5678); + RtcpTransceiverImpl rtcp_transceiver(config); + + // Assemble compound packet with multiple rtcp packets in it. + rtcp::CompoundPacket packet; + auto sr = std::make_unique(); + sr->SetSenderSsrc(1234); + sr->SetReportBlocks(std::vector(31)); + packet.Append(std::move(sr)); + auto rr1 = std::make_unique(); + rr1->SetReportBlocks(std::vector(31)); + packet.Append(std::move(rr1)); + auto rr2 = std::make_unique(); + rr2->SetReportBlocks(std::vector(2)); + packet.Append(std::move(rr2)); + + EXPECT_CALL(link_observer, OnReportBlocks(receive_time, SizeIs(64))); + + rtcp_transceiver.ReceivePacket(packet.Build(), receive_time); +} + } // namespace +} // namespace webrtc