diff --git a/modules/rtp_rtcp/source/rtp_sender_audio.cc b/modules/rtp_rtcp/source/rtp_sender_audio.cc index 67e98f674f..4a47d33573 100644 --- a/modules/rtp_rtcp/source/rtp_sender_audio.cc +++ b/modules/rtp_rtcp/source/rtp_sender_audio.cc @@ -16,15 +16,21 @@ #include #include "absl/strings/match.h" +#include "absl/types/optional.h" #include "api/audio_codecs/audio_format.h" +#include "api/rtp_headers.h" +#include "modules/audio_coding/include/audio_coding_module_typedefs.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/rtp_rtcp/source/time_util.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/trace_event.h" +#include "system_wrappers/include/ntp_time.h" namespace webrtc { @@ -46,7 +52,9 @@ const char* FrameTypeToString(AudioFrameType frame_type) { } // namespace RTPSenderAudio::RTPSenderAudio(Clock* clock, RTPSender* rtp_sender) - : clock_(clock), rtp_sender_(rtp_sender) { + : clock_(clock), + rtp_sender_(rtp_sender), + absolute_capture_time_sender_(clock) { RTC_DCHECK(clock_); } @@ -83,6 +91,10 @@ int32_t RTPSenderAudio::RegisterAudioPayload(absl::string_view payload_name, dtmf_payload_type_ = payload_type; dtmf_payload_freq_ = frequency; return 0; + } else if (payload_name == "audio") { + rtc::CritScope cs(&send_audio_critsect_); + encoder_rtp_timestamp_frequency_ = frequency; + return 0; } return 0; } @@ -135,7 +147,19 @@ bool RTPSenderAudio::SendAudio(AudioFrameType frame_type, uint32_t rtp_timestamp, const uint8_t* payload_data, size_t payload_size) { - #if RTC_TRACE_EVENTS_ENABLED + return SendAudio(frame_type, payload_type, rtp_timestamp, payload_data, + payload_size, + // TODO(bugs.webrtc.org/10739) replace once plumbed. + /*absolute_capture_timestamp_ms=*/0); +} + +bool RTPSenderAudio::SendAudio(AudioFrameType frame_type, + int8_t payload_type, + uint32_t rtp_timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms) { +#if RTC_TRACE_EVENTS_ENABLED TRACE_EVENT_ASYNC_STEP1("webrtc", "Audio", rtp_timestamp, "Send", "type", FrameTypeToString(frame_type)); #endif @@ -148,10 +172,12 @@ bool RTPSenderAudio::SendAudio(AudioFrameType frame_type, constexpr int kDtmfIntervalTimeMs = 50; uint8_t audio_level_dbov = 0; uint32_t dtmf_payload_freq = 0; + absl::optional encoder_rtp_timestamp_frequency; { rtc::CritScope cs(&send_audio_critsect_); audio_level_dbov = audio_level_dbov_; dtmf_payload_freq = dtmf_payload_freq_; + encoder_rtp_timestamp_frequency = encoder_rtp_timestamp_frequency_; } // Check if we have pending DTMFs to send @@ -244,6 +270,23 @@ bool RTPSenderAudio::SendAudio(AudioFrameType frame_type, packet->SetExtension( frame_type == AudioFrameType::kAudioFrameSpeech, audio_level_dbov); + // Send absolute capture time periodically in order to optimize and save + // network traffic. Missing absolute capture times can be interpolated on the + // receiving end if sending intervals are small enough. + auto absolute_capture_time = absolute_capture_time_sender_.OnSendPacket( + AbsoluteCaptureTimeSender::GetSource(packet->Ssrc(), packet->Csrcs()), + packet->Timestamp(), + // Replace missing value with 0 (invalid frequency), this will trigger + // absolute capture time sending. + encoder_rtp_timestamp_frequency.value_or(0), + Int64MsToUQ32x32(absolute_capture_timestamp_ms + NtpOffsetMs()), + /*estimated_capture_clock_offset=*/absl::nullopt); + if (absolute_capture_time) { + // It also checks that extension was registered during SDP negotiation. If + // not then setter won't do anything. + packet->SetExtension(*absolute_capture_time); + } + uint8_t* payload = packet->AllocatePayload(payload_size); if (!payload) // Too large payload buffer. return false; diff --git a/modules/rtp_rtcp/source/rtp_sender_audio.h b/modules/rtp_rtcp/source/rtp_sender_audio.h index ccc26371aa..c2d8074a60 100644 --- a/modules/rtp_rtcp/source/rtp_sender_audio.h +++ b/modules/rtp_rtcp/source/rtp_sender_audio.h @@ -18,6 +18,7 @@ #include "absl/strings/string_view.h" #include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" #include "modules/rtp_rtcp/source/dtmf_queue.h" #include "modules/rtp_rtcp/source/rtp_sender.h" #include "rtc_base/constructor_magic.h" @@ -41,10 +42,17 @@ class RTPSenderAudio { bool SendAudio(AudioFrameType frame_type, int8_t payload_type, - uint32_t capture_timestamp, + uint32_t rtp_timestamp, const uint8_t* payload_data, size_t payload_size); + bool SendAudio(AudioFrameType frame_type, + int8_t payload_type, + uint32_t rtp_timestamp, + const uint8_t* payload_data, + size_t payload_size, + int64_t absolute_capture_timestamp_ms); + // Store the audio level in dBov for // header-extension-for-audio-level-indication. // Valid range is [0,100]. Actual value is negative. @@ -93,6 +101,11 @@ class RTPSenderAudio { uint8_t audio_level_dbov_ RTC_GUARDED_BY(send_audio_critsect_) = 0; OneTimeEvent first_packet_sent_; + absl::optional encoder_rtp_timestamp_frequency_ + RTC_GUARDED_BY(send_audio_critsect_); + + AbsoluteCaptureTimeSender absolute_capture_time_sender_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RTPSenderAudio); }; diff --git a/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc index f4840d1e75..3e35f42bff 100644 --- a/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc @@ -18,6 +18,7 @@ #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/time_util.h" #include "test/gmock.h" #include "test/gtest.h" @@ -26,6 +27,7 @@ namespace webrtc { namespace { enum : int { // The first valid value is 1. kAudioLevelExtensionId = 1, + kAbsoluteCaptureTimeExtensionId = 2, }; const uint16_t kSeqNum = 33; @@ -39,6 +41,8 @@ class LoopbackTransportTest : public webrtc::Transport { public: LoopbackTransportTest() { receivers_extensions_.Register(kAudioLevelExtensionId); + receivers_extensions_.Register( + kAbsoluteCaptureTimeExtensionId); } bool SendRtp(const uint8_t* data, @@ -90,7 +94,8 @@ TEST_F(RtpSenderAudioTest, SendAudio) { ASSERT_TRUE(rtp_sender_audio_.SendAudio(AudioFrameType::kAudioFrameCN, payload_type, 4321, payload, - sizeof(payload))); + sizeof(payload), + /*absolute_capture_timestamp_ms=*/0)); auto sent_payload = transport_.last_sent_packet().payload(); EXPECT_THAT(sent_payload, ElementsAreArray(payload)); @@ -110,7 +115,8 @@ TEST_F(RtpSenderAudioTest, SendAudioWithAudioLevelExtension) { ASSERT_TRUE(rtp_sender_audio_.SendAudio(AudioFrameType::kAudioFrameCN, payload_type, 4321, payload, - sizeof(payload))); + sizeof(payload), + /*absolute_capture_timestamp_ms=*/0)); auto sent_payload = transport_.last_sent_packet().payload(); EXPECT_THAT(sent_payload, ElementsAreArray(payload)); @@ -123,6 +129,44 @@ TEST_F(RtpSenderAudioTest, SendAudioWithAudioLevelExtension) { EXPECT_FALSE(voice_activity); } +TEST_F(RtpSenderAudioTest, SendAudioWithoutAbsoluteCaptureTime) { + constexpr uint32_t kAbsoluteCaptureTimestampMs = 521; + const char payload_name[] = "audio"; + const uint8_t payload_type = 127; + ASSERT_EQ(0, rtp_sender_audio_.RegisterAudioPayload( + payload_name, payload_type, 48000, 0, 1500)); + uint8_t payload[] = {47, 11, 32, 93, 89}; + + ASSERT_TRUE(rtp_sender_audio_.SendAudio( + AudioFrameType::kAudioFrameCN, payload_type, 4321, payload, + sizeof(payload), kAbsoluteCaptureTimestampMs)); + + EXPECT_FALSE(transport_.last_sent_packet() + .HasExtension()); +} + +TEST_F(RtpSenderAudioTest, SendAudioWithAbsoluteCaptureTime) { + rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::kUri, + kAbsoluteCaptureTimeExtensionId); + constexpr uint32_t kAbsoluteCaptureTimestampMs = 521; + const char payload_name[] = "audio"; + const uint8_t payload_type = 127; + ASSERT_EQ(0, rtp_sender_audio_.RegisterAudioPayload( + payload_name, payload_type, 48000, 0, 1500)); + uint8_t payload[] = {47, 11, 32, 93, 89}; + + ASSERT_TRUE(rtp_sender_audio_.SendAudio( + AudioFrameType::kAudioFrameCN, payload_type, 4321, payload, + sizeof(payload), kAbsoluteCaptureTimestampMs)); + + auto absolute_capture_time = + transport_.last_sent_packet() + .GetExtension(); + EXPECT_TRUE(absolute_capture_time); + EXPECT_EQ(absolute_capture_time->absolute_capture_timestamp, + Int64MsToUQ32x32(kAbsoluteCaptureTimestampMs + NtpOffsetMs())); +} + // As RFC4733, named telephone events are carried as part of the audio stream // and must use the same sequence number and timestamp base as the regular // audio channel. @@ -148,22 +192,25 @@ TEST_F(RtpSenderAudioTest, CheckMarkerBitForTelephoneEvents) { // During start, it takes the starting timestamp as last sent timestamp. // The duration is calculated as the difference of current and last sent // timestamp. So for first call it will skip since the duration is zero. - ASSERT_TRUE(rtp_sender_audio_.SendAudio(AudioFrameType::kEmptyFrame, - kPayloadType, capture_timestamp, - nullptr, 0)); + ASSERT_TRUE(rtp_sender_audio_.SendAudio( + AudioFrameType::kEmptyFrame, kPayloadType, capture_timestamp, nullptr, 0, + /*absolute_capture_time_ms=0*/ 0)); + // DTMF Sample Length is (Frequency/1000) * Duration. // So in this case, it is (8000/1000) * 500 = 4000. // Sending it as two packets. - ASSERT_TRUE( - rtp_sender_audio_.SendAudio(AudioFrameType::kEmptyFrame, kPayloadType, - capture_timestamp + 2000, nullptr, 0)); + ASSERT_TRUE(rtp_sender_audio_.SendAudio(AudioFrameType::kEmptyFrame, + kPayloadType, + capture_timestamp + 2000, nullptr, 0, + /*absolute_capture_time_ms=0*/ 0)); // Marker Bit should be set to 1 for first packet. EXPECT_TRUE(transport_.last_sent_packet().Marker()); - ASSERT_TRUE( - rtp_sender_audio_.SendAudio(AudioFrameType::kEmptyFrame, kPayloadType, - capture_timestamp + 4000, nullptr, 0)); + ASSERT_TRUE(rtp_sender_audio_.SendAudio(AudioFrameType::kEmptyFrame, + kPayloadType, + capture_timestamp + 4000, nullptr, 0, + /*absolute_capture_time_ms=0*/ 0)); // Marker Bit should be set to 0 for rest of the packets. EXPECT_FALSE(transport_.last_sent_packet().Marker()); }