diff --git a/api/BUILD.gn b/api/BUILD.gn index c8ff1dd282..63c1326adf 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -1345,6 +1345,7 @@ if (rtc_include_tests) { deps = [ ":frame_transformer_interface", + "../api/units:timestamp", "../test:test_support", ] } diff --git a/api/frame_transformer_factory.cc b/api/frame_transformer_factory.cc index 8807301185..a619f0df53 100644 --- a/api/frame_transformer_factory.cc +++ b/api/frame_transformer_factory.cc @@ -13,6 +13,7 @@ #include #include "api/frame_transformer_interface.h" +#include "audio/channel_receive_frame_transformer_delegate.h" #include "audio/channel_send_frame_transformer_delegate.h" #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" #include "rtc_base/checks.h" @@ -31,13 +32,16 @@ std::unique_ptr CreateVideoReceiverFrame() { std::unique_ptr CloneAudioFrame( TransformableAudioFrameInterface* original) { - // At the moment, only making sender frames is supported. + if (original->GetDirection() == + TransformableAudioFrameInterface::Direction::kReceiver) + return CloneReceiverAudioFrame(original); return CloneSenderAudioFrame(original); } std::unique_ptr CloneVideoFrame( TransformableVideoFrameInterface* original) { - // At the moment, only making sender frames from receiver frames is supported. + // At the moment, only making sender frames from receiver frames is + // supported. return CloneSenderVideoFrame(original); } diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 6f632bcb72..b1f5426dd3 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -105,6 +105,10 @@ class TransformableAudioFrameInterface : public TransformableFrameInterface { // dBov. 127 represents digital silence. Only present on remote frames if // the audio level header extension was included. virtual absl::optional AudioLevel() const = 0; + + // Timestamp at which the packet has been first seen on the network interface. + // Only defined for received audio packet. + virtual absl::optional ReceiveTime() const = 0; }; // Objects implement this interface to be notified with the transformed frame. diff --git a/api/test/mock_transformable_audio_frame.h b/api/test/mock_transformable_audio_frame.h index d1f40440d6..c51d2f0920 100644 --- a/api/test/mock_transformable_audio_frame.h +++ b/api/test/mock_transformable_audio_frame.h @@ -14,6 +14,7 @@ #include #include "api/frame_transformer_interface.h" +#include "api/units/timestamp.h" #include "test/gmock.h" namespace webrtc { @@ -50,6 +51,8 @@ class MockTransformableAudioFrame : public TransformableAudioFrameInterface { (), (const, override)); MOCK_METHOD(absl::optional, AudioLevel, (), (const, override)); + + MOCK_METHOD(absl::optional, ReceiveTime, (), (const, override)); }; } // namespace webrtc diff --git a/audio/BUILD.gn b/audio/BUILD.gn index fb9e904d83..86fc82b543 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -65,6 +65,7 @@ rtc_library("audio") { "../api/task_queue:pending_task_safety_flag", "../api/transport/rtp:rtp_source", "../api/units:time_delta", + "../api/units:timestamp", "../call:audio_sender_interface", "../call:bitrate_allocator", "../call:call_interfaces", @@ -162,6 +163,7 @@ if (rtc_include_tests) { ":audio_end_to_end_test", ":channel_receive_unittest", "../api:array_view", + "../api:frame_transformer_factory", "../api:frame_transformer_interface", "../api:libjingle_peerconnection_api", "../api:make_ref_counted", diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index b2fa0bf3cc..9f2662edd0 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -25,6 +25,7 @@ #include "api/task_queue/pending_task_safety_flag.h" #include "api/task_queue/task_queue_base.h" #include "api/units/time_delta.h" +#include "api/units/timestamp.h" #include "audio/audio_level.h" #include "audio/channel_receive_frame_transformer_delegate.h" #include "audio/channel_send.h" @@ -371,11 +372,10 @@ void ChannelReceive::InitFrameTransformerDelegate( // the delegate to receive transformed audio. ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback receive_audio_callback = [this](rtc::ArrayView packet, - const RTPHeader& header) { + const RTPHeader& header, + Timestamp receive_time) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); - // TODO(lionelk): Get the receive time. - OnReceivedPayloadData(packet, header, - /*receive_time=*/Timestamp::MinusInfinity()); + OnReceivedPayloadData(packet, header, receive_time); }; frame_transformer_delegate_ = rtc::make_ref_counted( @@ -723,7 +723,7 @@ void ChannelReceive::ReceivePacket(const uint8_t* packet, << (it != payload_type_map_.end() ? it->second.name : "x-unknown"); frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_, - mime_type.str()); + mime_type.str(), receive_time); } else { OnReceivedPayloadData(payload_data, header, receive_time); } diff --git a/audio/channel_receive_frame_transformer_delegate.cc b/audio/channel_receive_frame_transformer_delegate.cc index 0b8eacf96e..795f46fcc4 100644 --- a/audio/channel_receive_frame_transformer_delegate.cc +++ b/audio/channel_receive_frame_transformer_delegate.cc @@ -22,7 +22,9 @@ #include "api/scoped_refptr.h" #include "api/sequence_checker.h" #include "api/task_queue/task_queue_base.h" +#include "api/units/timestamp.h" #include "rtc_base/buffer.h" +#include "rtc_base/string_encode.h" namespace webrtc { @@ -32,12 +34,14 @@ class TransformableIncomingAudioFrame TransformableIncomingAudioFrame(rtc::ArrayView payload, const RTPHeader& header, uint32_t ssrc, - const std::string& codec_mime_type) + const std::string& codec_mime_type, + Timestamp receive_time) : TransformableAudioFrameInterface(Passkey()), payload_(payload.data(), payload.size()), header_(header), ssrc_(ssrc), - codec_mime_type_(codec_mime_type) {} + codec_mime_type_(codec_mime_type), + receive_time_(receive_time) {} ~TransformableIncomingAudioFrame() override = default; rtc::ArrayView GetData() const override { return payload_; } @@ -86,11 +90,18 @@ class TransformableIncomingAudioFrame return absl::nullopt; } + absl::optional ReceiveTime() const override { + return receive_time_ == Timestamp::MinusInfinity() + ? absl::nullopt + : absl::optional(receive_time_); + } + private: rtc::Buffer payload_; RTPHeader header_; uint32_t ssrc_; std::string codec_mime_type_; + Timestamp receive_time_; }; ChannelReceiveFrameTransformerDelegate::ChannelReceiveFrameTransformerDelegate( @@ -118,14 +129,15 @@ void ChannelReceiveFrameTransformerDelegate::Transform( rtc::ArrayView packet, const RTPHeader& header, uint32_t ssrc, - const std::string& codec_mime_type) { + const std::string& codec_mime_type, + Timestamp receive_time) { RTC_DCHECK_RUN_ON(&sequence_checker_); if (short_circuit_) { - receive_frame_callback_(packet, header); + receive_frame_callback_(packet, header, receive_time); } else { frame_transformer_->Transform( - std::make_unique(packet, header, ssrc, - codec_mime_type)); + std::make_unique( + packet, header, ssrc, codec_mime_type, receive_time)); } } @@ -152,11 +164,13 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( if (!receive_frame_callback_) return; + auto* transformed_frame = + static_cast(frame.get()); + Timestamp receive_time = + transformed_frame->ReceiveTime().value_or(Timestamp::MinusInfinity()); RTPHeader header; if (frame->GetDirection() == TransformableFrameInterface::Direction::kSender) { - auto* transformed_frame = - static_cast(frame.get()); header.payloadType = transformed_frame->GetPayloadType(); header.timestamp = transformed_frame->GetTimestamp(); header.ssrc = transformed_frame->GetSsrc(); @@ -166,16 +180,16 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( transformed_frame->AbsoluteCaptureTimestamp().value(); } } else { - auto* transformed_frame = + auto* transformed_incoming_frame = static_cast(frame.get()); - header = transformed_frame->Header(); + header = transformed_incoming_frame->Header(); } // TODO(crbug.com/1464860): Take an explicit struct with the required // information rather than the RTPHeader to make it easier to // construct the required information when injecting transformed frames not // originally from this receiver. - receive_frame_callback_(frame->GetData(), header); + receive_frame_callback_(frame->GetData(), header, receive_time); } rtc::scoped_refptr @@ -184,4 +198,16 @@ ChannelReceiveFrameTransformerDelegate::FrameTransformer() { return frame_transformer_; } +std::unique_ptr CloneReceiverAudioFrame( + TransformableAudioFrameInterface* original) { + RTC_CHECK(original->GetDirection() == + TransformableFrameInterface::Direction::kReceiver); + + auto* original_incoming_frame = + static_cast(original); + return std::make_unique( + original->GetData(), original_incoming_frame->Header(), + original->GetSsrc(), original->GetMimeType(), + original->ReceiveTime().value_or(Timestamp::MinusInfinity())); +} } // namespace webrtc diff --git a/audio/channel_receive_frame_transformer_delegate.h b/audio/channel_receive_frame_transformer_delegate.h index 0502458c55..586b0b33f1 100644 --- a/audio/channel_receive_frame_transformer_delegate.h +++ b/audio/channel_receive_frame_transformer_delegate.h @@ -18,6 +18,7 @@ #include "api/rtp_headers.h" #include "api/sequence_checker.h" #include "api/task_queue/task_queue_base.h" +#include "api/units/timestamp.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/thread.h" @@ -30,7 +31,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { public: using ReceiveFrameCallback = std::function packet, - const RTPHeader& header)>; + const RTPHeader& header, + Timestamp receive_time)>; ChannelReceiveFrameTransformerDelegate( ReceiveFrameCallback receive_frame_callback, rtc::scoped_refptr frame_transformer, @@ -51,7 +53,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { void Transform(rtc::ArrayView packet, const RTPHeader& header, uint32_t ssrc, - const std::string& codec_mime_type); + const std::string& codec_mime_type, + Timestamp receive_time); // Implements TransformedFrameCallback. Can be called on any thread. void OnTransformedFrame( @@ -78,5 +81,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { bool short_circuit_ RTC_GUARDED_BY(sequence_checker_) = false; }; +std::unique_ptr CloneReceiverAudioFrame( + TransformableAudioFrameInterface* original); + } // namespace webrtc #endif // AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_ diff --git a/audio/channel_receive_frame_transformer_delegate_unittest.cc b/audio/channel_receive_frame_transformer_delegate_unittest.cc index 3ab2b49baf..32e56c45a3 100644 --- a/audio/channel_receive_frame_transformer_delegate_unittest.cc +++ b/audio/channel_receive_frame_transformer_delegate_unittest.cc @@ -15,13 +15,14 @@ #include #include "api/array_view.h" +#include "api/frame_transformer_factory.h" #include "api/frame_transformer_interface.h" #include "api/make_ref_counted.h" #include "api/rtp_headers.h" #include "api/scoped_refptr.h" #include "api/test/mock_frame_transformer.h" #include "api/test/mock_transformable_audio_frame.h" -#include "audio/channel_send_frame_transformer_delegate.h" +#include "api/units/timestamp.h" #include "rtc_base/thread.h" #include "test/gmock.h" #include "test/gtest.h" @@ -34,15 +35,21 @@ using ::testing::ElementsAre; using ::testing::NiceMock; using ::testing::SaveArg; +constexpr Timestamp kFakeReceiveTimestamp = Timestamp::Millis(1234567); + class MockChannelReceive { public: MOCK_METHOD(void, ReceiveFrame, - (rtc::ArrayView packet, const RTPHeader& header)); + (rtc::ArrayView packet, + const RTPHeader& header, + Timestamp receive_time)); ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback callback() { - return [this](rtc::ArrayView packet, - const RTPHeader& header) { ReceiveFrame(packet, header); }; + return [this](rtc::ArrayView packet, const RTPHeader& header, + Timestamp receive_time) { + ReceiveFrame(packet, header, receive_time); + }; } }; @@ -100,7 +107,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, [&callback](std::unique_ptr frame) { callback->OnTransformedFrame(std::move(frame)); }); - delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus", + kFakeReceiveTimestamp); rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } @@ -125,15 +133,17 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, const uint8_t data[] = {1, 2, 3, 4}; rtc::ArrayView packet(data, sizeof(data)); RTPHeader header; - EXPECT_CALL(mock_channel, ReceiveFrame(ElementsAre(1, 2, 3, 4), _)); + EXPECT_CALL(mock_channel, + ReceiveFrame(ElementsAre(1, 2, 3, 4), _, kFakeReceiveTimestamp)); ON_CALL(*mock_frame_transformer, Transform) - .WillByDefault([&callback]( - std::unique_ptr frame) { - auto* transformed_frame = - static_cast(frame.get()); - callback->OnTransformedFrame(CloneSenderAudioFrame(transformed_frame)); - }); - delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); + .WillByDefault( + [&callback](std::unique_ptr frame) { + auto* transformed_frame = + static_cast(frame.get()); + callback->OnTransformedFrame(CloneAudioFrame(transformed_frame)); + }); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus", + kFakeReceiveTimestamp); rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } @@ -178,7 +188,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, EXPECT_CALL(*mock_frame_transformer, Transform).Times(0); // Will pass the frame straight to the channel. EXPECT_CALL(mock_channel, ReceiveFrame); - delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus", + kFakeReceiveTimestamp); } TEST(ChannelReceiveFrameTransformerDelegateTest, @@ -205,7 +216,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, [&](std::unique_ptr transform_frame) { frame = std::move(transform_frame); }); - delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus", + kFakeReceiveTimestamp); EXPECT_TRUE(frame); auto* audio_frame = @@ -242,7 +254,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, [&](std::unique_ptr transform_frame) { frame = std::move(transform_frame); }); - delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus", + kFakeReceiveTimestamp); EXPECT_TRUE(frame); auto* audio_frame = diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc index ef6ec2602a..64adef94d6 100644 --- a/audio/channel_send_frame_transformer_delegate.cc +++ b/audio/channel_send_frame_transformer_delegate.cc @@ -109,6 +109,10 @@ class TransformableOutgoingAudioFrame return audio_level_dbov_; } + absl::optional ReceiveTime() const override { + return absl::nullopt; + } + private: AudioFrameType frame_type_; uint8_t payload_type_;