diff --git a/api/BUILD.gn b/api/BUILD.gn index 010695ac33..e2d2a3b0fd 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -410,6 +410,7 @@ rtc_source_set("frame_transformer_interface") { ":scoped_refptr", "../rtc_base:refcount", "../rtc_base/system:rtc_export", + "units:time_delta", "units:timestamp", "video:encoded_frame", "video:video_frame_metadata", diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 7133efaacc..63df71cbf9 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -19,6 +19,7 @@ #include "api/array_view.h" #include "api/ref_count.h" #include "api/scoped_refptr.h" +#include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/video_frame_metadata.h" #include "rtc_base/system/rtc_export.h" @@ -99,6 +100,7 @@ class TransformableAudioFrameInterface : public TransformableFrameInterface { virtual const std::optional SequenceNumber() const = 0; + // TODO(crbug.com/391114797): Delete this function. virtual std::optional AbsoluteCaptureTimestamp() const = 0; enum class FrameType { kEmptyFrame, kAudioFrameSpeech, kAudioFrameCN }; @@ -115,6 +117,21 @@ class TransformableAudioFrameInterface : public TransformableFrameInterface { // Timestamp at which the packet has been first seen on the network interface. // Only defined for received audio packet. virtual std::optional ReceiveTime() const = 0; + + // Timestamp at which the frame was captured in the capturer system. + // The timestamp is expressed in the capturer system's clock relative to the + // NTP epoch (January 1st 1970 00:00 UTC) + // Accessible only if the absolute capture timestamp header extension is + // enabled. + virtual std::optional CaptureTime() const = 0; + + // Offset between the sender system's clock and the capturer system's clock. + // Can be used to express the capture time in the local system's clock as + // long as the local system can determine the offset between its local clock + // and the sender system's clock. + // Accessible only if the absolute capture timestamp header extension is + // enabled. + virtual std::optional SenderCaptureTimeOffset() const = 0; }; // Objects implement this interface to be notified with the transformed frame. diff --git a/api/test/mock_transformable_audio_frame.h b/api/test/mock_transformable_audio_frame.h index d2100e6dcd..762a9dc415 100644 --- a/api/test/mock_transformable_audio_frame.h +++ b/api/test/mock_transformable_audio_frame.h @@ -56,6 +56,11 @@ class MockTransformableAudioFrame : public TransformableAudioFrameInterface { MOCK_METHOD(std::optional, AudioLevel, (), (const, override)); MOCK_METHOD(std::optional, ReceiveTime, (), (const, override)); + MOCK_METHOD(std::optional, CaptureTime, (), (const, override)); + MOCK_METHOD(std::optional, + SenderCaptureTimeOffset, + (), + (const, override)); }; } // namespace webrtc diff --git a/audio/channel_receive_frame_transformer_delegate.cc b/audio/channel_receive_frame_transformer_delegate.cc index d82925a8ad..27d4161f35 100644 --- a/audio/channel_receive_frame_transformer_delegate.cc +++ b/audio/channel_receive_frame_transformer_delegate.cc @@ -22,9 +22,11 @@ #include "api/scoped_refptr.h" #include "api/sequence_checker.h" #include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "rtc_base/buffer.h" #include "rtc_base/string_encode.h" +#include "system_wrappers/include/ntp_time.h" namespace webrtc { @@ -96,6 +98,25 @@ class TransformableIncomingAudioFrame : std::optional(receive_time_); } + std::optional CaptureTime() const override { + if (header_.extension.absolute_capture_time) { + return Timestamp::Micros(UQ32x32ToInt64Us( + header_.extension.absolute_capture_time->absolute_capture_timestamp)); + } + return std::nullopt; + } + + std::optional SenderCaptureTimeOffset() const override { + if (header_.extension.absolute_capture_time && + header_.extension.absolute_capture_time + ->estimated_capture_clock_offset) { + return TimeDelta::Micros( + UQ32x32ToInt64Us(*header_.extension.absolute_capture_time + ->estimated_capture_clock_offset)); + } + return std::nullopt; + } + private: rtc::Buffer payload_; RTPHeader header_; diff --git a/audio/channel_receive_frame_transformer_delegate_unittest.cc b/audio/channel_receive_frame_transformer_delegate_unittest.cc index 32e56c45a3..dad1f63141 100644 --- a/audio/channel_receive_frame_transformer_delegate_unittest.cc +++ b/audio/channel_receive_frame_transformer_delegate_unittest.cc @@ -24,6 +24,7 @@ #include "api/test/mock_transformable_audio_frame.h" #include "api/units/timestamp.h" #include "rtc_base/thread.h" +#include "system_wrappers/include/ntp_time.h" #include "test/gmock.h" #include "test/gtest.h" @@ -193,7 +194,7 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, } TEST(ChannelReceiveFrameTransformerDelegateTest, - AudioLevelAbsentWithoutExtension) { + AudioLevelAndCaptureTimeAbsentWithoutExtension) { rtc::AutoThread main_thread; rtc::scoped_refptr mock_frame_transformer = rtc::make_ref_counted>(); @@ -223,6 +224,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, auto* audio_frame = static_cast(frame.get()); EXPECT_FALSE(audio_frame->AudioLevel()); + EXPECT_FALSE(audio_frame->CaptureTime()); + EXPECT_FALSE(audio_frame->SenderCaptureTimeOffset()); EXPECT_EQ(audio_frame->Type(), TransformableAudioFrameInterface::FrameType::kAudioFrameCN); } @@ -265,5 +268,48 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, TransformableAudioFrameInterface::FrameType::kAudioFrameSpeech); } +TEST(ChannelReceiveFrameTransformerDelegateTest, + CaptureTimePresentWithExtension) { + rtc::AutoThread main_thread; + rtc::scoped_refptr mock_frame_transformer = + rtc::make_ref_counted>(); + rtc::scoped_refptr delegate = + rtc::make_ref_counted( + /*receive_frame_callback=*/nullptr, mock_frame_transformer, + rtc::Thread::Current()); + rtc::scoped_refptr callback; + EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback) + .WillOnce(SaveArg<0>(&callback)); + delegate->Init(); + ASSERT_TRUE(callback); + + const uint8_t data[] = {1, 2, 3, 4}; + rtc::ArrayView packet(data, sizeof(data)); + Timestamp capture_time = Timestamp::Millis(1234); + TimeDelta sender_capture_time_offset = TimeDelta::Millis(56); + AbsoluteCaptureTime absolute_capture_time = { + .absolute_capture_timestamp = Int64MsToUQ32x32(capture_time.ms()), + .estimated_capture_clock_offset = + Int64MsToUQ32x32(sender_capture_time_offset.ms())}; + RTPHeader header; + header.extension.absolute_capture_time = absolute_capture_time; + + std::unique_ptr frame; + ON_CALL(*mock_frame_transformer, Transform) + .WillByDefault( + [&](std::unique_ptr transform_frame) { + frame = std::move(transform_frame); + }); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus", + kFakeReceiveTimestamp); + + EXPECT_TRUE(frame); + auto* audio_frame = + static_cast(frame.get()); + EXPECT_EQ(*audio_frame->CaptureTime(), capture_time); + EXPECT_EQ(*audio_frame->SenderCaptureTimeOffset(), + sender_capture_time_offset); +} + } // namespace } // namespace webrtc diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc index dacbfcd43a..53d8648db6 100644 --- a/audio/channel_send_frame_transformer_delegate.cc +++ b/audio/channel_send_frame_transformer_delegate.cc @@ -13,6 +13,9 @@ #include #include +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" + namespace webrtc { namespace { @@ -110,6 +113,10 @@ class TransformableOutgoingAudioFrame } std::optional ReceiveTime() const override { return std::nullopt; } + std::optional CaptureTime() const override { return std::nullopt; } + std::optional SenderCaptureTimeOffset() const override { + return std::nullopt; + } private: AudioFrameType frame_type_;