Pass receive_time through frame transformer

Bug: webrtc:344347965
Change-Id: Iee5ae13487f57f2b0c98dd6fb6a14286ff317fbb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/358100
Reviewed-by: Tony Herre <herre@google.com>
Commit-Queue: Lionel Koenig <lionelk@webrtc.org>
Reviewed-by: Jakob Ivarsson‎ <jakobi@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42717}

api/BUILD.gn

@@ -1345,6 +1345,7 @@ if (rtc_include_tests) {
 
     deps = [
       ":frame_transformer_interface",
+      "../api/units:timestamp",
       "../test:test_support",
     ]
   }

api/frame_transformer_factory.cc

@@ -13,6 +13,7 @@
 #include <memory>
 
 #include "api/frame_transformer_interface.h"
+#include "audio/channel_receive_frame_transformer_delegate.h"
 #include "audio/channel_send_frame_transformer_delegate.h"
 #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h"
 #include "rtc_base/checks.h"
@@ -31,13 +32,16 @@ std::unique_ptr<TransformableVideoFrameInterface> CreateVideoReceiverFrame() {
 
 std::unique_ptr<TransformableAudioFrameInterface> CloneAudioFrame(
     TransformableAudioFrameInterface* original) {
-  // At the moment, only making sender frames is supported.
+  if (original->GetDirection() ==
+      TransformableAudioFrameInterface::Direction::kReceiver)
+    return CloneReceiverAudioFrame(original);
   return CloneSenderAudioFrame(original);
 }
 
 std::unique_ptr<TransformableVideoFrameInterface> CloneVideoFrame(
     TransformableVideoFrameInterface* original) {
-  // At the moment, only making sender frames from receiver frames is supported.
+  // At the moment, only making sender frames from receiver frames is
+  // supported.
   return CloneSenderVideoFrame(original);
 }
 

api/frame_transformer_interface.h

@@ -105,6 +105,10 @@ class TransformableAudioFrameInterface : public TransformableFrameInterface {
   // dBov. 127 represents digital silence. Only present on remote frames if
   // the audio level header extension was included.
   virtual absl::optional<uint8_t> AudioLevel() const = 0;
+
+  // Timestamp at which the packet has been first seen on the network interface.
+  // Only defined for received audio packet.
+  virtual absl::optional<Timestamp> ReceiveTime() const = 0;
 };
 
 // Objects implement this interface to be notified with the transformed frame.

api/test/mock_transformable_audio_frame.h

@@ -14,6 +14,7 @@
 #include <string>
 
 #include "api/frame_transformer_interface.h"
+#include "api/units/timestamp.h"
 #include "test/gmock.h"
 
 namespace webrtc {
@@ -50,6 +51,8 @@ class MockTransformableAudioFrame : public TransformableAudioFrameInterface {
               (),
               (const, override));
   MOCK_METHOD(absl::optional<uint8_t>, AudioLevel, (), (const, override));
+
+  MOCK_METHOD(absl::optional<Timestamp>, ReceiveTime, (), (const, override));
 };
 
 }  // namespace webrtc

audio/BUILD.gn

@@ -65,6 +65,7 @@ rtc_library("audio") {
     "../api/task_queue:pending_task_safety_flag",
     "../api/transport/rtp:rtp_source",
     "../api/units:time_delta",
+    "../api/units:timestamp",
     "../call:audio_sender_interface",
     "../call:bitrate_allocator",
     "../call:call_interfaces",
@@ -162,6 +163,7 @@ if (rtc_include_tests) {
       ":audio_end_to_end_test",
       ":channel_receive_unittest",
       "../api:array_view",
+      "../api:frame_transformer_factory",
       "../api:frame_transformer_interface",
       "../api:libjingle_peerconnection_api",
       "../api:make_ref_counted",

audio/channel_receive.cc

@@ -25,6 +25,7 @@
 #include "api/task_queue/pending_task_safety_flag.h"
 #include "api/task_queue/task_queue_base.h"
 #include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
 #include "audio/audio_level.h"
 #include "audio/channel_receive_frame_transformer_delegate.h"
 #include "audio/channel_send.h"
@@ -371,11 +372,10 @@ void ChannelReceive::InitFrameTransformerDelegate(
   // the delegate to receive transformed audio.
   ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback
       receive_audio_callback = [this](rtc::ArrayView<const uint8_t> packet,
-                                      const RTPHeader& header) {
+                                      const RTPHeader& header,
+                                      Timestamp receive_time) {
         RTC_DCHECK_RUN_ON(&worker_thread_checker_);
-        // TODO(lionelk): Get the receive time.
+        OnReceivedPayloadData(packet, header, receive_time);
-        OnReceivedPayloadData(packet, header,
-                              /*receive_time=*/Timestamp::MinusInfinity());
       };
   frame_transformer_delegate_ =
       rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>(
@@ -723,7 +723,7 @@ void ChannelReceive::ReceivePacket(const uint8_t* packet,
               << (it != payload_type_map_.end() ? it->second.name
                                                 : "x-unknown");
     frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_,
-                                           mime_type.str());
+                                           mime_type.str(), receive_time);
   } else {
     OnReceivedPayloadData(payload_data, header, receive_time);
   }

audio/channel_receive_frame_transformer_delegate.cc

@@ -22,7 +22,9 @@
 #include "api/scoped_refptr.h"
 #include "api/sequence_checker.h"
 #include "api/task_queue/task_queue_base.h"
+#include "api/units/timestamp.h"
 #include "rtc_base/buffer.h"
+#include "rtc_base/string_encode.h"
 
 namespace webrtc {
 
@@ -32,12 +34,14 @@ class TransformableIncomingAudioFrame
   TransformableIncomingAudioFrame(rtc::ArrayView<const uint8_t> payload,
                                   const RTPHeader& header,
                                   uint32_t ssrc,
-                                  const std::string& codec_mime_type)
+                                  const std::string& codec_mime_type,
+                                  Timestamp receive_time)
       : TransformableAudioFrameInterface(Passkey()),
         payload_(payload.data(), payload.size()),
         header_(header),
         ssrc_(ssrc),
-        codec_mime_type_(codec_mime_type) {}
+        codec_mime_type_(codec_mime_type),
+        receive_time_(receive_time) {}
   ~TransformableIncomingAudioFrame() override = default;
   rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
 
@@ -86,11 +90,18 @@ class TransformableIncomingAudioFrame
     return absl::nullopt;
   }
 
+  absl::optional<Timestamp> ReceiveTime() const override {
+    return receive_time_ == Timestamp::MinusInfinity()
+               ? absl::nullopt
+               : absl::optional<Timestamp>(receive_time_);
+  }
+
  private:
   rtc::Buffer payload_;
   RTPHeader header_;
   uint32_t ssrc_;
   std::string codec_mime_type_;
+  Timestamp receive_time_;
 };
 
 ChannelReceiveFrameTransformerDelegate::ChannelReceiveFrameTransformerDelegate(
@@ -118,14 +129,15 @@ void ChannelReceiveFrameTransformerDelegate::Transform(
     rtc::ArrayView<const uint8_t> packet,
     const RTPHeader& header,
     uint32_t ssrc,
-    const std::string& codec_mime_type) {
+    const std::string& codec_mime_type,
+    Timestamp receive_time) {
   RTC_DCHECK_RUN_ON(&sequence_checker_);
   if (short_circuit_) {
-    receive_frame_callback_(packet, header);
+    receive_frame_callback_(packet, header, receive_time);
   } else {
     frame_transformer_->Transform(
-        std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc,
+        std::make_unique<TransformableIncomingAudioFrame>(
-                                                          codec_mime_type));
+            packet, header, ssrc, codec_mime_type, receive_time));
   }
 }
 
@@ -152,11 +164,13 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
   if (!receive_frame_callback_)
     return;
 
+  auto* transformed_frame =
+      static_cast<TransformableAudioFrameInterface*>(frame.get());
+  Timestamp receive_time =
+      transformed_frame->ReceiveTime().value_or(Timestamp::MinusInfinity());
   RTPHeader header;
   if (frame->GetDirection() ==
       TransformableFrameInterface::Direction::kSender) {
-    auto* transformed_frame =
-        static_cast<TransformableAudioFrameInterface*>(frame.get());
     header.payloadType = transformed_frame->GetPayloadType();
     header.timestamp = transformed_frame->GetTimestamp();
     header.ssrc = transformed_frame->GetSsrc();
@@ -166,16 +180,16 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
           transformed_frame->AbsoluteCaptureTimestamp().value();
     }
   } else {
-    auto* transformed_frame =
+    auto* transformed_incoming_frame =
         static_cast<TransformableIncomingAudioFrame*>(frame.get());
-    header = transformed_frame->Header();
+    header = transformed_incoming_frame->Header();
   }
 
   // TODO(crbug.com/1464860): Take an explicit struct with the required
   // information rather than the RTPHeader to make it easier to
   // construct the required information when injecting transformed frames not
   // originally from this receiver.
-  receive_frame_callback_(frame->GetData(), header);
+  receive_frame_callback_(frame->GetData(), header, receive_time);
 }
 
 rtc::scoped_refptr<FrameTransformerInterface>
@@ -184,4 +198,16 @@ ChannelReceiveFrameTransformerDelegate::FrameTransformer() {
   return frame_transformer_;
 }
 
+std::unique_ptr<TransformableAudioFrameInterface> CloneReceiverAudioFrame(
+    TransformableAudioFrameInterface* original) {
+  RTC_CHECK(original->GetDirection() ==
+            TransformableFrameInterface::Direction::kReceiver);
+
+  auto* original_incoming_frame =
+      static_cast<TransformableIncomingAudioFrame*>(original);
+  return std::make_unique<TransformableIncomingAudioFrame>(
+      original->GetData(), original_incoming_frame->Header(),
+      original->GetSsrc(), original->GetMimeType(),
+      original->ReceiveTime().value_or(Timestamp::MinusInfinity()));
+}
 }  // namespace webrtc

audio/channel_receive_frame_transformer_delegate.h

@@ -18,6 +18,7 @@
 #include "api/rtp_headers.h"
 #include "api/sequence_checker.h"
 #include "api/task_queue/task_queue_base.h"
+#include "api/units/timestamp.h"
 #include "rtc_base/system/no_unique_address.h"
 #include "rtc_base/thread.h"
 
@@ -30,7 +31,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
  public:
   using ReceiveFrameCallback =
       std::function<void(rtc::ArrayView<const uint8_t> packet,
-                         const RTPHeader& header)>;
+                         const RTPHeader& header,
+                         Timestamp receive_time)>;
   ChannelReceiveFrameTransformerDelegate(
       ReceiveFrameCallback receive_frame_callback,
       rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
@@ -51,7 +53,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
   void Transform(rtc::ArrayView<const uint8_t> packet,
                  const RTPHeader& header,
                  uint32_t ssrc,
-                 const std::string& codec_mime_type);
+                 const std::string& codec_mime_type,
+                 Timestamp receive_time);
 
   // Implements TransformedFrameCallback. Can be called on any thread.
   void OnTransformedFrame(
@@ -78,5 +81,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
   bool short_circuit_ RTC_GUARDED_BY(sequence_checker_) = false;
 };
 
+std::unique_ptr<TransformableAudioFrameInterface> CloneReceiverAudioFrame(
+    TransformableAudioFrameInterface* original);
+
 }  // namespace webrtc
 #endif  // AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_

audio/channel_receive_frame_transformer_delegate_unittest.cc

@@ -15,13 +15,14 @@
 #include <utility>
 
 #include "api/array_view.h"
+#include "api/frame_transformer_factory.h"
 #include "api/frame_transformer_interface.h"
 #include "api/make_ref_counted.h"
 #include "api/rtp_headers.h"
 #include "api/scoped_refptr.h"
 #include "api/test/mock_frame_transformer.h"
 #include "api/test/mock_transformable_audio_frame.h"
-#include "audio/channel_send_frame_transformer_delegate.h"
+#include "api/units/timestamp.h"
 #include "rtc_base/thread.h"
 #include "test/gmock.h"
 #include "test/gtest.h"
@@ -34,15 +35,21 @@ using ::testing::ElementsAre;
 using ::testing::NiceMock;
 using ::testing::SaveArg;
 
+constexpr Timestamp kFakeReceiveTimestamp = Timestamp::Millis(1234567);
+
 class MockChannelReceive {
  public:
   MOCK_METHOD(void,
               ReceiveFrame,
-              (rtc::ArrayView<const uint8_t> packet, const RTPHeader& header));
+              (rtc::ArrayView<const uint8_t> packet,
+               const RTPHeader& header,
+               Timestamp receive_time));
 
   ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback callback() {
-    return [this](rtc::ArrayView<const uint8_t> packet,
+    return [this](rtc::ArrayView<const uint8_t> packet, const RTPHeader& header,
-                  const RTPHeader& header) { ReceiveFrame(packet, header); };
+                  Timestamp receive_time) {
+      ReceiveFrame(packet, header, receive_time);
+    };
   }
 };
 
@@ -100,7 +107,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
           [&callback](std::unique_ptr<TransformableFrameInterface> frame) {
             callback->OnTransformedFrame(std::move(frame));
           });
-  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
+  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
+                      kFakeReceiveTimestamp);
   rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
 }
 
@@ -125,15 +133,17 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
   const uint8_t data[] = {1, 2, 3, 4};
   rtc::ArrayView<const uint8_t> packet(data, sizeof(data));
   RTPHeader header;
-  EXPECT_CALL(mock_channel, ReceiveFrame(ElementsAre(1, 2, 3, 4), _));
+  EXPECT_CALL(mock_channel,
+              ReceiveFrame(ElementsAre(1, 2, 3, 4), _, kFakeReceiveTimestamp));
   ON_CALL(*mock_frame_transformer, Transform)
-      .WillByDefault([&callback](
+      .WillByDefault(
-                         std::unique_ptr<TransformableFrameInterface> frame) {
+          [&callback](std::unique_ptr<TransformableFrameInterface> frame) {
-        auto* transformed_frame =
+            auto* transformed_frame =
-            static_cast<TransformableAudioFrameInterface*>(frame.get());
+                static_cast<TransformableAudioFrameInterface*>(frame.get());
-        callback->OnTransformedFrame(CloneSenderAudioFrame(transformed_frame));
+            callback->OnTransformedFrame(CloneAudioFrame(transformed_frame));
-      });
+          });
-  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
+  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
+                      kFakeReceiveTimestamp);
   rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
 }
 
@@ -178,7 +188,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
   EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
   // Will pass the frame straight to the channel.
   EXPECT_CALL(mock_channel, ReceiveFrame);
-  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
+  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
+                      kFakeReceiveTimestamp);
 }
 
 TEST(ChannelReceiveFrameTransformerDelegateTest,
@@ -205,7 +216,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
           [&](std::unique_ptr<TransformableFrameInterface> transform_frame) {
             frame = std::move(transform_frame);
           });
-  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
+  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
+                      kFakeReceiveTimestamp);
 
   EXPECT_TRUE(frame);
   auto* audio_frame =
@@ -242,7 +254,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
           [&](std::unique_ptr<TransformableFrameInterface> transform_frame) {
             frame = std::move(transform_frame);
           });
-  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
+  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
+                      kFakeReceiveTimestamp);
 
   EXPECT_TRUE(frame);
   auto* audio_frame =

audio/channel_send_frame_transformer_delegate.cc

@@ -109,6 +109,10 @@ class TransformableOutgoingAudioFrame
     return audio_level_dbov_;
   }
 
+  absl::optional<Timestamp> ReceiveTime() const override {
+    return absl::nullopt;
+  }
+
  private:
   AudioFrameType frame_type_;
   uint8_t payload_type_;