Support shortcircuiting encoded transforms

Add a StartShortCircuiting() callback to allow clients which have
configured Encoded Transforms when creating a PeerConnection to have
all frames skip the transform. This offers a zero cost path for streams
which don't need transforms.

This is preferable to uninstalling/not installing the transform to allow
implementing the behaviour in
https://w3c.github.io/webrtc-encoded-transform/#stream-creation -
giving web apps a chance to configure transforms within a short window
(before the next JS event loop run, so usually sub-millisecond) after stream creation, without any untransformed frames passing.

Usage in Chromium: crrev.com/c/5040731

Bug: chromium:1502781
Change-Id: I803477db1df51e80bdedf6c84d2d3695b088de83
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/327601
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Tony Herre <herre@google.com>
Cr-Commit-Position: refs/heads/main@{#41184}

api/frame_transformer_interface.h

@@ -94,6 +94,12 @@ class TransformedFrameCallback : public rtc::RefCountInterface {
   virtual void OnTransformedFrame(
       std::unique_ptr<TransformableFrameInterface> frame) = 0;
 
+  // Request to no longer be called on each frame, instead having frames be
+  // sent directly to OnTransformedFrame without additional work.
+  // TODO(crbug.com/1502781): Make pure virtual once all mocks have
+  // implementations.
+  virtual void StartShortCircuiting() {}
+
  protected:
   ~TransformedFrameCallback() override = default;
 };

audio/channel_receive_frame_transformer_delegate.cc

@@ -100,9 +100,13 @@ void ChannelReceiveFrameTransformerDelegate::Transform(
     uint32_t ssrc,
     const std::string& codec_mime_type) {
   RTC_DCHECK_RUN_ON(&sequence_checker_);
-  frame_transformer_->Transform(
-      std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc,
-                                                        codec_mime_type));
+  if (short_circuit_) {
+    receive_frame_callback_(packet, header);
+  } else {
+    frame_transformer_->Transform(
+        std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc,
+                                                          codec_mime_type));
+  }
 }
 
 void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame(
@@ -114,6 +118,14 @@ void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame(
       });
 }
 
+void ChannelReceiveFrameTransformerDelegate::StartShortCircuiting() {
+  rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate(this);
+  channel_receive_thread_->PostTask([delegate = std::move(delegate)]() mutable {
+    RTC_DCHECK_RUN_ON(&delegate->sequence_checker_);
+    delegate->short_circuit_ = true;
+  });
+}
+
 void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
     std::unique_ptr<TransformableFrameInterface> frame) const {
   RTC_DCHECK_RUN_ON(&sequence_checker_);

audio/channel_receive_frame_transformer_delegate.h

@@ -56,6 +56,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
   void OnTransformedFrame(
       std::unique_ptr<TransformableFrameInterface> frame) override;
 
+  void StartShortCircuiting() override;
+
   // Delegates the call to ChannelReceive::OnReceivedPayloadData on the
   // `channel_receive_thread_`, by calling `receive_frame_callback_`.
   void ReceiveFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
@@ -70,6 +72,7 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
   rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
       RTC_GUARDED_BY(sequence_checker_);
   TaskQueueBase* const channel_receive_thread_;
+  bool short_circuit_ RTC_GUARDED_BY(sequence_checker_) = false;
 };
 
 }  // namespace webrtc

audio/channel_receive_frame_transformer_delegate_unittest.cc

@@ -150,5 +150,29 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
   rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
 }
 
+TEST(ChannelReceiveFrameTransformerDelegateTest,
+     ShortCircuitingSkipsTransform) {
+  rtc::AutoThread main_thread;
+  rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+      rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>();
+  MockChannelReceive mock_channel;
+  rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate =
+      rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>(
+          mock_channel.callback(), mock_frame_transformer,
+          rtc::Thread::Current());
+  const uint8_t data[] = {1, 2, 3, 4};
+  rtc::ArrayView<const uint8_t> packet(data, sizeof(data));
+  RTPHeader header;
+
+  delegate->StartShortCircuiting();
+  rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
+
+  // Will not call the actual transformer.
+  EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
+  // Will pass the frame straight to the channel.
+  EXPECT_CALL(mock_channel, ReceiveFrame);
+  delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
+}
+
 }  // namespace
 }  // namespace webrtc

audio/channel_send_frame_transformer_delegate.cc

@@ -137,6 +137,16 @@ void ChannelSendFrameTransformerDelegate::Transform(
     int64_t absolute_capture_timestamp_ms,
     uint32_t ssrc,
     const std::string& codec_mimetype) {
+  {
+    MutexLock lock(&send_lock_);
+    if (short_circuit_) {
+      send_frame_callback_(
+          frame_type, payload_type, rtp_timestamp,
+          rtc::ArrayView<const uint8_t>(payload_data, payload_size),
+          absolute_capture_timestamp_ms);
+      return;
+    }
+  }
   frame_transformer_->Transform(
       std::make_unique<TransformableOutgoingAudioFrame>(
           frame_type, payload_type, rtp_timestamp, payload_data, payload_size,
@@ -155,6 +165,11 @@ void ChannelSendFrameTransformerDelegate::OnTransformedFrame(
       });
 }
 
+void ChannelSendFrameTransformerDelegate::StartShortCircuiting() {
+  MutexLock lock(&send_lock_);
+  short_circuit_ = true;
+}
+
 void ChannelSendFrameTransformerDelegate::SendFrame(
     std::unique_ptr<TransformableFrameInterface> frame) const {
   MutexLock lock(&send_lock_);

audio/channel_send_frame_transformer_delegate.h

@@ -65,6 +65,8 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback {
   void OnTransformedFrame(
       std::unique_ptr<TransformableFrameInterface> frame) override;
 
+  void StartShortCircuiting() override;
+
   // Delegates the call to ChannelSend::SendRtpAudio on the `encoder_queue_`,
   // by calling `send_audio_callback_`.
   void SendFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
@@ -77,6 +79,7 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback {
   SendFrameCallback send_frame_callback_ RTC_GUARDED_BY(send_lock_);
   rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
   rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_);
+  bool short_circuit_ RTC_GUARDED_BY(send_lock_) = false;
 };
 
 std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame(

audio/channel_send_frame_transformer_delegate_unittest.cc

@@ -168,5 +168,25 @@ TEST(ChannelSendFrameTransformerDelegateTest,
   channel_queue.WaitForPreviouslyPostedTasks();
 }
 
+TEST(ChannelSendFrameTransformerDelegateTest, ShortCircuitingSkipsTransform) {
+  TaskQueueForTest channel_queue("channel_queue");
+  rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+      rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>();
+  MockChannelSend mock_channel;
+  rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate =
+      rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>(
+          mock_channel.callback(), mock_frame_transformer, &channel_queue);
+
+  delegate->StartShortCircuiting();
+
+  // Will not call the actual transformer.
+  EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
+  // Will pass the frame straight to the channel.
+  EXPECT_CALL(mock_channel, SendFrame);
+  const uint8_t data[] = {1, 2, 3, 4};
+  delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0,
+                      /*ssrc=*/0, /*mimeType=*/"audio/opus");
+}
+
 }  // namespace
 }  // namespace webrtc

modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc

@@ -18,6 +18,7 @@
 #include "api/task_queue/task_queue_factory.h"
 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
 #include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
 
 namespace webrtc {
 namespace {
@@ -147,6 +148,17 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame(
     const EncodedImage& encoded_image,
     RTPVideoHeader video_header,
     TimeDelta expected_retransmission_time) {
+  {
+    MutexLock lock(&sender_lock_);
+    if (short_circuit_) {
+      sender_->SendVideo(payload_type, codec_type, rtp_timestamp,
+                         encoded_image.CaptureTime(),
+                         *encoded_image.GetEncodedData(), encoded_image.size(),
+                         video_header, expected_retransmission_time,
+                         /*csrcs=*/{});
+      return true;
+    }
+  }
   frame_transformer_->Transform(std::make_unique<TransformableVideoSenderFrame>(
       encoded_image, video_header, payload_type, codec_type, rtp_timestamp,
       expected_retransmission_time, ssrc_,
@@ -169,6 +181,11 @@ void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame(
       });
 }
 
+void RTPSenderVideoFrameTransformerDelegate::StartShortCircuiting() {
+  MutexLock lock(&sender_lock_);
+  short_circuit_ = true;
+}
+
 void RTPSenderVideoFrameTransformerDelegate::SendVideo(
     std::unique_ptr<TransformableFrameInterface> transformed_frame) const {
   RTC_DCHECK_RUN_ON(transformation_queue_.get());

modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h

@@ -75,6 +75,8 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback {
   void OnTransformedFrame(
       std::unique_ptr<TransformableFrameInterface> frame) override;
 
+  void StartShortCircuiting() override;
+
   // Delegates the call to RTPSendVideo::SendVideo on the `encoder_queue_`.
   void SendVideo(std::unique_ptr<TransformableFrameInterface> frame) const
       RTC_RUN_ON(transformation_queue_);
@@ -107,6 +109,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback {
   // Used when the encoded frames arrives without a current task queue. This can
   // happen if a hardware encoder was used.
   std::unique_ptr<TaskQueueBase, TaskQueueDeleter> transformation_queue_;
+  bool short_circuit_ RTC_GUARDED_BY(sender_lock_) = false;
 };
 
 // Method to support cloning a Sender frame from another frame

modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc

@@ -289,5 +289,29 @@ TEST_F(RtpSenderVideoFrameTransformerDelegateTest, SettingRTPTimestamp) {
   EXPECT_EQ(video_frame.GetTimestamp(), rtp_timestamp);
 }
 
+TEST_F(RtpSenderVideoFrameTransformerDelegateTest,
+       ShortCircuitingSkipsTransform) {
+  auto delegate = rtc::make_ref_counted<RTPSenderVideoFrameTransformerDelegate>(
+      &test_sender_, frame_transformer_,
+      /*ssrc=*/1111, time_controller_.CreateTaskQueueFactory().get());
+  EXPECT_CALL(*frame_transformer_,
+              RegisterTransformedFrameSinkCallback(_, 1111));
+  delegate->Init();
+
+  delegate->StartShortCircuiting();
+
+  // Will not call the actual transformer.
+  EXPECT_CALL(*frame_transformer_, Transform).Times(0);
+  // Will pass the frame straight to the reciever.
+  EXPECT_CALL(test_sender_, SendVideo);
+
+  EncodedImage encoded_image;
+  encoded_image.SetEncodedData(EncodedImageBuffer::Create(1));
+  delegate->TransformFrame(
+      /*payload_type=*/1, VideoCodecType::kVideoCodecVP8, /*rtp_timestamp=*/2,
+      encoded_image, RTPVideoHeader(),
+      /*expected_retransmission_time=*/TimeDelta::PlusInfinity());
+}
+
 }  // namespace
 }  // namespace webrtc

modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc

@@ -17,6 +17,7 @@
 #include "absl/memory/memory.h"
 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
 #include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
 #include "rtc_base/thread.h"
 
 namespace webrtc {
@@ -119,9 +120,14 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::Reset() {
 void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame(
     std::unique_ptr<RtpFrameObject> frame) {
   RTC_DCHECK_RUN_ON(&network_sequence_checker_);
-  frame_transformer_->Transform(
-      std::make_unique<TransformableVideoReceiverFrame>(std::move(frame), ssrc_,
-                                                        receiver_));
+  if (short_circuit_) {
+    // Just pass the frame straight back.
+    receiver_->ManageFrame(std::move(frame));
+  } else {
+    frame_transformer_->Transform(
+        std::make_unique<TransformableVideoReceiverFrame>(std::move(frame),
+                                                          ssrc_, receiver_));
+  }
 }
 
 void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame(
@@ -134,6 +140,20 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame(
       });
 }
 
+void RtpVideoStreamReceiverFrameTransformerDelegate::StartShortCircuiting() {
+  rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate(
+      this);
+  network_thread_->PostTask([delegate = std::move(delegate)]() mutable {
+    delegate->StartShortCircuitingOnNetworkSequence();
+  });
+}
+
+void RtpVideoStreamReceiverFrameTransformerDelegate::
+    StartShortCircuitingOnNetworkSequence() {
+  RTC_DCHECK_RUN_ON(&network_sequence_checker_);
+  short_circuit_ = true;
+}
+
 void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
     std::unique_ptr<TransformableFrameInterface> frame) {
   RTC_DCHECK_RUN_ON(&network_sequence_checker_);

modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h

@@ -55,6 +55,8 @@ class RtpVideoStreamReceiverFrameTransformerDelegate
   void OnTransformedFrame(
       std::unique_ptr<TransformableFrameInterface> frame) override;
 
+  void StartShortCircuiting() override;
+
   // Delegates the call to RtpVideoFrameReceiver::ManageFrame on the
   // `network_thread_`.
   void ManageFrame(std::unique_ptr<TransformableFrameInterface> frame);
@@ -63,6 +65,8 @@ class RtpVideoStreamReceiverFrameTransformerDelegate
   ~RtpVideoStreamReceiverFrameTransformerDelegate() override = default;
 
  private:
+  void StartShortCircuitingOnNetworkSequence();
+
   RTC_NO_UNIQUE_ADDRESS SequenceChecker network_sequence_checker_;
   RtpVideoFrameReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_);
   rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
@@ -70,6 +74,7 @@ class RtpVideoStreamReceiverFrameTransformerDelegate
   rtc::Thread* const network_thread_;
   const uint32_t ssrc_;
   Clock* const clock_;
+  bool short_circuit_ RTC_GUARDED_BY(network_sequence_checker_) = false;
 };
 
 }  // namespace webrtc

modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc

@@ -349,5 +349,28 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
   rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
 }
 
+TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
+     ShortCircuitingSkipsTransform) {
+  rtc::AutoThread main_thread_;
+  TestRtpVideoFrameReceiver receiver;
+  auto mock_frame_transformer =
+      rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
+  SimulatedClock clock(0);
+  auto delegate =
+      rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
+          &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
+          1111);
+  delegate->Init();
+
+  delegate->StartShortCircuiting();
+  rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
+
+  // Will not call the actual transformer.
+  EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
+  // Will pass the frame straight to the reciever.
+  EXPECT_CALL(receiver, ManageFrame);
+  delegate->TransformFrame(CreateRtpFrameObject());
+}
+
 }  // namespace
 }  // namespace webrtc