diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 94f8dab448..5bbcffe28e 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -94,6 +94,12 @@ class TransformedFrameCallback : public rtc::RefCountInterface { virtual void OnTransformedFrame( std::unique_ptr frame) = 0; + // Request to no longer be called on each frame, instead having frames be + // sent directly to OnTransformedFrame without additional work. + // TODO(crbug.com/1502781): Make pure virtual once all mocks have + // implementations. + virtual void StartShortCircuiting() {} + protected: ~TransformedFrameCallback() override = default; }; diff --git a/audio/channel_receive_frame_transformer_delegate.cc b/audio/channel_receive_frame_transformer_delegate.cc index e87566b44f..3cad530936 100644 --- a/audio/channel_receive_frame_transformer_delegate.cc +++ b/audio/channel_receive_frame_transformer_delegate.cc @@ -100,9 +100,13 @@ void ChannelReceiveFrameTransformerDelegate::Transform( uint32_t ssrc, const std::string& codec_mime_type) { RTC_DCHECK_RUN_ON(&sequence_checker_); - frame_transformer_->Transform( - std::make_unique(packet, header, ssrc, - codec_mime_type)); + if (short_circuit_) { + receive_frame_callback_(packet, header); + } else { + frame_transformer_->Transform( + std::make_unique(packet, header, ssrc, + codec_mime_type)); + } } void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame( @@ -114,6 +118,14 @@ void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame( }); } +void ChannelReceiveFrameTransformerDelegate::StartShortCircuiting() { + rtc::scoped_refptr delegate(this); + channel_receive_thread_->PostTask([delegate = std::move(delegate)]() mutable { + RTC_DCHECK_RUN_ON(&delegate->sequence_checker_); + delegate->short_circuit_ = true; + }); +} + void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( std::unique_ptr frame) const { RTC_DCHECK_RUN_ON(&sequence_checker_); diff --git a/audio/channel_receive_frame_transformer_delegate.h b/audio/channel_receive_frame_transformer_delegate.h index 97bcacdeaf..ac7cef0f76 100644 --- a/audio/channel_receive_frame_transformer_delegate.h +++ b/audio/channel_receive_frame_transformer_delegate.h @@ -56,6 +56,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { void OnTransformedFrame( std::unique_ptr frame) override; + void StartShortCircuiting() override; + // Delegates the call to ChannelReceive::OnReceivedPayloadData on the // `channel_receive_thread_`, by calling `receive_frame_callback_`. void ReceiveFrame(std::unique_ptr frame) const; @@ -70,6 +72,7 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { rtc::scoped_refptr frame_transformer_ RTC_GUARDED_BY(sequence_checker_); TaskQueueBase* const channel_receive_thread_; + bool short_circuit_ RTC_GUARDED_BY(sequence_checker_) = false; }; } // namespace webrtc diff --git a/audio/channel_receive_frame_transformer_delegate_unittest.cc b/audio/channel_receive_frame_transformer_delegate_unittest.cc index 9e655cc37d..8bdf217d5a 100644 --- a/audio/channel_receive_frame_transformer_delegate_unittest.cc +++ b/audio/channel_receive_frame_transformer_delegate_unittest.cc @@ -150,5 +150,29 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } +TEST(ChannelReceiveFrameTransformerDelegateTest, + ShortCircuitingSkipsTransform) { + rtc::AutoThread main_thread; + rtc::scoped_refptr mock_frame_transformer = + rtc::make_ref_counted>(); + MockChannelReceive mock_channel; + rtc::scoped_refptr delegate = + rtc::make_ref_counted( + mock_channel.callback(), mock_frame_transformer, + rtc::Thread::Current()); + const uint8_t data[] = {1, 2, 3, 4}; + rtc::ArrayView packet(data, sizeof(data)); + RTPHeader header; + + delegate->StartShortCircuiting(); + rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); + + // Will not call the actual transformer. + EXPECT_CALL(*mock_frame_transformer, Transform).Times(0); + // Will pass the frame straight to the channel. + EXPECT_CALL(mock_channel, ReceiveFrame); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); +} + } // namespace } // namespace webrtc diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc index 4e8d9fead6..f66f29df08 100644 --- a/audio/channel_send_frame_transformer_delegate.cc +++ b/audio/channel_send_frame_transformer_delegate.cc @@ -137,6 +137,16 @@ void ChannelSendFrameTransformerDelegate::Transform( int64_t absolute_capture_timestamp_ms, uint32_t ssrc, const std::string& codec_mimetype) { + { + MutexLock lock(&send_lock_); + if (short_circuit_) { + send_frame_callback_( + frame_type, payload_type, rtp_timestamp, + rtc::ArrayView(payload_data, payload_size), + absolute_capture_timestamp_ms); + return; + } + } frame_transformer_->Transform( std::make_unique( frame_type, payload_type, rtp_timestamp, payload_data, payload_size, @@ -155,6 +165,11 @@ void ChannelSendFrameTransformerDelegate::OnTransformedFrame( }); } +void ChannelSendFrameTransformerDelegate::StartShortCircuiting() { + MutexLock lock(&send_lock_); + short_circuit_ = true; +} + void ChannelSendFrameTransformerDelegate::SendFrame( std::unique_ptr frame) const { MutexLock lock(&send_lock_); diff --git a/audio/channel_send_frame_transformer_delegate.h b/audio/channel_send_frame_transformer_delegate.h index 2306dfcc59..bcec018922 100644 --- a/audio/channel_send_frame_transformer_delegate.h +++ b/audio/channel_send_frame_transformer_delegate.h @@ -65,6 +65,8 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { void OnTransformedFrame( std::unique_ptr frame) override; + void StartShortCircuiting() override; + // Delegates the call to ChannelSend::SendRtpAudio on the `encoder_queue_`, // by calling `send_audio_callback_`. void SendFrame(std::unique_ptr frame) const; @@ -77,6 +79,7 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { SendFrameCallback send_frame_callback_ RTC_GUARDED_BY(send_lock_); rtc::scoped_refptr frame_transformer_; rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_); + bool short_circuit_ RTC_GUARDED_BY(send_lock_) = false; }; std::unique_ptr CloneSenderAudioFrame( diff --git a/audio/channel_send_frame_transformer_delegate_unittest.cc b/audio/channel_send_frame_transformer_delegate_unittest.cc index 0c8e387ce1..a47cbd59c8 100644 --- a/audio/channel_send_frame_transformer_delegate_unittest.cc +++ b/audio/channel_send_frame_transformer_delegate_unittest.cc @@ -168,5 +168,25 @@ TEST(ChannelSendFrameTransformerDelegateTest, channel_queue.WaitForPreviouslyPostedTasks(); } +TEST(ChannelSendFrameTransformerDelegateTest, ShortCircuitingSkipsTransform) { + TaskQueueForTest channel_queue("channel_queue"); + rtc::scoped_refptr mock_frame_transformer = + rtc::make_ref_counted>(); + MockChannelSend mock_channel; + rtc::scoped_refptr delegate = + rtc::make_ref_counted( + mock_channel.callback(), mock_frame_transformer, &channel_queue); + + delegate->StartShortCircuiting(); + + // Will not call the actual transformer. + EXPECT_CALL(*mock_frame_transformer, Transform).Times(0); + // Will pass the frame straight to the channel. + EXPECT_CALL(mock_channel, SendFrame); + const uint8_t data[] = {1, 2, 3, 4}; + delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0, + /*ssrc=*/0, /*mimeType=*/"audio/opus"); +} + } // namespace } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index d255ef4aa9..26152df0f0 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -18,6 +18,7 @@ #include "api/task_queue/task_queue_factory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" namespace webrtc { namespace { @@ -147,6 +148,17 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame( const EncodedImage& encoded_image, RTPVideoHeader video_header, TimeDelta expected_retransmission_time) { + { + MutexLock lock(&sender_lock_); + if (short_circuit_) { + sender_->SendVideo(payload_type, codec_type, rtp_timestamp, + encoded_image.CaptureTime(), + *encoded_image.GetEncodedData(), encoded_image.size(), + video_header, expected_retransmission_time, + /*csrcs=*/{}); + return true; + } + } frame_transformer_->Transform(std::make_unique( encoded_image, video_header, payload_type, codec_type, rtp_timestamp, expected_retransmission_time, ssrc_, @@ -169,6 +181,11 @@ void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame( }); } +void RTPSenderVideoFrameTransformerDelegate::StartShortCircuiting() { + MutexLock lock(&sender_lock_); + short_circuit_ = true; +} + void RTPSenderVideoFrameTransformerDelegate::SendVideo( std::unique_ptr transformed_frame) const { RTC_DCHECK_RUN_ON(transformation_queue_.get()); diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index a333db235a..243f22ca9f 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -75,6 +75,8 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { void OnTransformedFrame( std::unique_ptr frame) override; + void StartShortCircuiting() override; + // Delegates the call to RTPSendVideo::SendVideo on the `encoder_queue_`. void SendVideo(std::unique_ptr frame) const RTC_RUN_ON(transformation_queue_); @@ -107,6 +109,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { // Used when the encoded frames arrives without a current task queue. This can // happen if a hardware encoder was used. std::unique_ptr transformation_queue_; + bool short_circuit_ RTC_GUARDED_BY(sender_lock_) = false; }; // Method to support cloning a Sender frame from another frame diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc index a376be77b4..ad00e6449f 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc @@ -289,5 +289,29 @@ TEST_F(RtpSenderVideoFrameTransformerDelegateTest, SettingRTPTimestamp) { EXPECT_EQ(video_frame.GetTimestamp(), rtp_timestamp); } +TEST_F(RtpSenderVideoFrameTransformerDelegateTest, + ShortCircuitingSkipsTransform) { + auto delegate = rtc::make_ref_counted( + &test_sender_, frame_transformer_, + /*ssrc=*/1111, time_controller_.CreateTaskQueueFactory().get()); + EXPECT_CALL(*frame_transformer_, + RegisterTransformedFrameSinkCallback(_, 1111)); + delegate->Init(); + + delegate->StartShortCircuiting(); + + // Will not call the actual transformer. + EXPECT_CALL(*frame_transformer_, Transform).Times(0); + // Will pass the frame straight to the reciever. + EXPECT_CALL(test_sender_, SendVideo); + + EncodedImage encoded_image; + encoded_image.SetEncodedData(EncodedImageBuffer::Create(1)); + delegate->TransformFrame( + /*payload_type=*/1, VideoCodecType::kVideoCodecVP8, /*rtp_timestamp=*/2, + encoded_image, RTPVideoHeader(), + /*expected_retransmission_time=*/TimeDelta::PlusInfinity()); +} + } // namespace } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc index ddbd22e34a..fbd10c4c7b 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -17,6 +17,7 @@ #include "absl/memory/memory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" #include "rtc_base/thread.h" namespace webrtc { @@ -119,9 +120,14 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::Reset() { void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame( std::unique_ptr frame) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); - frame_transformer_->Transform( - std::make_unique(std::move(frame), ssrc_, - receiver_)); + if (short_circuit_) { + // Just pass the frame straight back. + receiver_->ManageFrame(std::move(frame)); + } else { + frame_transformer_->Transform( + std::make_unique(std::move(frame), + ssrc_, receiver_)); + } } void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( @@ -134,6 +140,20 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( }); } +void RtpVideoStreamReceiverFrameTransformerDelegate::StartShortCircuiting() { + rtc::scoped_refptr delegate( + this); + network_thread_->PostTask([delegate = std::move(delegate)]() mutable { + delegate->StartShortCircuitingOnNetworkSequence(); + }); +} + +void RtpVideoStreamReceiverFrameTransformerDelegate:: + StartShortCircuitingOnNetworkSequence() { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + short_circuit_ = true; +} + void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( std::unique_ptr frame) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h index 62a42fdddf..f08fc692dd 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h @@ -55,6 +55,8 @@ class RtpVideoStreamReceiverFrameTransformerDelegate void OnTransformedFrame( std::unique_ptr frame) override; + void StartShortCircuiting() override; + // Delegates the call to RtpVideoFrameReceiver::ManageFrame on the // `network_thread_`. void ManageFrame(std::unique_ptr frame); @@ -63,6 +65,8 @@ class RtpVideoStreamReceiverFrameTransformerDelegate ~RtpVideoStreamReceiverFrameTransformerDelegate() override = default; private: + void StartShortCircuitingOnNetworkSequence(); + RTC_NO_UNIQUE_ADDRESS SequenceChecker network_sequence_checker_; RtpVideoFrameReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_); rtc::scoped_refptr frame_transformer_ @@ -70,6 +74,7 @@ class RtpVideoStreamReceiverFrameTransformerDelegate rtc::Thread* const network_thread_; const uint32_t ssrc_; Clock* const clock_; + bool short_circuit_ RTC_GUARDED_BY(network_sequence_checker_) = false; }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc index f403c91a74..cf3062610f 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc @@ -349,5 +349,28 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } +TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, + ShortCircuitingSkipsTransform) { + rtc::AutoThread main_thread_; + TestRtpVideoFrameReceiver receiver; + auto mock_frame_transformer = + rtc::make_ref_counted>(); + SimulatedClock clock(0); + auto delegate = + rtc::make_ref_counted( + &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(), + 1111); + delegate->Init(); + + delegate->StartShortCircuiting(); + rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); + + // Will not call the actual transformer. + EXPECT_CALL(*mock_frame_transformer, Transform).Times(0); + // Will pass the frame straight to the reciever. + EXPECT_CALL(receiver, ManageFrame); + delegate->TransformFrame(CreateRtpFrameObject()); +} + } // namespace } // namespace webrtc