From 272b464e92d08d5abe36af5542f53bd6a82252c5 Mon Sep 17 00:00:00 2001 From: Tony Herre Date: Wed, 26 Apr 2023 12:01:22 +0000 Subject: [PATCH] Allow feeding a Receiver encoded videoframe into a Sender Transform Instead of crashing with a CHECK fail when an insertable stream of a Video RTPSender is given a frame from an RTPReceiver's insertable stream, construct a reasonable analogous sender frame and pass it through to be decoded. A small step towards removing the split we have between Sender and Receiver implementations of TransformableFrameInterface which just confuses users of the API. Counterpart to https://webrtc-review.googlesource.com/c/src/+/301181 in the opposite direction. Bug: chromium:1250638 Change-Id: If66da7d553f14979ff1c5b4e00bff715f58cfce0 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/303480 Reviewed-by: Stefan Holmer Commit-Queue: Tony Herre Reviewed-by: Palak Agarwal Cr-Commit-Position: refs/heads/main@{#39963} --- ...sender_video_frame_transformer_delegate.cc | 37 ++++++++----- ...deo_frame_transformer_delegate_unittest.cc | 52 +++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 34b4af0ec9..7dfd7ca4ad 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -158,21 +158,34 @@ void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame( void RTPSenderVideoFrameTransformerDelegate::SendVideo( std::unique_ptr transformed_frame) const { RTC_DCHECK_RUN_ON(transformation_queue_.get()); - RTC_CHECK_EQ(transformed_frame->GetDirection(), - TransformableFrameInterface::Direction::kSender); MutexLock lock(&sender_lock_); if (!sender_) return; - auto* transformed_video_frame = - static_cast(transformed_frame.get()); - sender_->SendVideo(transformed_video_frame->GetPayloadType(), - transformed_video_frame->GetCodecType(), - transformed_video_frame->GetTimestamp(), - transformed_video_frame->GetCaptureTimeMs(), - transformed_video_frame->GetData(), - transformed_video_frame->GetHeader(), - transformed_video_frame->GetExpectedRetransmissionTimeMs(), - transformed_video_frame->Metadata().GetCsrcs()); + if (transformed_frame->GetDirection() == + TransformableFrameInterface::Direction::kSender) { + auto* transformed_video_frame = + static_cast(transformed_frame.get()); + sender_->SendVideo( + transformed_video_frame->GetPayloadType(), + transformed_video_frame->GetCodecType(), + transformed_video_frame->GetTimestamp(), + transformed_video_frame->GetCaptureTimeMs(), + transformed_video_frame->GetData(), + transformed_video_frame->GetHeader(), + transformed_video_frame->GetExpectedRetransmissionTimeMs(), + transformed_video_frame->Metadata().GetCsrcs()); + } else { + auto* transformed_video_frame = + static_cast(transformed_frame.get()); + VideoFrameMetadata metadata = transformed_video_frame->Metadata(); + sender_->SendVideo( + transformed_video_frame->GetPayloadType(), metadata.GetCodec(), + transformed_video_frame->GetTimestamp(), + /*capture_time_ms=*/0, transformed_video_frame->GetData(), + RTPVideoHeader::FromMetadata(metadata), + /*expected_retransmission_time_ms_=*/absl::nullopt, + metadata.GetCsrcs()); + } } void RTPSenderVideoFrameTransformerDelegate::SetVideoStructureUnderLock( diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc index ad1d0b3d99..a3cd81e0d1 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc @@ -12,6 +12,7 @@ #include +#include "api/test/mock_transformable_video_frame.h" #include "rtc_base/event.h" #include "test/gmock.h" #include "test/gtest.h" @@ -22,6 +23,8 @@ namespace webrtc { namespace { using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; using ::testing::SaveArg; using ::testing::WithoutArgs; @@ -217,5 +220,54 @@ TEST_F(RtpSenderVideoFrameTransformerDelegateTest, MetadataAfterSetMetadata) { EXPECT_EQ(metadata.GetCsrcs(), actual_metadata.GetCsrcs()); } +TEST_F(RtpSenderVideoFrameTransformerDelegateTest, + ReceiverFrameConvertedToSenderFrame) { + auto delegate = rtc::make_ref_counted( + &test_sender_, frame_transformer_, + /*ssrc=*/1111, /*csrcs=*/std::vector(), + time_controller_.CreateTaskQueueFactory().get()); + + const uint8_t payload_type = 1; + const uint32_t timestamp = 2; + const std::vector frame_csrcs = {123, 456, 789}; + + auto mock_receiver_frame = + std::make_unique>(); + ON_CALL(*mock_receiver_frame, GetDirection) + .WillByDefault(Return(TransformableFrameInterface::Direction::kReceiver)); + VideoFrameMetadata metadata; + metadata.SetCodec(kVideoCodecVP8); + metadata.SetRTPVideoHeaderCodecSpecifics(RTPVideoHeaderVP8()); + metadata.SetCsrcs(frame_csrcs); + ON_CALL(*mock_receiver_frame, Metadata).WillByDefault(Return(metadata)); + rtc::ArrayView buffer = + (rtc::ArrayView)*EncodedImageBuffer::Create(1); + ON_CALL(*mock_receiver_frame, GetData).WillByDefault(Return(buffer)); + ON_CALL(*mock_receiver_frame, GetPayloadType) + .WillByDefault(Return(payload_type)); + ON_CALL(*mock_receiver_frame, GetTimestamp).WillByDefault(Return(timestamp)); + + rtc::scoped_refptr callback; + EXPECT_CALL(*frame_transformer_, RegisterTransformedFrameSinkCallback) + .WillOnce(SaveArg<0>(&callback)); + delegate->Init(); + ASSERT_TRUE(callback); + + rtc::Event event; + EXPECT_CALL(test_sender_, + SendVideo(payload_type, absl::make_optional(kVideoCodecVP8), + timestamp, /*capture_time_ms=*/0, buffer, _, + /*expected_retransmission_time_ms_=*/ + (absl::optional)absl::nullopt, frame_csrcs)) + .WillOnce(WithoutArgs([&] { + event.Set(); + return true; + })); + + callback->OnTransformedFrame(std::move(mock_receiver_frame)); + + event.Wait(TimeDelta::Seconds(1)); +} + } // namespace } // namespace webrtc