diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 91c4c64011..1c5510fb30 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -605,6 +605,7 @@ if (rtc_include_tests) { "source/rtp_sender_audio_unittest.cc", "source/rtp_sender_egress_unittest.cc", "source/rtp_sender_unittest.cc", + "source/rtp_sender_video_frame_transformer_delegate_unittest.cc", "source/rtp_sender_video_unittest.cc", "source/rtp_sequence_number_map_unittest.cc", "source/rtp_util_unittest.cc", diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index c556a3a6c0..83abbbca50 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -467,6 +467,19 @@ void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header, } } +bool RTPSenderVideo::SendVideo( + int payload_type, + absl::optional codec_type, + uint32_t rtp_timestamp, + int64_t capture_time_ms, + rtc::ArrayView payload, + RTPVideoHeader video_header, + absl::optional expected_retransmission_time_ms) { + return SendVideo(payload_type, codec_type, rtp_timestamp, capture_time_ms, + payload, video_header, expected_retransmission_time_ms, + /*csrcs=*/{}); +} + bool RTPSenderVideo::SendVideo( int payload_type, absl::optional codec_type, diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index 825209f246..d9474f3db5 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -59,7 +59,7 @@ enum RetransmissionMode : uint8_t { kConditionallyRetransmitHigherLayers = 0x8 }; -class RTPSenderVideo { +class RTPSenderVideo : public RTPVideoFrameSenderInterface { public: static constexpr int64_t kTLRateWindowSizeMs = 2500; @@ -92,6 +92,13 @@ class RTPSenderVideo { // expected_retransmission_time_ms.has_value() -> retransmission allowed. // `capture_time_ms` and `clock::CurrentTime` should be using the same epoch. // Calls to this method are assumed to be externally serialized. + bool SendVideo(int payload_type, + absl::optional codec_type, + uint32_t rtp_timestamp, + int64_t capture_time_ms, + rtc::ArrayView payload, + RTPVideoHeader video_header, + absl::optional expected_retransmission_time_ms); bool SendVideo(int payload_type, absl::optional codec_type, uint32_t rtp_timestamp, @@ -99,7 +106,7 @@ class RTPSenderVideo { rtc::ArrayView payload, RTPVideoHeader video_header, absl::optional expected_retransmission_time_ms, - std::vector csrcs = {}); + std::vector csrcs) override; bool SendEncodedImage( int payload_type, @@ -118,7 +125,7 @@ class RTPSenderVideo { // Should only be used by a RTPSenderVideoFrameTransformerDelegate and exists // to ensure correct syncronization. void SetVideoStructureAfterTransformation( - const FrameDependencyStructure* video_structure); + const FrameDependencyStructure* video_structure) override; // Sets current active VideoLayersAllocation. The allocation will be sent // using the rtp video layers allocation extension. The allocation will be @@ -129,7 +136,7 @@ class RTPSenderVideo { // Should only be used by a RTPSenderVideoFrameTransformerDelegate and exists // to ensure correct syncronization. void SetVideoLayersAllocationAfterTransformation( - VideoLayersAllocation allocation); + VideoLayersAllocation allocation) override; // Returns the current packetization overhead rate, in bps. Note that this is // the payload overhead, eg the VP8 payload headers, not the RTP headers diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index aeda625e5a..d36ecc04ef 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -16,7 +16,6 @@ #include "api/sequence_checker.h" #include "api/task_queue/task_queue_factory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" -#include "modules/rtp_rtcp/source/rtp_sender_video.h" #include "rtc_base/checks.h" namespace webrtc { @@ -117,7 +116,7 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { } // namespace RTPSenderVideoFrameTransformerDelegate::RTPSenderVideoFrameTransformerDelegate( - RTPSenderVideo* sender, + RTPVideoFrameSenderInterface* sender, rtc::scoped_refptr frame_transformer, uint32_t ssrc, std::vector csrcs, diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index 085f29bf28..a397041811 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -24,7 +24,28 @@ namespace webrtc { -class RTPSenderVideo; +// Interface for sending videoframes on an RTP connection, after a transform +// have been applied. +class RTPVideoFrameSenderInterface { + public: + virtual bool SendVideo( + int payload_type, + absl::optional codec_type, + uint32_t rtp_timestamp, + int64_t capture_time_ms, + rtc::ArrayView payload, + RTPVideoHeader video_header, + absl::optional expected_retransmission_time_ms, + std::vector csrcs) = 0; + + virtual void SetVideoStructureAfterTransformation( + const FrameDependencyStructure* video_structure) = 0; + virtual void SetVideoLayersAllocationAfterTransformation( + VideoLayersAllocation allocation) = 0; + + protected: + virtual ~RTPVideoFrameSenderInterface() = default; +}; // Delegates calls to FrameTransformerInterface to transform frames, and to // RTPSenderVideo to send the transformed frames. Ensures thread-safe access to @@ -32,7 +53,7 @@ class RTPSenderVideo; class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { public: RTPSenderVideoFrameTransformerDelegate( - RTPSenderVideo* sender, + RTPVideoFrameSenderInterface* sender, rtc::scoped_refptr frame_transformer, uint32_t ssrc, std::vector csrcs, @@ -79,7 +100,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { void EnsureEncoderQueueCreated(); mutable Mutex sender_lock_; - RTPSenderVideo* sender_ RTC_GUARDED_BY(sender_lock_); + RTPVideoFrameSenderInterface* sender_ RTC_GUARDED_BY(sender_lock_); rtc::scoped_refptr frame_transformer_; const uint32_t ssrc_; std::vector csrcs_; diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc new file mode 100644 index 0000000000..16c88704ad --- /dev/null +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" + +#include + +#include "rtc_base/event.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_frame_transformer.h" +#include "test/time_controller/simulated_time_controller.h" + +namespace webrtc { +namespace { + +using ::testing::_; +using ::testing::SaveArg; +using ::testing::WithoutArgs; + +class MockRTPVideoFrameSenderInterface : public RTPVideoFrameSenderInterface { + public: + MOCK_METHOD(bool, + SendVideo, + (int payload_type, + absl::optional codec_type, + uint32_t rtp_timestamp, + int64_t capture_time_ms, + rtc::ArrayView payload, + RTPVideoHeader video_header, + absl::optional expected_retransmission_time_ms, + std::vector csrcs), + (override)); + + MOCK_METHOD(void, + SetVideoStructureAfterTransformation, + (const FrameDependencyStructure* video_structure), + (override)); + MOCK_METHOD(void, + SetVideoLayersAllocationAfterTransformation, + (VideoLayersAllocation allocation), + (override)); +}; + +class RtpSenderVideoFrameTransformerDelegateTest : public ::testing::Test { + protected: + RtpSenderVideoFrameTransformerDelegateTest() + : frame_transformer_(rtc::make_ref_counted()), + time_controller_(Timestamp::Seconds(0)) {} + + ~RtpSenderVideoFrameTransformerDelegateTest() override = default; + + std::unique_ptr GetTransformableFrame( + rtc::scoped_refptr delegate) { + EncodedImage encoded_image; + encoded_image.SetEncodedData(EncodedImageBuffer::Create(1)); + std::unique_ptr frame = nullptr; + EXPECT_CALL(*frame_transformer_, Transform) + .WillOnce([&](std::unique_ptr + frame_to_transform) { + frame = std::move(frame_to_transform); + }); + delegate->TransformFrame( + /*payload_type=*/1, VideoCodecType::kVideoCodecVP8, /*rtp_timestamp=*/2, + encoded_image, RTPVideoHeader(), + /*expected_retransmission_time_ms=*/absl::nullopt); + return frame; + } + + MockRTPVideoFrameSenderInterface test_sender_; + rtc::scoped_refptr frame_transformer_; + GlobalSimulatedTimeController time_controller_; +}; + +TEST_F(RtpSenderVideoFrameTransformerDelegateTest, + RegisterTransformedFrameCallbackSinkOnInit) { + auto delegate = rtc::make_ref_counted( + &test_sender_, frame_transformer_, + /*ssrc=*/1111, /*csrcs=*/std::vector(), + time_controller_.CreateTaskQueueFactory().get()); + EXPECT_CALL(*frame_transformer_, + RegisterTransformedFrameSinkCallback(_, 1111)); + delegate->Init(); +} + +TEST_F(RtpSenderVideoFrameTransformerDelegateTest, + UnregisterTransformedFrameSinkCallbackOnReset) { + auto delegate = rtc::make_ref_counted( + &test_sender_, frame_transformer_, + /*ssrc=*/1111, /*csrcs=*/std::vector(), + time_controller_.CreateTaskQueueFactory().get()); + EXPECT_CALL(*frame_transformer_, + UnregisterTransformedFrameSinkCallback(1111)); + delegate->Reset(); +} + +TEST_F(RtpSenderVideoFrameTransformerDelegateTest, + TransformFrameCallsTransform) { + auto delegate = rtc::make_ref_counted( + &test_sender_, frame_transformer_, + /*ssrc=*/1111, /*csrcs=*/std::vector(), + time_controller_.CreateTaskQueueFactory().get()); + + EncodedImage encoded_image; + EXPECT_CALL(*frame_transformer_, Transform); + delegate->TransformFrame( + /*payload_type=*/1, VideoCodecType::kVideoCodecVP8, /*rtp_timestamp=*/2, + encoded_image, RTPVideoHeader(), + /*expected_retransmission_time_ms=*/absl::nullopt); +} + +TEST_F(RtpSenderVideoFrameTransformerDelegateTest, + OnTransformedFrameCallsSenderSendVideo) { + auto delegate = rtc::make_ref_counted( + &test_sender_, frame_transformer_, + /*ssrc=*/1111, /*csrcs=*/std::vector(), + time_controller_.CreateTaskQueueFactory().get()); + + rtc::scoped_refptr callback; + EXPECT_CALL(*frame_transformer_, RegisterTransformedFrameSinkCallback) + .WillOnce(SaveArg<0>(&callback)); + delegate->Init(); + ASSERT_TRUE(callback); + + std::unique_ptr frame = + GetTransformableFrame(delegate); + ASSERT_TRUE(frame); + + rtc::Event event; + EXPECT_CALL(test_sender_, SendVideo).WillOnce(WithoutArgs([&] { + event.Set(); + return true; + })); + + callback->OnTransformedFrame(std::move(frame)); + + event.Wait(TimeDelta::Seconds(1)); +} + +TEST_F(RtpSenderVideoFrameTransformerDelegateTest, CloneSenderVideoFrame) { + auto delegate = rtc::make_ref_counted( + &test_sender_, frame_transformer_, + /*ssrc=*/1111, /*csrcs=*/std::vector(), + time_controller_.CreateTaskQueueFactory().get()); + + std::unique_ptr frame = + GetTransformableFrame(delegate); + ASSERT_TRUE(frame); + + TransformableVideoFrameInterface* video_frame = + static_cast(frame.get()); + std::unique_ptr clone = + CloneSenderVideoFrame(video_frame); + + EXPECT_EQ(video_frame->IsKeyFrame(), clone->IsKeyFrame()); + EXPECT_EQ(video_frame->GetPayloadType(), clone->GetPayloadType()); + EXPECT_EQ(video_frame->GetSsrc(), clone->GetSsrc()); + EXPECT_EQ(video_frame->GetTimestamp(), clone->GetTimestamp()); + // TODO(bugs.webrtc.org/14708): Expect equality of GetMetadata() once we have + // an equality operator defined. +} + +} // namespace +} // namespace webrtc