From 78964c1e0afa60d8e628f4cac304ae550ee92490 Mon Sep 17 00:00:00 2001 From: Marina Ciocea Date: Tue, 10 Mar 2020 21:31:52 +0100 Subject: [PATCH] Transform encoded frames in RtpVideoStreamReceiver. This change is part of the implementation of the Insertable Streams Web API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md Design doc for WebRTC library changes: http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk Bug: webrtc:11380 Change-Id: If4ffcfe5761492a2ae5513ec46deb9f837e8aee8 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169130 Reviewed-by: Magnus Flodman Reviewed-by: Danil Chapovalov Reviewed-by: Rasmus Brandt Commit-Queue: Marina Ciocea Cr-Commit-Position: refs/heads/master@{#30755} --- video/BUILD.gn | 5 + video/rtp_video_stream_receiver.cc | 27 ++- video/rtp_video_stream_receiver.h | 6 +- ...eam_receiver_frame_transformer_delegate.cc | 75 +++++++ ...ream_receiver_frame_transformer_delegate.h | 64 ++++++ ...ver_frame_transformer_delegate_unittest.cc | 199 ++++++++++++++++++ video/rtp_video_stream_receiver_unittest.cc | 41 ++++ 7 files changed, 411 insertions(+), 6 deletions(-) create mode 100644 video/rtp_video_stream_receiver_frame_transformer_delegate.cc create mode 100644 video/rtp_video_stream_receiver_frame_transformer_delegate.h create mode 100644 video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc diff --git a/video/BUILD.gn b/video/BUILD.gn index 09cbca492a..a12cc036fc 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -28,6 +28,8 @@ rtc_library("video") { "rtp_streams_synchronizer.h", "rtp_video_stream_receiver.cc", "rtp_video_stream_receiver.h", + "rtp_video_stream_receiver_frame_transformer_delegate.cc", + "rtp_video_stream_receiver_frame_transformer_delegate.h", "send_delay_stats.cc", "send_delay_stats.h", "send_statistics_proxy.cc", @@ -99,6 +101,7 @@ rtc_library("video") { "../modules/video_processing", "../rtc_base:checks", "../rtc_base:rate_limiter", + "../rtc_base:rtc_base", "../rtc_base:rtc_base_approved", "../rtc_base:rtc_numerics", "../rtc_base:rtc_task_queue", @@ -503,6 +506,7 @@ if (rtc_include_tests) { "quality_threshold_unittest.cc", "receive_statistics_proxy_unittest.cc", "report_block_stats_unittest.cc", + "rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc", "rtp_video_stream_receiver_unittest.cc", "send_delay_stats_unittest.cc", "send_statistics_proxy_unittest.cc", @@ -532,6 +536,7 @@ if (rtc_include_tests) { "../api:rtp_parameters", "../api:scoped_refptr", "../api:simulated_network_api", + "../api:transport_api", "../api/crypto:options", "../api/rtc_event_log", "../api/task_queue", diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 5bc8c7e6ab..ae43713764 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -224,8 +224,7 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()), has_received_frame_(false), frames_decryptable_(false), - absolute_capture_time_receiver_(clock), - frame_transformer_(frame_transformer) { + absolute_capture_time_receiver_(clock) { constexpr bool remb_candidate = true; if (packet_router_) packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); @@ -285,6 +284,13 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor)); } } + + if (frame_transformer) { + frame_transformer_delegate_ = new rtc::RefCountedObject< + RtpVideoStreamReceiverFrameTransformerDelegate>( + this, std::move(frame_transformer), rtc::Thread::Current()); + frame_transformer_delegate_->Init(); + } } RtpVideoStreamReceiver::RtpVideoStreamReceiver( @@ -326,6 +332,8 @@ RtpVideoStreamReceiver::~RtpVideoStreamReceiver() { if (packet_router_) packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get()); UpdateHistograms(); + if (frame_transformer_delegate_) + frame_transformer_delegate_->Reset(); } void RtpVideoStreamReceiver::AddReceiveCodec( @@ -796,10 +804,13 @@ void RtpVideoStreamReceiver::OnAssembledFrame( last_assembled_frame_rtp_timestamp_ = frame->Timestamp(); } - if (buffered_frame_decryptor_ == nullptr) { - reference_finder_->ManageFrame(std::move(frame)); - } else { + if (buffered_frame_decryptor_ != nullptr) { buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame)); + } else if (frame_transformer_delegate_) { + frame_transformer_delegate_->TransformFrame(std::move(frame), + config_.rtp.remote_ssrc); + } else { + reference_finder_->ManageFrame(std::move(frame)); } } @@ -874,6 +885,12 @@ void RtpVideoStreamReceiver::RemoveSecondarySink( secondary_sinks_.erase(it); } +void RtpVideoStreamReceiver::ManageFrame( + std::unique_ptr frame) { + rtc::CritScope lock(&reference_finder_lock_); + reference_finder_->ManageFrame(std::move(frame)); +} + void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) { if (packet.payload_size() == 0) { // Padding or keep-alive packet. diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h index 98b324ca96..f9b04a3cc7 100644 --- a/video/rtp_video_stream_receiver.h +++ b/video/rtp_video_stream_receiver.h @@ -48,6 +48,7 @@ #include "rtc_base/thread_annotations.h" #include "rtc_base/thread_checker.h" #include "video/buffered_frame_decryptor.h" +#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" namespace webrtc { @@ -198,6 +199,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender, void AddSecondarySink(RtpPacketSinkInterface* sink); void RemoveSecondarySink(const RtpPacketSinkInterface* sink); + virtual void ManageFrame(std::unique_ptr frame); + private: // Used for buffering RTCP feedback messages and sending them all together. // Note: @@ -370,7 +373,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender, int64_t last_completed_picture_id_ = 0; - rtc::scoped_refptr frame_transformer_; + rtc::scoped_refptr + frame_transformer_delegate_; }; } // namespace webrtc diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc new file mode 100644 index 0000000000..acef31cb96 --- /dev/null +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" + +#include + +#include "absl/memory/memory.h" +#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "rtc_base/thread.h" +#include "video/rtp_video_stream_receiver.h" + +namespace webrtc { + +RtpVideoStreamReceiverFrameTransformerDelegate:: + RtpVideoStreamReceiverFrameTransformerDelegate( + RtpVideoStreamReceiver* receiver, + rtc::scoped_refptr frame_transformer, + rtc::Thread* network_thread) + : receiver_(receiver), + frame_transformer_(std::move(frame_transformer)), + network_thread_(network_thread) {} + +void RtpVideoStreamReceiverFrameTransformerDelegate::Init() { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + frame_transformer_->RegisterTransformedFrameCallback( + rtc::scoped_refptr(this)); +} + +void RtpVideoStreamReceiverFrameTransformerDelegate::Reset() { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + frame_transformer_->UnregisterTransformedFrameCallback(); + frame_transformer_ = nullptr; + receiver_ = nullptr; +} + +void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame( + std::unique_ptr frame, + uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + auto additional_data = + RtpDescriptorAuthentication(frame->GetRtpVideoHeader()); + frame_transformer_->TransformFrame(std::move(frame), + std::move(additional_data), ssrc); +} + +void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( + std::unique_ptr frame) { + rtc::scoped_refptr delegate = + this; + network_thread_->PostTask(ToQueuedTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->ManageFrame(std::move(frame)); + })); +} + +void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( + std::unique_ptr frame) { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + if (!receiver_) + return; + auto transformed_frame = absl::WrapUnique( + static_cast(frame.release())); + receiver_->ManageFrame(std::move(transformed_frame)); +} + +} // namespace webrtc diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.h b/video/rtp_video_stream_receiver_frame_transformer_delegate.h new file mode 100644 index 0000000000..2309796258 --- /dev/null +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_FRAME_TRANSFORMER_DELEGATE_H_ +#define VIDEO_RTP_VIDEO_STREAM_RECEIVER_FRAME_TRANSFORMER_DELEGATE_H_ + +#include + +#include "api/frame_transformer_interface.h" +#include "modules/video_coding/frame_object.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/thread.h" + +namespace webrtc { + +class RtpVideoStreamReceiver; + +// Delegates calls to FrameTransformerInterface to transform frames, and to +// RtpVideoStreamReceiver to manage transformed frames on the |network_thread_|. +class RtpVideoStreamReceiverFrameTransformerDelegate + : public TransformedFrameCallback { + public: + RtpVideoStreamReceiverFrameTransformerDelegate( + RtpVideoStreamReceiver* receiver, + rtc::scoped_refptr frame_transformer, + rtc::Thread* network_thread); + + void Init(); + void Reset(); + + // Delegates the call to FrameTransformerInterface::TransformFrame. + void TransformFrame(std::unique_ptr frame, + uint32_t ssrc); + + // Implements TransformedFrameCallback. Can be called on any thread. Posts + // the transformed frame to be managed on the |network_thread_|. + void OnTransformedFrame( + std::unique_ptr frame) override; + + // Delegates the call to RtpVideoReceiver::ManageFrame on the + // |network_thread_|. + void ManageFrame(std::unique_ptr frame); + + protected: + ~RtpVideoStreamReceiverFrameTransformerDelegate() override = default; + + private: + SequenceChecker network_sequence_checker_; + RtpVideoStreamReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_); + rtc::scoped_refptr frame_transformer_ + RTC_GUARDED_BY(network_sequence_checker_); + rtc::Thread* const network_thread_; +}; + +} // namespace webrtc + +#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_FRAME_TRANSFORMER_DELEGATE_H_ diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc new file mode 100644 index 0000000000..bc9fe13a72 --- /dev/null +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" + +#include +#include +#include +#include + +#include "api/call/transport.h" +#include "call/video_receive_stream.h" +#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" +#include "modules/utility/include/process_thread.h" +#include "rtc_base/event.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "video/rtp_video_stream_receiver.h" + +namespace webrtc { +namespace { + +using ::testing::_; + +std::unique_ptr CreateRtpFrameObject() { + return std::make_unique( + 0, 0, true, 0, 0, 0, 0, 0, VideoSendTiming(), 0, kVideoCodecGeneric, + kVideoRotation_0, VideoContentType::UNSPECIFIED, RTPVideoHeader(), + absl::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(0)); +} + +class FakeTransport : public Transport { + public: + bool SendRtp(const uint8_t* packet, + size_t length, + const PacketOptions& options) { + return true; + } + bool SendRtcp(const uint8_t* packet, size_t length) { return true; } +}; + +class FakeNackSender : public NackSender { + public: + void SendNack(const std::vector& sequence_numbers) {} + void SendNack(const std::vector& sequence_numbers, + bool buffering_allowed) {} +}; + +class FakeOnCompleteFrameCallback + : public video_coding::OnCompleteFrameCallback { + public: + void OnCompleteFrame( + std::unique_ptr frame) override {} +}; + +class TestRtpVideoStreamReceiverInitializer { + public: + TestRtpVideoStreamReceiverInitializer() + : test_config_(nullptr), + test_process_thread_(ProcessThread::Create("TestThread")) { + test_config_.rtp.remote_ssrc = 1111; + test_config_.rtp.local_ssrc = 2222; + test_rtp_receive_statistics_ = + ReceiveStatistics::Create(Clock::GetRealTimeClock()); + } + + protected: + VideoReceiveStream::Config test_config_; + FakeTransport fake_transport_; + FakeNackSender fake_nack_sender_; + FakeOnCompleteFrameCallback fake_on_complete_frame_callback_; + std::unique_ptr test_process_thread_; + std::unique_ptr test_rtp_receive_statistics_; +}; + +class TestRtpVideoStreamReceiver : public TestRtpVideoStreamReceiverInitializer, + public RtpVideoStreamReceiver { + public: + TestRtpVideoStreamReceiver() + : TestRtpVideoStreamReceiverInitializer(), + RtpVideoStreamReceiver(Clock::GetRealTimeClock(), + &fake_transport_, + nullptr, + nullptr, + &test_config_, + test_rtp_receive_statistics_.get(), + nullptr, + test_process_thread_.get(), + &fake_nack_sender_, + nullptr, + &fake_on_complete_frame_callback_, + nullptr, + nullptr) {} + ~TestRtpVideoStreamReceiver() override = default; + + MOCK_METHOD(void, + ManageFrame, + (std::unique_ptr frame), + (override)); +}; + +class MockFrameTransformer : public FrameTransformerInterface { + public: + ~MockFrameTransformer() override = default; + MOCK_METHOD(void, + TransformFrame, + (std::unique_ptr, + std::vector, + uint32_t), + (override)); + MOCK_METHOD(void, + RegisterTransformedFrameCallback, + (rtc::scoped_refptr), + (override)); + MOCK_METHOD(void, UnregisterTransformedFrameCallback, (), (override)); +}; + +TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, + RegisterTransformedFrameCallbackOnInit) { + TestRtpVideoStreamReceiver receiver; + rtc::scoped_refptr frame_transformer( + new rtc::RefCountedObject()); + rtc::scoped_refptr delegate( + new rtc::RefCountedObject( + &receiver, frame_transformer, rtc::Thread::Current())); + EXPECT_CALL(*frame_transformer, RegisterTransformedFrameCallback); + delegate->Init(); +} + +TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, + UnregisterTransformedFrameCallbackOnReset) { + TestRtpVideoStreamReceiver receiver; + rtc::scoped_refptr frame_transformer( + new rtc::RefCountedObject()); + rtc::scoped_refptr delegate( + new rtc::RefCountedObject( + &receiver, frame_transformer, rtc::Thread::Current())); + EXPECT_CALL(*frame_transformer, UnregisterTransformedFrameCallback); + delegate->Reset(); +} + +TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformFrame) { + TestRtpVideoStreamReceiver receiver; + rtc::scoped_refptr frame_transformer( + new rtc::RefCountedObject()); + rtc::scoped_refptr delegate( + new rtc::RefCountedObject( + &receiver, frame_transformer, rtc::Thread::Current())); + auto frame = CreateRtpFrameObject(); + EXPECT_CALL(*frame_transformer, + TransformFrame(_, RtpDescriptorAuthentication(RTPVideoHeader()), + /*remote_ssrc*/ 1111)); + delegate->TransformFrame(std::move(frame), /*remote_ssrc*/ 1111); +} + +TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, + ManageFrameOnTransformedFrame) { + auto main_thread = rtc::Thread::Create(); + main_thread->Start(); + auto network_thread = rtc::Thread::Create(); + network_thread->Start(); + + TestRtpVideoStreamReceiver receiver; + rtc::scoped_refptr frame_transformer( + new rtc::RefCountedObject()); + auto delegate = network_thread->Invoke< + rtc::scoped_refptr>( + RTC_FROM_HERE, [&]() mutable { + return new rtc::RefCountedObject< + RtpVideoStreamReceiverFrameTransformerDelegate>( + &receiver, frame_transformer, network_thread.get()); + }); + + auto frame = CreateRtpFrameObject(); + + EXPECT_CALL(receiver, ManageFrame) + .WillOnce([&network_thread]( + std::unique_ptr frame) { + EXPECT_TRUE(network_thread->IsCurrent()); + }); + main_thread->Invoke(RTC_FROM_HERE, [&]() mutable { + delegate->OnTransformedFrame(std::move(frame)); + }); + rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); + + main_thread->Stop(); + network_thread->Stop(); +} + +} // namespace +} // namespace webrtc diff --git a/video/rtp_video_stream_receiver_unittest.cc b/video/rtp_video_stream_receiver_unittest.cc index d7d02b0731..7e7dd7d904 100644 --- a/video/rtp_video_stream_receiver_unittest.cc +++ b/video/rtp_video_stream_receiver_unittest.cc @@ -17,6 +17,7 @@ #include "api/video/video_frame_type.h" #include "common_video/h264/h264_common.h" #include "media/base/media_constants.h" +#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "modules/rtp_rtcp/source/rtp_format.h" #include "modules/rtp_rtcp/source/rtp_format_vp9.h" #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" @@ -124,6 +125,17 @@ class MockRtpPacketSink : public RtpPacketSinkInterface { MOCK_METHOD1(OnRtpPacket, void(const RtpPacketReceived&)); }; +class MockFrameTransformer : public FrameTransformerInterface { + public: + MOCK_METHOD3(TransformFrame, + void(std::unique_ptr frame, + std::vector additional_data, + uint32_t ssrc)); + MOCK_METHOD1(RegisterTransformedFrameCallback, + void(rtc::scoped_refptr)); + MOCK_METHOD0(UnregisterTransformedFrameCallback, void()); +}; + constexpr uint32_t kSsrc = 111; constexpr uint16_t kSequenceNumber = 222; std::unique_ptr CreateRtpPacketReceived( @@ -1205,4 +1217,33 @@ TEST_F(RtpVideoStreamReceiverTest, RepeatedSecondarySinkDisallowed) { } #endif +TEST_F(RtpVideoStreamReceiverTest, TransformFrame) { + rtc::scoped_refptr mock_frame_transformer = + new rtc::RefCountedObject(); + EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback); + auto receiver = std::make_unique( + Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, + rtp_receive_statistics_.get(), nullptr, process_thread_.get(), + &mock_nack_sender_, nullptr, &mock_on_complete_frame_callback_, nullptr, + mock_frame_transformer); + + RtpPacketReceived rtp_packet; + RTPVideoHeader video_header; + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); + rtp_packet.SetSequenceNumber(1); + video_header.is_first_packet_in_frame = true; + video_header.is_last_packet_in_frame = true; + video_header.codec = kVideoCodecGeneric; + video_header.frame_type = VideoFrameType::kVideoFrameKey; + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(*mock_frame_transformer, + TransformFrame(_, RtpDescriptorAuthentication(video_header), + config_.rtp.remote_ssrc)); + receiver->OnReceivedPayloadData(data, rtp_packet, video_header); + + EXPECT_CALL(*mock_frame_transformer, UnregisterTransformedFrameCallback()); + receiver = nullptr; +} + } // namespace webrtc