From c24b6b7815ceada81afbd75e77339fefb27b7846 Mon Sep 17 00:00:00 2001 From: Marina Ciocea Date: Mon, 30 Mar 2020 14:51:10 +0200 Subject: [PATCH] Introduce TransformableFrameInterface. Add a new frame interface to be used by frame transformers in Insertable Streams. TransformableFrameInterface will replace video_coding::EncodedFrame in a follow up CL, once downstream dependecies are updated to use the new interface. Until the functions using video_coding::EncodedFrame are removed from the API, the video sender and receiver frame transformer delegates call both function versions to avoid breaking tests downstream. The TransformableFrameInterface will be used for both audio and video frame transformers in follow-up CLs. Bug: webrtc:11380 Change-Id: I9389a8549c156e13b1d8c938ff51eaa69c502f33 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171863 Commit-Queue: Marina Ciocea Reviewed-by: Magnus Flodman Reviewed-by: Karl Wiberg Reviewed-by: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#30941} --- api/frame_transformer_interface.h | 52 +++++- ...sender_video_frame_transformer_delegate.cc | 158 ++++++++++++++++-- ..._sender_video_frame_transformer_delegate.h | 5 +- ...eam_receiver_frame_transformer_delegate.cc | 70 +++++++- ...ream_receiver_frame_transformer_delegate.h | 3 + 5 files changed, 264 insertions(+), 24 deletions(-) diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 666a791f91..615c97466e 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -20,18 +20,54 @@ namespace webrtc { +// Owns the frame payload data. +class TransformableFrameInterface { + public: + virtual ~TransformableFrameInterface() = default; + + // Returns the frame payload data. The data is valid until the next non-const + // method call. + virtual rtc::ArrayView GetData() const = 0; + + // Copies |data| into the owned frame payload data. + virtual void SetData(rtc::ArrayView data) = 0; + + virtual uint32_t GetTimestamp() const = 0; + virtual uint32_t GetSsrc() const = 0; +}; + +class TransformableVideoFrameInterface : public TransformableFrameInterface { + public: + virtual ~TransformableVideoFrameInterface() = default; + virtual bool IsKeyFrame() const = 0; + + // Returns data needed in the frame transformation logic; for example, + // when the transformation applied to the frame is encryption/decryption, the + // additional data holds the serialized generic frame descriptor extension + // calculated in webrtc::RtpDescriptorAuthentication. + // TODO(bugs.webrtc.org/11380) remove from interface once + // webrtc::RtpDescriptorAuthentication is exposed in api/. + virtual std::vector GetAdditionalData() const = 0; +}; + // Objects implement this interface to be notified with the transformed frame. class TransformedFrameCallback : public rtc::RefCountInterface { public: + // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies + // to use new OnTransformedFrame signature. virtual void OnTransformedFrame( - std::unique_ptr transformed_frame) = 0; + std::unique_ptr transformed_frame) {} + // TODO(bugs.webrtc.org/11380) make pure virtual after updating usage + // downstream. + virtual void OnTransformedFrame( + std::unique_ptr transformed_frame) {} protected: ~TransformedFrameCallback() override = default; }; -// Transformes encoded frames. The transformed frame is sent in a callback using -// the TransformedFrameCallback interface (see below). +// Transforms encoded frames. The transformed frame is sent in a callback using +// the TransformedFrameCallback interface (see above). class FrameTransformerInterface : public rtc::RefCountInterface { public: // Transforms |frame| using the implementing class' processing logic. @@ -41,9 +77,17 @@ class FrameTransformerInterface : public rtc::RefCountInterface { // holds the serialized generic frame descriptor extension calculated in // webrtc::RtpDescriptorAuthentication, needed in the encryption/decryption // algorithms. + // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies + // to use new OnTransformedFrame() signature. virtual void TransformFrame(std::unique_ptr frame, std::vector additional_data, - uint32_t ssrc) = 0; + uint32_t ssrc) {} + + // Transforms |frame| using the implementing class' processing logic. + // TODO(bugs.webrtc.org/11380) make pure virtual after updating usage + // downstream. + virtual void Transform( + std::unique_ptr transformable_frame) {} virtual void RegisterTransformedFrameCallback( rtc::scoped_refptr) = 0; diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 80753f3fb0..fa8309bac7 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -11,6 +11,7 @@ #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" #include +#include #include "absl/memory/memory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" @@ -19,6 +20,88 @@ #include "rtc_base/task_utils/to_queued_task.h" namespace webrtc { +namespace { + +std::unique_ptr CreateFragmentationHeader( + const RTPFragmentationHeader* fragmentation_header) { + if (!fragmentation_header) + return nullptr; + auto ret = std::make_unique(); + ret->CopyFrom(*fragmentation_header); + return ret; +} + +class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { + public: + TransformableVideoSenderFrame( + const EncodedImage& encoded_image, + const RTPVideoHeader& video_header, + int payload_type, + absl::optional codec_type, + uint32_t rtp_timestamp, + const RTPFragmentationHeader* fragmentation_header, + absl::optional expected_retransmission_time_ms, + uint32_t ssrc) + : encoded_data_(encoded_image.GetEncodedData()), + header_(video_header), + frame_type_(encoded_image._frameType), + payload_type_(payload_type), + codec_type_(codec_type), + timestamp_(rtp_timestamp), + capture_time_ms_(encoded_image.capture_time_ms_), + expected_retransmission_time_ms_(expected_retransmission_time_ms), + ssrc_(ssrc), + fragmentation_header_(CreateFragmentationHeader(fragmentation_header)) { + } + + ~TransformableVideoSenderFrame() override = default; + + // Implements TransformableVideoFrameInterface. + rtc::ArrayView GetData() const override { + return *encoded_data_; + } + + void SetData(rtc::ArrayView data) override { + encoded_data_ = EncodedImageBuffer::Create(data.data(), data.size()); + } + + uint32_t GetTimestamp() const override { return timestamp_; } + uint32_t GetSsrc() const override { return ssrc_; } + + bool IsKeyFrame() const override { + return frame_type_ == VideoFrameType::kVideoFrameKey; + } + + std::vector GetAdditionalData() const override { + return RtpDescriptorAuthentication(header_); + } + + const RTPVideoHeader& GetHeader() const { return header_; } + int GetPayloadType() const { return payload_type_; } + absl::optional GetCodecType() const { return codec_type_; } + int64_t GetCaptureTimeMs() const { return capture_time_ms_; } + + RTPFragmentationHeader* GetFragmentationHeader() const { + return fragmentation_header_.get(); + } + + const absl::optional& GetExpectedRetransmissionTimeMs() const { + return expected_retransmission_time_ms_; + } + + private: + rtc::scoped_refptr encoded_data_; + const RTPVideoHeader header_; + const VideoFrameType frame_type_; + const int payload_type_; + const absl::optional codec_type_ = absl::nullopt; + const uint32_t timestamp_; + const int64_t capture_time_ms_; + const absl::optional expected_retransmission_time_ms_; + const uint32_t ssrc_; + const std::unique_ptr fragmentation_header_; +}; +} // namespace RTPSenderVideoFrameTransformerDelegate::RTPSenderVideoFrameTransformerDelegate( RTPSenderVideo* sender, @@ -41,45 +124,84 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame( uint32_t ssrc) { if (!encoder_queue_) encoder_queue_ = TaskQueueBase::Current(); + // TODO(bugs.webrtc.org/11380) remove once this version of TransformFrame() is + // deprecated. frame_transformer_->TransformFrame( std::make_unique( encoded_image.GetEncodedData(), video_header, payload_type, codec_type, rtp_timestamp, encoded_image.capture_time_ms_, fragmentation, expected_retransmission_time_ms), RtpDescriptorAuthentication(video_header), ssrc); + frame_transformer_->Transform(std::make_unique( + encoded_image, video_header, payload_type, codec_type, rtp_timestamp, + fragmentation, expected_retransmission_time_ms, ssrc)); return true; } void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame( std::unique_ptr frame) { - { - rtc::CritScope lock(&sender_lock_); - if (!sender_) - return; - } - auto transformed_frame = absl::WrapUnique( - static_cast(frame.release())); + rtc::CritScope lock(&sender_lock_); + + // The encoder queue gets destroyed after the sender; as long as the sender is + // alive, it's safe to post. + if (!sender_) + return; rtc::scoped_refptr delegate = this; - encoder_queue_->PostTask( - ToQueuedTask([delegate = std::move(delegate), - transformed_frame = std::move(transformed_frame)]() { - delegate->SendVideo(*transformed_frame.get()); + encoder_queue_->PostTask(ToQueuedTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->SendVideo(std::move(frame)); + })); +} + +void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame( + std::unique_ptr frame) { + rtc::CritScope lock(&sender_lock_); + + // The encoder queue gets destroyed after the sender; as long as the sender is + // alive, it's safe to post. + if (!sender_) + return; + rtc::scoped_refptr delegate = this; + encoder_queue_->PostTask(ToQueuedTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->SendVideo(std::move(frame)); })); } void RTPSenderVideoFrameTransformerDelegate::SendVideo( - const TransformableEncodedFrame& transformed_frame) const { + std::unique_ptr frame) const { RTC_CHECK(encoder_queue_->IsCurrent()); rtc::CritScope lock(&sender_lock_); if (!sender_) return; + auto* transformed_frame = + static_cast(frame.get()); sender_->SendVideo( - transformed_frame.PayloadType(), transformed_frame.codec_type(), - transformed_frame.Timestamp(), transformed_frame.capture_time_ms(), - transformed_frame.EncodedImage(), - transformed_frame.fragmentation_header(), - transformed_frame.video_header(), - transformed_frame.expected_retransmission_time_ms()); + transformed_frame->PayloadType(), transformed_frame->codec_type(), + transformed_frame->Timestamp(), transformed_frame->capture_time_ms(), + transformed_frame->EncodedImage(), + transformed_frame->fragmentation_header(), + transformed_frame->video_header(), + transformed_frame->expected_retransmission_time_ms()); +} + +void RTPSenderVideoFrameTransformerDelegate::SendVideo( + std::unique_ptr transformed_frame) const { + RTC_CHECK(encoder_queue_->IsCurrent()); + rtc::CritScope lock(&sender_lock_); + if (!sender_) + return; + auto* transformed_video_frame = + static_cast(transformed_frame.get()); + sender_->SendVideo( + transformed_video_frame->GetPayloadType(), + transformed_video_frame->GetCodecType(), + transformed_video_frame->GetTimestamp(), + transformed_video_frame->GetCaptureTimeMs(), + transformed_video_frame->GetData(), + transformed_video_frame->GetFragmentationHeader(), + transformed_video_frame->GetHeader(), + transformed_video_frame->GetExpectedRetransmissionTimeMs()); } void RTPSenderVideoFrameTransformerDelegate::SetVideoStructureUnderLock( diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index 383c4d5990..4a6482b2de 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -48,9 +48,12 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { // the transformed frame to be sent on the |encoder_queue_|. void OnTransformedFrame( std::unique_ptr frame) override; + void OnTransformedFrame( + std::unique_ptr frame) override; // Delegates the call to RTPSendVideo::SendVideo on the |encoder_queue_|. - void SendVideo(const TransformableEncodedFrame& transformed_frame) const; + void SendVideo(std::unique_ptr frame) const; + void SendVideo(std::unique_ptr frame) const; // Delegates the call to RTPSendVideo::SendVideo under |sender_lock_|. void SetVideoStructureUnderLock( diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc index acef31cb96..8b63c7e4df 100644 --- a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -11,6 +11,7 @@ #include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" #include +#include #include "absl/memory/memory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" @@ -20,6 +21,45 @@ namespace webrtc { +namespace { +class TransformableVideoReceiverFrame + : public TransformableVideoFrameInterface { + public: + TransformableVideoReceiverFrame( + std::unique_ptr frame, + uint32_t ssrc) + : frame_(std::move(frame)), ssrc_(ssrc) {} + ~TransformableVideoReceiverFrame() override = default; + + // Implements TransformableVideoFrameInterface. + rtc::ArrayView GetData() const override { + return *frame_->GetEncodedData(); + } + + void SetData(rtc::ArrayView data) override { + frame_->SetEncodedData( + EncodedImageBuffer::Create(data.data(), data.size())); + } + + uint32_t GetTimestamp() const override { return frame_->Timestamp(); } + uint32_t GetSsrc() const override { return ssrc_; } + + bool IsKeyFrame() const override { return frame_->is_keyframe(); } + + std::vector GetAdditionalData() const override { + return RtpDescriptorAuthentication(frame_->GetRtpVideoHeader()); + } + + std::unique_ptr ExtractFrame() && { + return std::move(frame_); + } + + private: + std::unique_ptr frame_; + const uint32_t ssrc_; +}; +} // namespace + RtpVideoStreamReceiverFrameTransformerDelegate:: RtpVideoStreamReceiverFrameTransformerDelegate( RtpVideoStreamReceiver* receiver, @@ -46,10 +86,18 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame( std::unique_ptr frame, uint32_t ssrc) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); + // TODO(bugs.webrtc.org/11380) remove once this version of TransformFrame is + // deprecated. auto additional_data = RtpDescriptorAuthentication(frame->GetRtpVideoHeader()); - frame_transformer_->TransformFrame(std::move(frame), + auto frame_copy = + std::make_unique(*frame.get()); + frame_transformer_->TransformFrame(std::move(frame_copy), std::move(additional_data), ssrc); + + frame_transformer_->Transform( + std::make_unique(std::move(frame), + ssrc)); } void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( @@ -62,6 +110,16 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( })); } +void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( + std::unique_ptr frame) { + rtc::scoped_refptr delegate = + this; + network_thread_->PostTask(ToQueuedTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->ManageFrame(std::move(frame)); + })); +} + void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( std::unique_ptr frame) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); @@ -72,4 +130,14 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( receiver_->ManageFrame(std::move(transformed_frame)); } +void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( + std::unique_ptr frame) { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + if (!receiver_) + return; + auto transformed_frame = absl::WrapUnique( + static_cast(frame.release())); + receiver_->ManageFrame(std::move(*transformed_frame).ExtractFrame()); +} + } // namespace webrtc diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.h b/video/rtp_video_stream_receiver_frame_transformer_delegate.h index 2309796258..85fad1e002 100644 --- a/video/rtp_video_stream_receiver_frame_transformer_delegate.h +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.h @@ -43,10 +43,13 @@ class RtpVideoStreamReceiverFrameTransformerDelegate // the transformed frame to be managed on the |network_thread_|. void OnTransformedFrame( std::unique_ptr frame) override; + void OnTransformedFrame( + std::unique_ptr frame) override; // Delegates the call to RtpVideoReceiver::ManageFrame on the // |network_thread_|. void ManageFrame(std::unique_ptr frame); + void ManageFrame(std::unique_ptr frame); protected: ~RtpVideoStreamReceiverFrameTransformerDelegate() override = default;