From b0e7057e1b15568d7e1ac9b8274b0e44a0101e63 Mon Sep 17 00:00:00 2001 From: Harald Alvestrand Date: Tue, 23 Apr 2024 14:04:18 +0000 Subject: [PATCH] Introduce the TransformerHost interface This is the first step in implementing custom codecs in SDP. Bug: none Change-Id: I7789478208a769eaefd58b410ae6f488c604594d Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/348662 Commit-Queue: Harald Alvestrand Reviewed-by: Tony Herre Cr-Commit-Position: refs/heads/main@{#42171} --- api/frame_transformer_interface.h | 12 ++++++++++++ api/rtp_receiver_interface.cc | 2 +- api/rtp_receiver_interface.h | 16 ++++++++++++++-- api/rtp_sender_interface.h | 16 ++++++++++++++-- api/test/mock_rtpsender.h | 2 +- pc/audio_rtp_receiver.cc | 2 +- pc/audio_rtp_receiver.h | 2 +- pc/rtp_receiver_proxy.h | 2 +- pc/rtp_sender.cc | 4 ++-- pc/rtp_sender.h | 2 +- pc/rtp_sender_proxy.h | 2 +- pc/test/mock_rtp_sender_internal.h | 2 +- pc/video_rtp_receiver.cc | 2 +- pc/video_rtp_receiver.h | 2 +- 14 files changed, 52 insertions(+), 16 deletions(-) diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 102ac33334..d3d15418af 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -120,6 +120,18 @@ class FrameTransformerInterface : public rtc::RefCountInterface { ~FrameTransformerInterface() override = default; }; +// An interface implemented by classes that can host a transform. +// Currently this is implemented by the RTCRtpSender and RTCRtpReceiver. +class FrameTransformerHost { + public: + virtual ~FrameTransformerHost() {} + virtual void SetFrameTransformer( + rtc::scoped_refptr frame_transformer) = 0; + // TODO: bugs.webrtc.org/15929 - To be added: + // virtual AddIncomingMediaType(RtpCodec codec) = 0; + // virtual AddOutgoingMediaType(RtpCodec codec) = 0; +}; + } // namespace webrtc #endif // API_FRAME_TRANSFORMER_INTERFACE_H_ diff --git a/api/rtp_receiver_interface.cc b/api/rtp_receiver_interface.cc index bc9aef5aef..ffa421683a 100644 --- a/api/rtp_receiver_interface.cc +++ b/api/rtp_receiver_interface.cc @@ -38,7 +38,7 @@ RtpReceiverInterface::dtls_transport() const { return nullptr; } -void RtpReceiverInterface::SetDepacketizerToDecoderFrameTransformer( +void RtpReceiverInterface::SetFrameTransformer( rtc::scoped_refptr frame_transformer) {} } // namespace webrtc diff --git a/api/rtp_receiver_interface.h b/api/rtp_receiver_interface.h index 0bf1af972b..b1b36b5c58 100644 --- a/api/rtp_receiver_interface.h +++ b/api/rtp_receiver_interface.h @@ -15,8 +15,10 @@ #define API_RTP_RECEIVER_INTERFACE_H_ #include +#include #include +#include "absl/types/optional.h" #include "api/crypto/frame_decryptor_interface.h" #include "api/dtls_transport_interface.h" #include "api/frame_transformer_interface.h" @@ -44,7 +46,8 @@ class RtpReceiverObserverInterface { virtual ~RtpReceiverObserverInterface() {} }; -class RTC_EXPORT RtpReceiverInterface : public webrtc::RefCountInterface { +class RTC_EXPORT RtpReceiverInterface : public webrtc::RefCountInterface, + public FrameTransformerHost { public: virtual rtc::scoped_refptr track() const = 0; @@ -111,8 +114,17 @@ class RTC_EXPORT RtpReceiverInterface : public webrtc::RefCountInterface { // Sets a frame transformer between the depacketizer and the decoder to enable // client code to transform received frames according to their own processing // logic. + // TODO: bugs.webrtc.org/15929 - add [[deprecated("Use SetFrameTransformer")]] + // when usage in Chrome is removed virtual void SetDepacketizerToDecoderFrameTransformer( - rtc::scoped_refptr frame_transformer); + rtc::scoped_refptr frame_transformer) { + SetFrameTransformer(std::move(frame_transformer)); + } + + // Default implementation of SetFrameTransformer. + // TODO: bugs.webrtc.org/15929 - Make pure virtual. + void SetFrameTransformer( + rtc::scoped_refptr frame_transformer) override; protected: ~RtpReceiverInterface() override = default; diff --git a/api/rtp_sender_interface.h b/api/rtp_sender_interface.h index 7b8ab135c8..7d92c42a18 100644 --- a/api/rtp_sender_interface.h +++ b/api/rtp_sender_interface.h @@ -14,8 +14,10 @@ #ifndef API_RTP_SENDER_INTERFACE_H_ #define API_RTP_SENDER_INTERFACE_H_ +#include #include #include +#include #include #include "absl/functional/any_invocable.h" @@ -36,7 +38,8 @@ namespace webrtc { using SetParametersCallback = absl::AnyInvocable; -class RTC_EXPORT RtpSenderInterface : public webrtc::RefCountInterface { +class RTC_EXPORT RtpSenderInterface : public webrtc::RefCountInterface, + public FrameTransformerHost { public: // Returns true if successful in setting the track. // Fails if an audio track is set on a video RtpSender, or vice-versa. @@ -100,8 +103,12 @@ class RTC_EXPORT RtpSenderInterface : public webrtc::RefCountInterface { virtual rtc::scoped_refptr GetFrameEncryptor() const = 0; + // TODO: bugs.webrtc.org/15929 - add [[deprecated("Use SetFrameTransformer")]] + // when usage in Chrome is removed virtual void SetEncoderToPacketizerFrameTransformer( - rtc::scoped_refptr frame_transformer) = 0; + rtc::scoped_refptr frame_transformer) { + SetFrameTransformer(std::move(frame_transformer)); + } // Sets a user defined encoder selector. // Overrides selector that is (optionally) provided by VideoEncoderFactory. @@ -109,6 +116,11 @@ class RTC_EXPORT RtpSenderInterface : public webrtc::RefCountInterface { std::unique_ptr encoder_selector) = 0; + // Default implementation of SetFrameTransformer. + // TODO: bugs.webrtc.org/15929 - remove when all implementations are good + void SetFrameTransformer(rtc::scoped_refptr + frame_transformer) override {} + protected: ~RtpSenderInterface() override = default; }; diff --git a/api/test/mock_rtpsender.h b/api/test/mock_rtpsender.h index 22113678b9..5a38d1df54 100644 --- a/api/test/mock_rtpsender.h +++ b/api/test/mock_rtpsender.h @@ -63,7 +63,7 @@ class MockRtpSender : public RtpSenderInterface { (), (const, override)); MOCK_METHOD(void, - SetEncoderToPacketizerFrameTransformer, + SetFrameTransformer, (rtc::scoped_refptr), (override)); MOCK_METHOD(void, diff --git a/pc/audio_rtp_receiver.cc b/pc/audio_rtp_receiver.cc index 6e7ca6d0b5..0f395a73c3 100644 --- a/pc/audio_rtp_receiver.cc +++ b/pc/audio_rtp_receiver.cc @@ -277,7 +277,7 @@ std::vector AudioRtpReceiver::GetSources() const { return media_channel_->GetSources(current_ssrc.value()); } -void AudioRtpReceiver::SetDepacketizerToDecoderFrameTransformer( +void AudioRtpReceiver::SetFrameTransformer( rtc::scoped_refptr frame_transformer) { RTC_DCHECK_RUN_ON(worker_thread_); if (media_channel_) { diff --git a/pc/audio_rtp_receiver.h b/pc/audio_rtp_receiver.h index 36cbdffc35..d0433c3696 100644 --- a/pc/audio_rtp_receiver.h +++ b/pc/audio_rtp_receiver.h @@ -117,7 +117,7 @@ class AudioRtpReceiver : public ObserverInterface, std::vector GetSources() const override; int AttachmentId() const override { return attachment_id_; } - void SetDepacketizerToDecoderFrameTransformer( + void SetFrameTransformer( rtc::scoped_refptr frame_transformer) override; private: diff --git a/pc/rtp_receiver_proxy.h b/pc/rtp_receiver_proxy.h index d4114e0f0b..db677d818a 100644 --- a/pc/rtp_receiver_proxy.h +++ b/pc/rtp_receiver_proxy.h @@ -45,7 +45,7 @@ PROXY_SECONDARY_METHOD1(void, PROXY_SECONDARY_CONSTMETHOD0(rtc::scoped_refptr, GetFrameDecryptor) PROXY_SECONDARY_METHOD1(void, - SetDepacketizerToDecoderFrameTransformer, + SetFrameTransformer, rtc::scoped_refptr) END_PROXY_MAP(RtpReceiver) diff --git a/pc/rtp_sender.cc b/pc/rtp_sender.cc index be0975778f..60b8813037 100644 --- a/pc/rtp_sender.cc +++ b/pc/rtp_sender.cc @@ -506,7 +506,7 @@ void RtpSenderBase::SetSsrc(uint32_t ssrc) { SetFrameEncryptor(frame_encryptor_); } if (frame_transformer_) { - SetEncoderToPacketizerFrameTransformer(frame_transformer_); + SetFrameTransformer(frame_transformer_); } if (encoder_selector_) { SetEncoderSelectorOnChannel(); @@ -580,7 +580,7 @@ RTCError RtpSenderBase::DisableEncodingLayers( return result; } -void RtpSenderBase::SetEncoderToPacketizerFrameTransformer( +void RtpSenderBase::SetFrameTransformer( rtc::scoped_refptr frame_transformer) { RTC_DCHECK_RUN_ON(signaling_thread_); frame_transformer_ = std::move(frame_transformer); diff --git a/pc/rtp_sender.h b/pc/rtp_sender.h index 90d33aaffc..bd2cf9df16 100644 --- a/pc/rtp_sender.h +++ b/pc/rtp_sender.h @@ -206,7 +206,7 @@ class RtpSenderBase : public RtpSenderInternal, public ObserverInterface { // If the specified list is empty, this is a no-op. RTCError DisableEncodingLayers(const std::vector& rid) override; - void SetEncoderToPacketizerFrameTransformer( + void SetFrameTransformer( rtc::scoped_refptr frame_transformer) override; void SetEncoderSelector( diff --git a/pc/rtp_sender_proxy.h b/pc/rtp_sender_proxy.h index 39862eb133..8ed32f63da 100644 --- a/pc/rtp_sender_proxy.h +++ b/pc/rtp_sender_proxy.h @@ -47,7 +47,7 @@ PROXY_CONSTMETHOD0(rtc::scoped_refptr, GetFrameEncryptor) PROXY_METHOD1(void, SetStreams, const std::vector&) PROXY_METHOD1(void, - SetEncoderToPacketizerFrameTransformer, + SetFrameTransformer, rtc::scoped_refptr) PROXY_METHOD1(void, SetEncoderSelector, diff --git a/pc/test/mock_rtp_sender_internal.h b/pc/test/mock_rtp_sender_internal.h index 9a01aecabb..bb7776b706 100644 --- a/pc/test/mock_rtp_sender_internal.h +++ b/pc/test/mock_rtp_sender_internal.h @@ -85,7 +85,7 @@ class MockRtpSenderInternal : public RtpSenderInternal { (), (const, override)); MOCK_METHOD(void, - SetEncoderToPacketizerFrameTransformer, + SetFrameTransformer, (rtc::scoped_refptr), (override)); MOCK_METHOD(void, diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 4432982027..6694d3e1fc 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -98,7 +98,7 @@ VideoRtpReceiver::GetFrameDecryptor() const { return frame_decryptor_; } -void VideoRtpReceiver::SetDepacketizerToDecoderFrameTransformer( +void VideoRtpReceiver::SetFrameTransformer( rtc::scoped_refptr frame_transformer) { RTC_DCHECK_RUN_ON(worker_thread_); frame_transformer_ = std::move(frame_transformer); diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index ef88016052..dfa491cb60 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -82,7 +82,7 @@ class VideoRtpReceiver : public RtpReceiverInternal { rtc::scoped_refptr GetFrameDecryptor() const override; - void SetDepacketizerToDecoderFrameTransformer( + void SetFrameTransformer( rtc::scoped_refptr frame_transformer) override; // RtpReceiverInternal implementation.