admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Insert frame transformer between Depacketizer and Decoder.

Browse Source 
Add a new API in RTReceiverInterface, to be called from the browser side
to insert a frame transformer between the Depacketizer and the Decoder.

The frame transformer is passed from RTReceiverInterface through the
library to be eventually set in RtpVideoStreamReceiver, where the frame
transformation will occur in the follow-up CL
https://webrtc-review.googlesource.com/c/src/+/169130.

This change is part of the implementation of the Insertable Streams Web
API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md

Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk

Bug: webrtc:11380
Change-Id: I6b73cd16e3907e8b7709b852d6a2540ee11b4fed
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169129
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Magnus Flodman <mflodman@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30654}
This commit is contained in:
Marina Ciocea 2020-02-28 16:02:06 +01:00 committed by Commit Bot
parent c6a65c8866
commit 412a31bbf8
14 changed files with 132 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/rtp_receiver_interface.cc
View File

@ -38,4 +38,7 @@ RtpReceiverInterface::dtls_transport() const {
return nullptr;
}
void RtpReceiverInterface::SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {}
} // namespace webrtc

10
api/rtp_receiver_interface.h
View File

@ -19,6 +19,7 @@
#include "api/crypto/frame_decryptor_interface.h"
#include "api/dtls_transport_interface.h"
#include "api/frame_transformer_interface.h"
#include "api/media_stream_interface.h"
#include "api/media_types.h"
#include "api/proxy.h"
@ -107,6 +108,12 @@ class RTC_EXPORT RtpReceiverInterface : public rtc::RefCountInterface {
// user. This can be used to update the state of the object.
virtual rtc::scoped_refptr<FrameDecryptorInterface> GetFrameDecryptor() const;
// Sets a frame transformer between the depacketizer and the decoder to enable
// client code to transform received frames according to their own processing
// logic.
virtual void SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
protected:
~RtpReceiverInterface() override = default;
};
@ -132,6 +139,9 @@ PROXY_METHOD1(void,
rtc::scoped_refptr<FrameDecryptorInterface>)
PROXY_CONSTMETHOD0(rtc::scoped_refptr<FrameDecryptorInterface>,
GetFrameDecryptor)
PROXY_METHOD1(void,
SetDepacketizerToDecoderFrameTransformer,
rtc::scoped_refptr<FrameTransformerInterface>)
END_PROXY_MAP()
} // namespace webrtc

3
call/video_receive_stream.h
View File

@ -21,6 +21,7 @@
#include "api/call/transport.h"
#include "api/crypto/crypto_options.h"
#include "api/crypto/frame_decryptor_interface.h"
#include "api/frame_transformer_interface.h"
#include "api/rtp_headers.h"
#include "api/rtp_parameters.h"
#include "api/transport/rtp/rtp_source.h"
@ -262,6 +263,8 @@ class VideoReceiveStream {
// Per PeerConnection cryptography options.
CryptoOptions crypto_options;
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer;
};
// Starts stream activity.

3
media/base/media_channel.cc
View File

@ -52,6 +52,9 @@ void MediaChannel::SetVideoCodecSwitchingEnabled(bool enabled) {}
void MediaChannel::SetEncoderToPacketizerFrameTransformer(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {}
void MediaChannel::SetDepacketizerToDecoderFrameTransformer(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {}
MediaSenderInfo::MediaSenderInfo() = default;
MediaSenderInfo::~MediaSenderInfo() = default;

3
media/base/media_channel.h
View File

@ -291,6 +291,9 @@ class MediaChannel : public sigslot::has_slots<> {
virtual void SetEncoderToPacketizerFrameTransformer(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer);
virtual void SetDepacketizerToDecoderFrameTransformer(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer);
protected:
bool DscpEnabled() const { return enable_dscp_; }

18
media/engine/webrtc_video_engine.cc
View File

@ -2941,6 +2941,13 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() {
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::
SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface>
frame_transformer) {
config_.frame_transformer = frame_transformer;
}
WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings()
: flexfec_payload_type(-1), rtx_payload_type(-1) {}
@ -3152,6 +3159,17 @@ void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer(
}
}
void WebRtcVideoChannel::SetDepacketizerToDecoderFrameTransformer(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
RTC_DCHECK_RUN_ON(&thread_checker_);
auto matching_stream = receive_streams_.find(ssrc);
if (matching_stream != receive_streams_.end()) {
matching_stream->second->SetDepacketizerToDecoderFrameTransformer(
std::move(frame_transformer));
}
}
// TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of
// EncoderStreamFactory and instead set this value individually for each stream
// in the VideoEncoderConfig.simulcast_layers.

8
media/engine/webrtc_video_engine.h
View File

@ -228,6 +228,10 @@ class WebRtcVideoChannel : public VideoMediaChannel,
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
override;
void SetDepacketizerToDecoderFrameTransformer(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
override;
private:
class WebRtcVideoReceiveStream;
@ -463,6 +467,10 @@ class WebRtcVideoChannel : public VideoMediaChannel,
void ClearRecordableEncodedFrameCallback();
void GenerateKeyFrame();
void SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface>
frame_transformer);
private:
void RecreateWebRtcVideoStream();
void MaybeRecreateWebRtcFlexfecStream();

17
pc/video_rtp_receiver.cc
View File

@ -104,6 +104,18 @@ VideoRtpReceiver::GetFrameDecryptor() const {
return frame_decryptor_;
}
void VideoRtpReceiver::SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
worker_thread_->Invoke<void>(RTC_FROM_HERE, [&] {
RTC_DCHECK_RUN_ON(worker_thread_);
frame_transformer_ = std::move(frame_transformer);
if (media_channel_ && ssrc_.has_value() && !stopped_) {
media_channel_->SetDepacketizerToDecoderFrameTransformer(
*ssrc_, frame_transformer_);
}
});
}
void VideoRtpReceiver::Stop() {
// TODO(deadbeef): Need to do more here to fully stop receiving packets.
if (stopped_) {
@ -144,6 +156,11 @@ void VideoRtpReceiver::RestartMediaChannel(absl::optional<uint32_t> ssrc) {
if (encoded_sink_enabled) {
SetEncodedSinkEnabled(true);
}
if (frame_transformer_ && media_channel_ && ssrc_.has_value()) {
media_channel_->SetDepacketizerToDecoderFrameTransformer(
*ssrc_, frame_transformer_);
}
});
// Attach any existing frame decryptor to the media channel.

6
pc/video_rtp_receiver.h
View File

@ -18,6 +18,7 @@
#include "absl/types/optional.h"
#include "api/crypto/frame_decryptor_interface.h"
#include "api/frame_transformer_interface.h"
#include "api/media_stream_interface.h"
#include "api/media_types.h"
#include "api/rtp_parameters.h"
@ -83,6 +84,9 @@ class VideoRtpReceiver : public rtc::RefCountedObject<RtpReceiverInternal>,
rtc::scoped_refptr<FrameDecryptorInterface> GetFrameDecryptor()
const override;
void SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override;
// RtpReceiverInternal implementation.
void Stop() override;
void SetupMediaChannel(uint32_t ssrc) override;
@ -134,6 +138,8 @@ class VideoRtpReceiver : public rtc::RefCountedObject<RtpReceiverInternal>,
int attachment_id_ = 0;
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor_;
rtc::scoped_refptr<DtlsTransportInterface> dtls_transport_;
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
RTC_GUARDED_BY(worker_thread_);
// Allows to thread safely change jitter buffer delay. Handles caching cases
// if |SetJitterBufferMinimumDelay| is called before start.
rtc::scoped_refptr<JitterBufferDelayInterface> delay_;

1
video/BUILD.gn
View File

@ -54,6 +54,7 @@ rtc_library("video") {
":frame_dumping_decoder",
"../api:array_view",
"../api:fec_controller_api",
"../api:frame_transformer_interface",
"../api:libjingle_peerconnection_api",
"../api:rtp_parameters",
"../api:scoped_refptr",

34
video/rtp_video_stream_receiver.cc
View File

@ -12,6 +12,7 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <utility>
#include <vector>
@ -194,7 +195,8 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver(
NackSender* nack_sender,
KeyFrameRequestSender* keyframe_request_sender,
video_coding::OnCompleteFrameCallback* complete_frame_callback,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor)
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
: clock_(clock),
config_(*config),
packet_router_(packet_router),
@ -221,7 +223,8 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver(
packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()),
has_received_frame_(false),
frames_decryptable_(false),
absolute_capture_time_receiver_(clock) {
absolute_capture_time_receiver_(clock),
frame_transformer_(frame_transformer) {
constexpr bool remb_candidate = true;
if (packet_router_)
packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
@ -283,6 +286,33 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver(
}
}
RtpVideoStreamReceiver::RtpVideoStreamReceiver(
Clock* clock,
Transport* transport,
RtcpRttStats* rtt_stats,
PacketRouter* packet_router,
const VideoReceiveStream::Config* config,
ReceiveStatistics* rtp_receive_statistics,
ReceiveStatisticsProxy* receive_stats_proxy,
ProcessThread* process_thread,
NackSender* nack_sender,
KeyFrameRequestSender* keyframe_request_sender,
video_coding::OnCompleteFrameCallback* complete_frame_callback,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor)
: RtpVideoStreamReceiver(clock,
transport,
rtt_stats,
packet_router,
config,
rtp_receive_statistics,
receive_stats_proxy,
process_thread,
nack_sender,
keyframe_request_sender,
complete_frame_callback,
frame_decryptor,
nullptr) {}
RtpVideoStreamReceiver::~RtpVideoStreamReceiver() {
RTC_DCHECK(secondary_sinks_.empty());

25
video/rtp_video_stream_receiver.h
View File

@ -69,6 +69,29 @@ class RtpVideoStreamReceiver : public LossNotificationSender,
public OnDecryptedFrameCallback,
public OnDecryptionStatusChangeCallback {
public:
RtpVideoStreamReceiver(
Clock* clock,
Transport* transport,
RtcpRttStats* rtt_stats,
// The packet router is optional; if provided, the RtpRtcp module for this
// stream is registered as a candidate for sending REMB and transport
// feedback.
PacketRouter* packet_router,
const VideoReceiveStream::Config* config,
ReceiveStatistics* rtp_receive_statistics,
ReceiveStatisticsProxy* receive_stats_proxy,
ProcessThread* process_thread,
NackSender* nack_sender,
// The KeyFrameRequestSender is optional; if not provided, key frame
// requests are sent via the internal RtpRtcp module.
KeyFrameRequestSender* keyframe_request_sender,
video_coding::OnCompleteFrameCallback* complete_frame_callback,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
// TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies
// to use the new constructor.
RTC_DEPRECATED
RtpVideoStreamReceiver(
Clock* clock,
Transport* transport,
@ -346,6 +369,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender,
RTC_GUARDED_BY(worker_task_checker_);
int64_t last_completed_picture_id_ = 0;
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
};
} // namespace webrtc

2
video/rtp_video_stream_receiver_unittest.cc
View File

@ -155,7 +155,7 @@ class RtpVideoStreamReceiverTest : public ::testing::Test {
Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_,
rtp_receive_statistics_.get(), nullptr, process_thread_.get(),
&mock_nack_sender_, &mock_key_frame_request_sender_,
&mock_on_complete_frame_callback_, nullptr);
&mock_on_complete_frame_callback_, nullptr, nullptr);
}
RTPVideoHeader GetDefaultH264VideoHeader() {

3
video/video_receive_stream.cc
View File

@ -215,7 +215,8 @@ VideoReceiveStream::VideoReceiveStream(
this, // NackSender
nullptr, // Use default KeyFrameRequestSender
this, // OnCompleteFrameCallback
config_.frame_decryptor),
config_.frame_decryptor,
config_.frame_transformer),
rtp_stream_sync_(this),
max_wait_for_keyframe_ms_(KeyframeIntervalSettings::ParseFromFieldTrials()
.MaxWaitForKeyframeMs()