admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Set surrogate receive times for transformed sender frames

Browse Source 
Without this, 'Sender' frames inserted into the writer of an encoded
transform have an invalid receive time (0), which breaks all later
heuristics which build on the receive time, eg the VCMTiming estimators
used for controlling the playback delay.

Bug: chromium:1463451
Change-Id: I413c884e08986148d4a854cd275212b21d093ceb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/311544
Reviewed-by: Guido Urdaneta <guidou@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Palak Agarwal <agpalak@google.com>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Tony Herre <herre@google.com>
Cr-Commit-Position: refs/heads/main@{#40416}
This commit is contained in:
Tony Herre 2023-07-11 12:41:06 +00:00 committed by WebRTC LUCI CQ
parent dfe026ce08
commit 9d677f4cdc
4 changed files with 28 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc
View File

@ -81,13 +81,15 @@ class TransformableVideoReceiverFrame
RtpVideoStreamReceiverFrameTransformerDelegate:: RtpVideoStreamReceiverFrameTransformerDelegate::
RtpVideoStreamReceiverFrameTransformerDelegate( RtpVideoStreamReceiverFrameTransformerDelegate(
RtpVideoFrameReceiver* receiver, RtpVideoFrameReceiver* receiver,
Clock* clock,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
rtc::Thread* network_thread, rtc::Thread* network_thread,
uint32_t ssrc) uint32_t ssrc)
: receiver_(receiver), : receiver_(receiver),
frame_transformer_(std::move(frame_transformer)), frame_transformer_(std::move(frame_transformer)),
network_thread_(network_thread), network_thread_(network_thread),
ssrc_(ssrc) {} ssrc_(ssrc),
clock_(clock) {}
void RtpVideoStreamReceiverFrameTransformerDelegate::Init() { void RtpVideoStreamReceiverFrameTransformerDelegate::Init() {
RTC_DCHECK_RUN_ON(&network_sequence_checker_); RTC_DCHECK_RUN_ON(&network_sequence_checker_);
@ -162,13 +164,14 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata);
VideoSendTiming timing; VideoSendTiming timing;
rtc::ArrayView<const uint8_t> data = transformed_frame->GetData(); rtc::ArrayView<const uint8_t> data = transformed_frame->GetData();
int64_t receive_time = clock_->CurrentTime().ms();
receiver_->ManageFrame(std::make_unique<RtpFrameObject>( receiver_->ManageFrame(std::make_unique<RtpFrameObject>(
/*first_seq_num=*/metadata.GetFrameId().value_or(0), /*first_seq_num=*/metadata.GetFrameId().value_or(0),
/*last_seq_num=*/metadata.GetFrameId().value_or(0), /*last_seq_num=*/metadata.GetFrameId().value_or(0),
/*markerBit=*/video_header.is_last_frame_in_picture, /*markerBit=*/video_header.is_last_frame_in_picture,
/*times_nacked=*/0, /*times_nacked=*/0,
/*first_packet_received_time=*/0, /*first_packet_received_time=*/receive_time,
/*last_packet_received_time=*/0, /*last_packet_received_time=*/receive_time,
/*rtp_timestamp=*/transformed_frame->GetTimestamp(), /*rtp_timestamp=*/transformed_frame->GetTimestamp(),
/*ntp_time_ms=*/0, timing, transformed_frame->GetPayloadType(), /*ntp_time_ms=*/0, timing, transformed_frame->GetPayloadType(),
metadata.GetCodec(), metadata.GetRotation(), metadata.GetContentType(), metadata.GetCodec(), metadata.GetRotation(), metadata.GetContentType(),

3
modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h
View File

@ -18,6 +18,7 @@
#include "modules/rtp_rtcp/source/frame_object.h" #include "modules/rtp_rtcp/source/frame_object.h"
#include "rtc_base/system/no_unique_address.h" #include "rtc_base/system/no_unique_address.h"
#include "rtc_base/thread.h" #include "rtc_base/thread.h"
#include "system_wrappers/include/clock.h"
namespace webrtc { namespace webrtc {
@ -38,6 +39,7 @@ class RtpVideoStreamReceiverFrameTransformerDelegate
public: public:
RtpVideoStreamReceiverFrameTransformerDelegate( RtpVideoStreamReceiverFrameTransformerDelegate(
RtpVideoFrameReceiver* receiver, RtpVideoFrameReceiver* receiver,
Clock* clock,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
rtc::Thread* network_thread, rtc::Thread* network_thread,
uint32_t ssrc); uint32_t ssrc);
@ -67,6 +69,7 @@ class RtpVideoStreamReceiverFrameTransformerDelegate
RTC_GUARDED_BY(network_sequence_checker_); RTC_GUARDED_BY(network_sequence_checker_);
rtc::Thread* const network_thread_; rtc::Thread* const network_thread_;
const uint32_t ssrc_; const uint32_t ssrc_;
Clock* const clock_;
}; };
} // namespace webrtc } // namespace webrtc

25
modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc
View File

@ -72,9 +72,10 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
RegisterTransformedFrameCallbackSinkOnInit) { RegisterTransformedFrameCallbackSinkOnInit) {
TestRtpVideoFrameReceiver receiver; TestRtpVideoFrameReceiver receiver;
auto frame_transformer(rtc::make_ref_counted<MockFrameTransformer>()); auto frame_transformer(rtc::make_ref_counted<MockFrameTransformer>());
SimulatedClock clock(0);
auto delegate( auto delegate(
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, frame_transformer, rtc::Thread::Current(), &receiver, &clock, frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111)); /*remote_ssrc*/ 1111));
EXPECT_CALL(*frame_transformer, EXPECT_CALL(*frame_transformer,
RegisterTransformedFrameSinkCallback(testing::_, 1111)); RegisterTransformedFrameSinkCallback(testing::_, 1111));
@ -85,9 +86,10 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
UnregisterTransformedFrameSinkCallbackOnReset) { UnregisterTransformedFrameSinkCallbackOnReset) {
TestRtpVideoFrameReceiver receiver; TestRtpVideoFrameReceiver receiver;
auto frame_transformer(rtc::make_ref_counted<MockFrameTransformer>()); auto frame_transformer(rtc::make_ref_counted<MockFrameTransformer>());
SimulatedClock clock(0);
auto delegate( auto delegate(
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, frame_transformer, rtc::Thread::Current(), &receiver, &clock, frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111)); /*remote_ssrc*/ 1111));
EXPECT_CALL(*frame_transformer, UnregisterTransformedFrameSinkCallback(1111)); EXPECT_CALL(*frame_transformer, UnregisterTransformedFrameSinkCallback(1111));
delegate->Reset(); delegate->Reset();
@ -97,9 +99,10 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformFrame) {
TestRtpVideoFrameReceiver receiver; TestRtpVideoFrameReceiver receiver;
auto frame_transformer( auto frame_transformer(
rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>()); rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>());
SimulatedClock clock(0);
auto delegate( auto delegate(
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, frame_transformer, rtc::Thread::Current(), &receiver, &clock, frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111)); /*remote_ssrc*/ 1111));
auto frame = CreateRtpFrameObject(); auto frame = CreateRtpFrameObject();
EXPECT_CALL(*frame_transformer, Transform); EXPECT_CALL(*frame_transformer, Transform);
@ -112,10 +115,11 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
TestRtpVideoFrameReceiver receiver; TestRtpVideoFrameReceiver receiver;
auto mock_frame_transformer( auto mock_frame_transformer(
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>()); rtc::make_ref_counted<NiceMock<MockFrameTransformer>>());
SimulatedClock clock(0);
std::vector<uint32_t> csrcs = {234, 345, 456}; std::vector<uint32_t> csrcs = {234, 345, 456};
auto delegate = auto delegate =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, mock_frame_transformer, rtc::Thread::Current(), &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111); /*remote_ssrc*/ 1111);
rtc::scoped_refptr<TransformedFrameCallback> callback; rtc::scoped_refptr<TransformedFrameCallback> callback;
@ -144,9 +148,11 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
TestRtpVideoFrameReceiver receiver; TestRtpVideoFrameReceiver receiver;
auto mock_frame_transformer = auto mock_frame_transformer =
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>(); rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
SimulatedClock clock(0);
auto delegate = auto delegate =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, mock_frame_transformer, rtc::Thread::Current(), 1111); &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
1111);
delegate->Init(); delegate->Init();
RTPVideoHeader video_header; RTPVideoHeader video_header;
video_header.width = 1280u; video_header.width = 1280u;
@ -191,9 +197,10 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
TestRtpVideoFrameReceiver receiver; TestRtpVideoFrameReceiver receiver;
auto mock_frame_transformer = auto mock_frame_transformer =
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>(); rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
SimulatedClock clock(/*initial_timestamp_us=*/12345000);
auto delegate = auto delegate =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, mock_frame_transformer, rtc::Thread::Current(), &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(),
/*remote_ssrc*/ 1111); /*remote_ssrc*/ 1111);
auto mock_sender_frame = auto mock_sender_frame =
@ -218,6 +225,7 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
EXPECT_CALL(receiver, ManageFrame) EXPECT_CALL(receiver, ManageFrame)
.WillOnce([&](std::unique_ptr<RtpFrameObject> frame) { .WillOnce([&](std::unique_ptr<RtpFrameObject> frame) {
EXPECT_EQ(frame->codec_type(), metadata.GetCodec()); EXPECT_EQ(frame->codec_type(), metadata.GetCodec());
EXPECT_EQ(frame->ReceivedTime(), 12345);
}); });
callback->OnTransformedFrame(std::move(mock_sender_frame)); callback->OnTransformedFrame(std::move(mock_sender_frame));
rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
@ -232,9 +240,10 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
TestRtpVideoFrameReceiver receiver1; TestRtpVideoFrameReceiver receiver1;
auto mock_frame_transformer1( auto mock_frame_transformer1(
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>()); rtc::make_ref_counted<NiceMock<MockFrameTransformer>>());
SimulatedClock clock(0);
auto delegate1 = auto delegate1 =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver1, mock_frame_transformer1, rtc::Thread::Current(), &receiver1, &clock, mock_frame_transformer1, rtc::Thread::Current(),
/*remote_ssrc*/ 1111); /*remote_ssrc*/ 1111);
TestRtpVideoFrameReceiver receiver2; TestRtpVideoFrameReceiver receiver2;
@ -242,7 +251,7 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
rtc::make_ref_counted<NiceMock<MockFrameTransformer>>()); rtc::make_ref_counted<NiceMock<MockFrameTransformer>>());
auto delegate2 = auto delegate2 =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver2, mock_frame_transformer2, rtc::Thread::Current(), &receiver2, &clock, mock_frame_transformer2, rtc::Thread::Current(),
/*remote_ssrc*/ 1111); /*remote_ssrc*/ 1111);
delegate1->Init(); delegate1->Init();

4
video/rtp_video_stream_receiver2.cc
View File

@ -336,7 +336,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2(
if (frame_transformer) { if (frame_transformer) {
frame_transformer_delegate_ = frame_transformer_delegate_ =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
this, std::move(frame_transformer), rtc::Thread::Current(), this, clock_, std::move(frame_transformer), rtc::Thread::Current(),
config_.rtp.remote_ssrc); config_.rtp.remote_ssrc);
frame_transformer_delegate_->Init(); frame_transformer_delegate_->Init();
} }
@ -950,7 +950,7 @@ void RtpVideoStreamReceiver2::SetDepacketizerToDecoderFrameTransformer(
RTC_DCHECK_RUN_ON(&worker_task_checker_); RTC_DCHECK_RUN_ON(&worker_task_checker_);
frame_transformer_delegate_ = frame_transformer_delegate_ =
rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>(
this, std::move(frame_transformer), rtc::Thread::Current(), this, clock_, std::move(frame_transformer), rtc::Thread::Current(),
config_.rtp.remote_ssrc); config_.rtp.remote_ssrc);
frame_transformer_delegate_->Init(); frame_transformer_delegate_->Init();
} }