diff --git a/webrtc/call/call_perf_tests.cc b/webrtc/call/call_perf_tests.cc index 4361872209..beb05a047d 100644 --- a/webrtc/call/call_perf_tests.cc +++ b/webrtc/call/call_perf_tests.cc @@ -7,7 +7,9 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ + #include +#include #include #include #include @@ -33,6 +35,7 @@ #include "webrtc/test/fake_encoder.h" #include "webrtc/test/frame_generator.h" #include "webrtc/test/frame_generator_capturer.h" +#include "webrtc/test/histogram.h" #include "webrtc/test/rtp_rtcp_observer.h" #include "webrtc/test/testsupport/fileutils.h" #include "webrtc/test/testsupport/perf_test.h" @@ -71,100 +74,35 @@ class CallPerfTest : public test::CallTest { int run_time_ms); }; -class SyncRtcpObserver : public test::RtpRtcpObserver { - public: - SyncRtcpObserver() : test::RtpRtcpObserver(CallPerfTest::kLongTimeoutMs) {} - - Action OnSendRtcp(const uint8_t* packet, size_t length) override { - RTCPUtility::RTCPParserV2 parser(packet, length, true); - EXPECT_TRUE(parser.IsValid()); - - for (RTCPUtility::RTCPPacketTypes packet_type = parser.Begin(); - packet_type != RTCPUtility::RTCPPacketTypes::kInvalid; - packet_type = parser.Iterate()) { - if (packet_type == RTCPUtility::RTCPPacketTypes::kSr) { - const RTCPUtility::RTCPPacket& packet = parser.Packet(); - RtcpMeasurement ntp_rtp_pair( - packet.SR.NTPMostSignificant, - packet.SR.NTPLeastSignificant, - packet.SR.RTPTimestamp); - StoreNtpRtpPair(ntp_rtp_pair); - } - } - return SEND_PACKET; - } - - int64_t RtpTimestampToNtp(uint32_t timestamp) const { - rtc::CritScope lock(&crit_); - int64_t timestamp_in_ms = -1; - if (ntp_rtp_pairs_.size() == 2) { - // TODO(stefan): We can't EXPECT_TRUE on this call due to a bug in the - // RTCP sender where it sends RTCP SR before any RTP packets, which leads - // to a bogus NTP/RTP mapping. - RtpToNtpMs(timestamp, ntp_rtp_pairs_, ×tamp_in_ms); - return timestamp_in_ms; - } - return -1; - } - - private: - void StoreNtpRtpPair(RtcpMeasurement ntp_rtp_pair) { - rtc::CritScope lock(&crit_); - for (RtcpList::iterator it = ntp_rtp_pairs_.begin(); - it != ntp_rtp_pairs_.end(); - ++it) { - if (ntp_rtp_pair.ntp_secs == it->ntp_secs && - ntp_rtp_pair.ntp_frac == it->ntp_frac) { - // This RTCP has already been added to the list. - return; - } - } - // We need two RTCP SR reports to map between RTP and NTP. More than two - // will not improve the mapping. - if (ntp_rtp_pairs_.size() == 2) { - ntp_rtp_pairs_.pop_back(); - } - ntp_rtp_pairs_.push_front(ntp_rtp_pair); - } - - rtc::CriticalSection crit_; - RtcpList ntp_rtp_pairs_ GUARDED_BY(crit_); -}; - -class VideoRtcpAndSyncObserver : public SyncRtcpObserver, public VideoRenderer { +class VideoRtcpAndSyncObserver : public test::RtpRtcpObserver, + public VideoRenderer { static const int kInSyncThresholdMs = 50; static const int kStartupTimeMs = 2000; static const int kMinRunTimeMs = 30000; public: - VideoRtcpAndSyncObserver(Clock* clock, - int voe_channel, - VoEVideoSync* voe_sync, - SyncRtcpObserver* audio_observer) - : clock_(clock), - voe_channel_(voe_channel), - voe_sync_(voe_sync), - audio_observer_(audio_observer), + explicit VideoRtcpAndSyncObserver(Clock* clock) + : test::RtpRtcpObserver(CallPerfTest::kLongTimeoutMs), + clock_(clock), creation_time_ms_(clock_->TimeInMilliseconds()), - first_time_in_sync_(-1) {} + first_time_in_sync_(-1), + receive_stream_(nullptr) {} void RenderFrame(const VideoFrame& video_frame, int time_to_render_ms) override { + VideoReceiveStream::Stats stats; + { + rtc::CritScope lock(&crit_); + if (receive_stream_) + stats = receive_stream_->GetStats(); + } + if (stats.sync_offset_ms == std::numeric_limits::max()) + return; + int64_t now_ms = clock_->TimeInMilliseconds(); - uint32_t playout_timestamp = 0; - if (voe_sync_->GetPlayoutTimestamp(voe_channel_, playout_timestamp) != 0) - return; - int64_t latest_audio_ntp = - audio_observer_->RtpTimestampToNtp(playout_timestamp); - int64_t latest_video_ntp = RtpTimestampToNtp(video_frame.timestamp()); - if (latest_audio_ntp < 0 || latest_video_ntp < 0) - return; - int time_until_render_ms = - std::max(0, static_cast(video_frame.render_time_ms() - now_ms)); - latest_video_ntp += time_until_render_ms; - int64_t stream_offset = latest_audio_ntp - latest_video_ntp; + std::stringstream ss; - ss << stream_offset; + ss << stats.sync_offset_ms; webrtc::test::PrintResult("stream_offset", "", "synchronization", @@ -176,7 +114,7 @@ class VideoRtcpAndSyncObserver : public SyncRtcpObserver, public VideoRenderer { // estimated as being synchronized. We don't want to trigger on those. if (time_since_creation < kStartupTimeMs) return; - if (std::abs(latest_audio_ntp - latest_video_ntp) < kInSyncThresholdMs) { + if (std::abs(stats.sync_offset_ms) < kInSyncThresholdMs) { if (first_time_in_sync_ == -1) { first_time_in_sync_ = now_ms; webrtc::test::PrintResult("sync_convergence_time", @@ -193,13 +131,17 @@ class VideoRtcpAndSyncObserver : public SyncRtcpObserver, public VideoRenderer { bool IsTextureSupported() const override { return false; } + void set_receive_stream(VideoReceiveStream* receive_stream) { + rtc::CritScope lock(&crit_); + receive_stream_ = receive_stream; + } + private: Clock* const clock_; - const int voe_channel_; - VoEVideoSync* const voe_sync_; - SyncRtcpObserver* const audio_observer_; const int64_t creation_time_ms_; int64_t first_time_in_sync_; + rtc::CriticalSection crit_; + VideoReceiveStream* receive_stream_ GUARDED_BY(crit_); }; void CallPerfTest::TestAudioVideoSync(FecMode fec, @@ -238,11 +180,11 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec, std::unique_ptr parser_; }; + test::ClearHistograms(); VoiceEngine* voice_engine = VoiceEngine::Create(); VoEBase* voe_base = VoEBase::GetInterface(voice_engine); VoECodec* voe_codec = VoECodec::GetInterface(voice_engine); VoENetwork* voe_network = VoENetwork::GetInterface(voice_engine); - VoEVideoSync* voe_sync = VoEVideoSync::GetInterface(voice_engine); const std::string audio_filename = test::ResourcePath("voice_engine/audio_long16", "pcm"); ASSERT_STRNE("", audio_filename.c_str()); @@ -254,8 +196,6 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec, int send_channel_id = voe_base->CreateChannel(voe_config); int recv_channel_id = voe_base->CreateChannel(); - SyncRtcpObserver audio_observer; - AudioState::Config send_audio_state_config; send_audio_state_config.voice_engine = voice_engine; Call::Config sender_config; @@ -267,14 +207,16 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec, AudioPacketReceiver voe_send_packet_receiver(send_channel_id, voe_network); AudioPacketReceiver voe_recv_packet_receiver(recv_channel_id, voe_network); + VideoRtcpAndSyncObserver observer(Clock::GetRealTimeClock()); + FakeNetworkPipe::Config net_config; net_config.queue_delay_ms = 500; net_config.loss_percent = 5; test::PacketTransport audio_send_transport( - nullptr, &audio_observer, test::PacketTransport::kSender, net_config); + nullptr, &observer, test::PacketTransport::kSender, net_config); audio_send_transport.SetReceiver(&voe_recv_packet_receiver); test::PacketTransport audio_receive_transport( - nullptr, &audio_observer, test::PacketTransport::kReceiver, net_config); + nullptr, &observer, test::PacketTransport::kReceiver, net_config); audio_receive_transport.SetReceiver(&voe_send_packet_receiver); internal::TransportAdapter send_transport_adapter(&audio_send_transport); @@ -287,9 +229,6 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec, EXPECT_EQ(0, voe_network->RegisterExternalTransport(recv_channel_id, recv_transport_adapter)); - VideoRtcpAndSyncObserver observer(Clock::GetRealTimeClock(), recv_channel_id, - voe_sync, &audio_observer); - test::PacketTransport sync_send_transport(sender_call_.get(), &observer, test::PacketTransport::kSender, FakeNetworkPipe::Config()); @@ -341,7 +280,8 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec, audio_receive_stream = receiver_call_->CreateAudioReceiveStream(audio_recv_config); } - + EXPECT_EQ(1u, video_receive_streams_.size()); + observer.set_receive_stream(video_receive_streams_[0]); DriftingClock drifting_clock(clock_, video_ntp_speed); CreateFrameGeneratorCapturerWithDrift(&drifting_clock, video_rtp_speed); @@ -376,11 +316,12 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec, voe_base->Release(); voe_codec->Release(); voe_network->Release(); - voe_sync->Release(); DestroyCalls(); VoiceEngine::Delete(voice_engine); + + EXPECT_EQ(1, test::NumHistogramSamples("WebRTC.Video.AVSyncOffsetInMs")); } TEST_F(CallPerfTest, PlaysOutAudioAndVideoInSyncWithVideoNtpDrift) { diff --git a/webrtc/modules/rtp_rtcp/source/remote_ntp_time_estimator.cc b/webrtc/modules/rtp_rtcp/source/remote_ntp_time_estimator.cc index ccc15ec417..69a8f2391f 100644 --- a/webrtc/modules/rtp_rtcp/source/remote_ntp_time_estimator.cc +++ b/webrtc/modules/rtp_rtcp/source/remote_ntp_time_estimator.cc @@ -51,10 +51,6 @@ bool RemoteNtpTimeEstimator::UpdateRtcpTimestamp(int64_t rtt, } int64_t RemoteNtpTimeEstimator::Estimate(uint32_t rtp_timestamp) { - if (rtcp_list_.size() < 2) { - // We need two RTCP SR reports to calculate NTP. - return -1; - } int64_t sender_capture_ntp_ms = 0; if (!RtpToNtpMs(rtp_timestamp, rtcp_list_, &sender_capture_ntp_ms)) { return -1; diff --git a/webrtc/system_wrappers/source/rtp_to_ntp.cc b/webrtc/system_wrappers/source/rtp_to_ntp.cc index 0aceb0625f..706d861b55 100644 --- a/webrtc/system_wrappers/source/rtp_to_ntp.cc +++ b/webrtc/system_wrappers/source/rtp_to_ntp.cc @@ -95,7 +95,9 @@ bool UpdateRtcpList(uint32_t ntp_secs, bool RtpToNtpMs(int64_t rtp_timestamp, const RtcpList& rtcp, int64_t* rtp_timestamp_in_ms) { - assert(rtcp.size() == 2); + if (rtcp.size() != 2) + return false; + int64_t rtcp_ntp_ms_new = Clock::NtpToMs(rtcp.front().ntp_secs, rtcp.front().ntp_frac); int64_t rtcp_ntp_ms_old = Clock::NtpToMs(rtcp.back().ntp_secs, diff --git a/webrtc/system_wrappers/source/rtp_to_ntp_unittest.cc b/webrtc/system_wrappers/source/rtp_to_ntp_unittest.cc index 4c166774a4..d2929f5cbc 100644 --- a/webrtc/system_wrappers/source/rtp_to_ntp_unittest.cc +++ b/webrtc/system_wrappers/source/rtp_to_ntp_unittest.cc @@ -12,6 +12,10 @@ #include "webrtc/system_wrappers/include/rtp_to_ntp.h" namespace webrtc { +namespace { +const uint32_t kOneMsInNtpFrac = 4294967; +const uint32_t kTimestampTicksPerMs = 90; +} // namespace TEST(WrapAroundTests, NoWrap) { EXPECT_EQ(0, CheckForWrapArounds(0xFFFFFFFF, 0xFFFFFFFE)); @@ -38,8 +42,6 @@ TEST(WrapAroundTests, OldRtcpWrapped) { uint32_t ntp_sec = 0; uint32_t ntp_frac = 0; uint32_t timestamp = 0; - const uint32_t kOneMsInNtpFrac = 4294967; - const uint32_t kTimestampTicksPerMs = 90; rtcp.push_front(RtcpMeasurement(ntp_sec, ntp_frac, timestamp)); ntp_frac += kOneMsInNtpFrac; timestamp -= kTimestampTicksPerMs; @@ -57,8 +59,6 @@ TEST(WrapAroundTests, NewRtcpWrapped) { uint32_t ntp_sec = 0; uint32_t ntp_frac = 0; uint32_t timestamp = 0xFFFFFFFF; - const uint32_t kOneMsInNtpFrac = 4294967; - const uint32_t kTimestampTicksPerMs = 90; rtcp.push_front(RtcpMeasurement(ntp_sec, ntp_frac, timestamp)); ntp_frac += kOneMsInNtpFrac; timestamp += kTimestampTicksPerMs; @@ -71,8 +71,6 @@ TEST(WrapAroundTests, NewRtcpWrapped) { } TEST(WrapAroundTests, RtpWrapped) { - const uint32_t kOneMsInNtpFrac = 4294967; - const uint32_t kTimestampTicksPerMs = 90; RtcpList rtcp; uint32_t ntp_sec = 0; uint32_t ntp_frac = 0; @@ -91,8 +89,6 @@ TEST(WrapAroundTests, RtpWrapped) { } TEST(WrapAroundTests, OldRtp_RtcpsWrapped) { - const uint32_t kOneMsInNtpFrac = 4294967; - const uint32_t kTimestampTicksPerMs = 90; RtcpList rtcp; uint32_t ntp_sec = 0; uint32_t ntp_frac = 0; @@ -108,8 +104,6 @@ TEST(WrapAroundTests, OldRtp_RtcpsWrapped) { } TEST(WrapAroundTests, OldRtp_NewRtcpWrapped) { - const uint32_t kOneMsInNtpFrac = 4294967; - const uint32_t kTimestampTicksPerMs = 90; RtcpList rtcp; uint32_t ntp_sec = 0; uint32_t ntp_frac = 0; @@ -128,8 +122,6 @@ TEST(WrapAroundTests, OldRtp_NewRtcpWrapped) { } TEST(WrapAroundTests, OldRtp_OldRtcpWrapped) { - const uint32_t kOneMsInNtpFrac = 4294967; - const uint32_t kTimestampTicksPerMs = 90; RtcpList rtcp; uint32_t ntp_sec = 0; uint32_t ntp_frac = 0; diff --git a/webrtc/video/receive_statistics_proxy.cc b/webrtc/video/receive_statistics_proxy.cc index c92d4083c0..b5e9ab0734 100644 --- a/webrtc/video/receive_statistics_proxy.cc +++ b/webrtc/video/receive_statistics_proxy.cc @@ -59,6 +59,10 @@ void ReceiveStatisticsProxy::UpdateHistograms() { RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.ReceivedWidthInPixels", width); RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.ReceivedHeightInPixels", height); } + int sync_offset_ms = sync_offset_counter_.Avg(kMinRequiredSamples); + if (sync_offset_ms != -1) + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.AVSyncOffsetInMs", sync_offset_ms); + int qp = qp_counters_.vp8.Avg(kMinRequiredSamples); if (qp != -1) RTC_HISTOGRAM_COUNTS_200("WebRTC.Video.Decoded.Vp8.Qp", qp); @@ -239,6 +243,12 @@ void ReceiveStatisticsProxy::OnRenderedFrame(const VideoFrame& frame) { } } +void ReceiveStatisticsProxy::OnSyncOffsetUpdated(int64_t sync_offset_ms) { + rtc::CritScope lock(&crit_); + sync_offset_counter_.Add(std::abs(sync_offset_ms)); + stats_.sync_offset_ms = sync_offset_ms; +} + void ReceiveStatisticsProxy::OnReceiveRatesUpdated(uint32_t bitRate, uint32_t frameRate) { } diff --git a/webrtc/video/receive_statistics_proxy.h b/webrtc/video/receive_statistics_proxy.h index 56fb3c32e9..04bfd7b330 100644 --- a/webrtc/video/receive_statistics_proxy.h +++ b/webrtc/video/receive_statistics_proxy.h @@ -46,6 +46,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, void OnDecodedFrame(); void OnRenderedFrame(const VideoFrame& frame); + void OnSyncOffsetUpdated(int64_t sync_offset_ms); void OnIncomingPayloadType(int payload_type); void OnDecoderImplementationName(const char* implementation_name); void OnIncomingRate(unsigned int framerate, unsigned int bitrate_bps); @@ -106,6 +107,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, rtc::RateTracker render_pixel_tracker_ GUARDED_BY(crit_); SampleCounter render_width_counter_ GUARDED_BY(crit_); SampleCounter render_height_counter_ GUARDED_BY(crit_); + SampleCounter sync_offset_counter_ GUARDED_BY(crit_); SampleCounter decode_time_counter_ GUARDED_BY(crit_); SampleCounter delay_counter_ GUARDED_BY(crit_); ReportBlockStats report_block_stats_ GUARDED_BY(crit_); diff --git a/webrtc/video/stream_synchronization.cc b/webrtc/video/stream_synchronization.cc index cb37d80ef5..3727f8fdb5 100644 --- a/webrtc/video/stream_synchronization.cc +++ b/webrtc/video/stream_synchronization.cc @@ -60,10 +60,6 @@ bool StreamSynchronization::ComputeRelativeDelay( const Measurements& video_measurement, int* relative_delay_ms) { assert(relative_delay_ms); - if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) { - // We need two RTCP SR reports per stream to do synchronization. - return false; - } int64_t audio_last_capture_time_ms; if (!RtpToNtpMs(audio_measurement.latest_timestamp, audio_measurement.rtcp, diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index cd487ac5d3..6cc2794190 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -384,6 +384,10 @@ void VideoReceiveStream::FrameCallback(VideoFrame* video_frame) { int VideoReceiveStream::RenderFrame(const uint32_t /*stream_id*/, const VideoFrame& video_frame) { + int64_t sync_offset_ms; + if (vie_sync_.GetStreamSyncOffsetInMs(video_frame, &sync_offset_ms)) + stats_proxy_.OnSyncOffsetUpdated(sync_offset_ms); + // TODO(pbos): Wire up config_.render->IsTextureSupported() and convert if not // supported. Or provide methods for converting a texture frame in // VideoFrame. diff --git a/webrtc/video/vie_sync_module.cc b/webrtc/video/vie_sync_module.cc index f8376e53d1..9feace79b9 100644 --- a/webrtc/video/vie_sync_module.cc +++ b/webrtc/video/vie_sync_module.cc @@ -16,11 +16,13 @@ #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h" #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" #include "webrtc/modules/video_coding/include/video_coding.h" +#include "webrtc/system_wrappers/include/clock.h" #include "webrtc/video/stream_synchronization.h" +#include "webrtc/video_frame.h" #include "webrtc/voice_engine/include/voe_video_sync.h" namespace webrtc { - +namespace { int UpdateMeasurements(StreamSynchronization::Measurements* stream, const RtpRtcp& rtp_rtcp, const RtpReceiver& receiver) { if (!receiver.Timestamp(&stream->latest_timestamp)) @@ -47,16 +49,17 @@ int UpdateMeasurements(StreamSynchronization::Measurements* stream, return 0; } +} // namespace ViESyncModule::ViESyncModule(VideoCodingModule* vcm) : vcm_(vcm), + clock_(Clock::GetRealTimeClock()), video_receiver_(NULL), video_rtp_rtcp_(NULL), voe_channel_id_(-1), voe_sync_interface_(NULL), last_sync_time_(TickTime::Now()), - sync_() { -} + sync_() {} ViESyncModule::~ViESyncModule() { } @@ -157,4 +160,37 @@ void ViESyncModule::Process() { vcm_->SetMinimumPlayoutDelay(target_video_delay_ms); } +bool ViESyncModule::GetStreamSyncOffsetInMs(const VideoFrame& frame, + int64_t* stream_offset_ms) const { + rtc::CritScope lock(&data_cs_); + if (voe_channel_id_ == -1) + return false; + + uint32_t playout_timestamp = 0; + if (voe_sync_interface_->GetPlayoutTimestamp(voe_channel_id_, + playout_timestamp) != 0) { + return false; + } + + int64_t latest_audio_ntp; + if (!RtpToNtpMs(playout_timestamp, audio_measurement_.rtcp, + &latest_audio_ntp)) { + return false; + } + + int64_t latest_video_ntp; + if (!RtpToNtpMs(frame.timestamp(), video_measurement_.rtcp, + &latest_video_ntp)) { + return false; + } + + int64_t time_to_render_ms = + frame.render_time_ms() - clock_->TimeInMilliseconds(); + if (time_to_render_ms > 0) + latest_video_ntp += time_to_render_ms; + + *stream_offset_ms = latest_audio_ntp - latest_video_ntp; + return true; +} + } // namespace webrtc diff --git a/webrtc/video/vie_sync_module.h b/webrtc/video/vie_sync_module.h index a5dff437f0..2b499ff4d3 100644 --- a/webrtc/video/vie_sync_module.h +++ b/webrtc/video/vie_sync_module.h @@ -24,8 +24,10 @@ namespace webrtc { +class Clock; class RtpRtcp; class VideoCodingModule; +class VideoFrame; class ViEChannel; class VoEVideoSync; @@ -43,9 +45,15 @@ class ViESyncModule : public Module { int64_t TimeUntilNextProcess() override; void Process() override; + // Gets the sync offset between the current played out audio frame and the + // video |frame|. Returns true on success, false otherwise. + bool GetStreamSyncOffsetInMs(const VideoFrame& frame, + int64_t* stream_offset_ms) const; + private: rtc::CriticalSection data_cs_; VideoCodingModule* const vcm_; + Clock* const clock_; RtpReceiver* video_receiver_; RtpRtcp* video_rtp_rtcp_; int voe_channel_id_; diff --git a/webrtc/video_receive_stream.h b/webrtc/video_receive_stream.h index 51d842616b..249523c29b 100644 --- a/webrtc/video_receive_stream.h +++ b/webrtc/video_receive_stream.h @@ -11,6 +11,7 @@ #ifndef WEBRTC_VIDEO_RECEIVE_STREAM_H_ #define WEBRTC_VIDEO_RECEIVE_STREAM_H_ +#include #include #include #include @@ -66,6 +67,8 @@ class VideoReceiveStream : public ReceiveStream { int total_bitrate_bps = 0; int discarded_packets = 0; + int sync_offset_ms = std::numeric_limits::max(); + uint32_t ssrc = 0; std::string c_name; StreamDataCounters rtp_stats;