From cd70119a106e42ab7eac58050d067bc050610739 Mon Sep 17 00:00:00 2001 From: "wu@webrtc.org" Date: Thu, 24 Apr 2014 22:10:24 +0000 Subject: [PATCH] Calculate local/remote clock delta and capture ntp timestamp in receiver's timebase. BUG=3111 TEST=new performance tests R=niklas.enbom@webrtc.org, stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/11689004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@5976 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../include/rtp_to_ntp.h | 9 + .../remote_bitrate_estimator/rtp_to_ntp.cc | 34 +++ .../modules/rtp_rtcp/source/rtp_rtcp_impl.cc | 7 +- webrtc/test/fake_decoder.cc | 1 + webrtc/test/frame_generator_capturer.cc | 6 +- webrtc/test/frame_generator_capturer.h | 4 + webrtc/video/call_perf_tests.cc | 193 ++++++++++++++++++ webrtc/video_engine/vie_receiver.cc | 82 +++++++- webrtc/video_engine/vie_receiver.h | 12 ++ webrtc/video_engine/vie_sync_module.cc | 31 ++- 10 files changed, 355 insertions(+), 24 deletions(-) diff --git a/webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h b/webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h index 7928abfacb..08a4d46251 100644 --- a/webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h +++ b/webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h @@ -29,6 +29,15 @@ struct RtcpMeasurement { typedef std::list RtcpList; +// Updates |rtcp_list| with timestamps from the latest RTCP SR. +// |new_rtcp_sr| will be set to true if these are the timestamps which have +// never be added to |rtcp_list|. +bool UpdateRtcpList(uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t rtp_timestamp, + RtcpList* rtcp_list, + bool* new_rtcp_sr); + // Converts an RTP timestamp to the NTP domain in milliseconds using two // (RTP timestamp, NTP timestamp) pairs. bool RtpToNtpMs(int64_t rtp_timestamp, const RtcpList& rtcp, diff --git a/webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc b/webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc index 109edae7cc..775cd0df0d 100644 --- a/webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc +++ b/webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc @@ -57,6 +57,40 @@ bool CompensateForWrapAround(uint32_t new_timestamp, return true; } +bool UpdateRtcpList(uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t rtp_timestamp, + RtcpList* rtcp_list, + bool* new_rtcp_sr) { + *new_rtcp_sr = false; + if (ntp_secs == 0 && ntp_frac == 0) { + return false; + } + + RtcpMeasurement measurement; + measurement.ntp_secs = ntp_secs; + measurement.ntp_frac = ntp_frac; + measurement.rtp_timestamp = rtp_timestamp; + + for (RtcpList::iterator it = rtcp_list->begin(); + it != rtcp_list->end(); ++it) { + if (measurement.ntp_secs == (*it).ntp_secs && + measurement.ntp_frac == (*it).ntp_frac) { + // This RTCP has already been added to the list. + return true; + } + } + + // We need two RTCP SR reports to map between RTP and NTP. More than two will + // not improve the mapping. + if (rtcp_list->size() == 2) { + rtcp_list->pop_back(); + } + rtcp_list->push_front(measurement); + *new_rtcp_sr = true; + return true; +} + // Converts |rtp_timestamp| to the NTP time base using the NTP and RTP timestamp // pairs in |rtcp|. The converted timestamp is returned in // |rtp_timestamp_in_ms|. This function compensates for wrap arounds in RTP diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index dc65336bd3..70a344346d 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -733,7 +733,12 @@ int32_t ModuleRtpRtcpImpl::RTT(const uint32_t remote_ssrc, uint16_t* avg_rtt, uint16_t* min_rtt, uint16_t* max_rtt) const { - return rtcp_receiver_.RTT(remote_ssrc, rtt, avg_rtt, min_rtt, max_rtt); + int32_t ret = rtcp_receiver_.RTT(remote_ssrc, rtt, avg_rtt, min_rtt, max_rtt); + if (rtt && *rtt == 0) { + // Try to get RTT from RtcpRttStats class. + *rtt = static_cast(rtt_ms()); + } + return ret; } // Reset RoundTripTime statistics. diff --git a/webrtc/test/fake_decoder.cc b/webrtc/test/fake_decoder.cc index a9e6f50e16..1624ea0144 100644 --- a/webrtc/test/fake_decoder.cc +++ b/webrtc/test/fake_decoder.cc @@ -36,6 +36,7 @@ int32_t FakeDecoder::Decode(const EncodedImage& input, const CodecSpecificInfo* codec_specific_info, int64_t render_time_ms) { frame_.set_timestamp(input._timeStamp); + frame_.set_ntp_time_ms(input.ntp_time_ms_); frame_.set_render_time_ms(render_time_ms); callback_->Decoded(frame_); diff --git a/webrtc/test/frame_generator_capturer.cc b/webrtc/test/frame_generator_capturer.cc index 6570c6f154..d1c17c7900 100644 --- a/webrtc/test/frame_generator_capturer.cc +++ b/webrtc/test/frame_generator_capturer.cc @@ -67,7 +67,8 @@ FrameGeneratorCapturer::FrameGeneratorCapturer(Clock* clock, tick_(EventWrapper::Create()), lock_(CriticalSectionWrapper::CreateCriticalSection()), frame_generator_(frame_generator), - target_fps_(target_fps) { + target_fps_(target_fps), + first_frame_capture_time_(-1) { assert(input != NULL); assert(frame_generator != NULL); assert(target_fps > 0); @@ -113,6 +114,9 @@ void FrameGeneratorCapturer::InsertFrame() { if (sending_) { I420VideoFrame* frame = frame_generator_->NextFrame(); frame->set_render_time_ms(clock_->CurrentNtpInMilliseconds()); + if (first_frame_capture_time_ == -1) { + first_frame_capture_time_ = frame->render_time_ms(); + } input_->SwapFrame(frame); } } diff --git a/webrtc/test/frame_generator_capturer.h b/webrtc/test/frame_generator_capturer.h index 5be6bb2cb1..84b3c4951f 100644 --- a/webrtc/test/frame_generator_capturer.h +++ b/webrtc/test/frame_generator_capturer.h @@ -43,6 +43,8 @@ class FrameGeneratorCapturer : public VideoCapturer { virtual void Start() OVERRIDE; virtual void Stop() OVERRIDE; + int64_t first_frame_capture_time() const { return first_frame_capture_time_; } + private: FrameGeneratorCapturer(Clock* clock, VideoSendStreamInput* input, @@ -61,6 +63,8 @@ class FrameGeneratorCapturer : public VideoCapturer { scoped_ptr frame_generator_; int target_fps_; + + int64_t first_frame_capture_time_; }; } // test } // webrtc diff --git a/webrtc/video/call_perf_tests.cc b/webrtc/video/call_perf_tests.cc index a5effe1d9a..85d7e73563 100644 --- a/webrtc/video/call_perf_tests.cc +++ b/webrtc/video/call_perf_tests.cc @@ -80,6 +80,11 @@ class CallPerfTest : public ::testing::Test { void TestMinTransmitBitrate(bool pad_to_min_bitrate); + void TestCaptureNtpTime(const FakeNetworkPipe::Config& net_config, + int threshold_ms, + int start_time_ms, + int run_time_ms); + VideoSendStream* send_stream_; test::FakeEncoder fake_encoder_; }; @@ -343,6 +348,194 @@ TEST_F(CallPerfTest, PlaysOutAudioAndVideoInSync) { VoiceEngine::Delete(voice_engine); } +class CaptureNtpTimeObserver : public test::RtpRtcpObserver, + public VideoRenderer { + public: + CaptureNtpTimeObserver(Clock* clock, + const FakeNetworkPipe::Config& config, + int threshold_ms, + int start_time_ms, + int run_time_ms) + : RtpRtcpObserver(kLongTimeoutMs, config), + clock_(clock), + threshold_ms_(threshold_ms), + start_time_ms_(start_time_ms), + run_time_ms_(run_time_ms), + creation_time_ms_(clock_->TimeInMilliseconds()), + capturer_(NULL), + rtp_start_timestamp_set_(false), + rtp_start_timestamp_(0) {} + + virtual void RenderFrame(const I420VideoFrame& video_frame, + int time_to_render_ms) OVERRIDE { + if (video_frame.ntp_time_ms() <= 0) { + // Haven't got enough RTCP SR in order to calculate the capture ntp time. + return; + } + + int64_t now_ms = clock_->TimeInMilliseconds(); + int64_t time_since_creation = now_ms - creation_time_ms_; + if (time_since_creation < start_time_ms_) { + // Wait for |start_time_ms_| before start measuring. + return; + } + + if (time_since_creation > run_time_ms_) { + observation_complete_->Set(); + } + + FrameCaptureTimeList::iterator iter = + capture_time_list_.find(video_frame.timestamp()); + EXPECT_TRUE(iter != capture_time_list_.end()); + + // The real capture time has been wrapped to uint32_t before converted + // to rtp timestamp in the sender side. So here we convert the estimated + // capture time to a uint32_t 90k timestamp also for comparing. + uint32_t estimated_capture_timestamp = + 90 * static_cast(video_frame.ntp_time_ms()); + uint32_t real_capture_timestamp = iter->second; + int time_offset_ms = real_capture_timestamp - estimated_capture_timestamp; + time_offset_ms = time_offset_ms / 90; + std::stringstream ss; + ss << time_offset_ms; + + webrtc::test::PrintResult("capture_ntp_time", + "", + "real - estimated", + ss.str(), + "ms", + true); + EXPECT_TRUE(std::abs(time_offset_ms) < threshold_ms_); + } + + virtual Action OnSendRtp(const uint8_t* packet, size_t length) { + RTPHeader header; + EXPECT_TRUE(parser_->Parse(packet, static_cast(length), &header)); + + if (!rtp_start_timestamp_set_) { + // Calculate the rtp timestamp offset in order to calculate the real + // capture time. + uint32_t first_capture_timestamp = + 90 * static_cast(capturer_->first_frame_capture_time()); + rtp_start_timestamp_ = header.timestamp - first_capture_timestamp; + rtp_start_timestamp_set_ = true; + } + + uint32_t capture_timestamp = header.timestamp - rtp_start_timestamp_; + capture_time_list_.insert(capture_time_list_.end(), + std::make_pair(header.timestamp, + capture_timestamp)); + return SEND_PACKET; + } + + void SetCapturer(test::FrameGeneratorCapturer* capturer) { + capturer_ = capturer; + } + + private: + Clock* clock_; + int threshold_ms_; + int start_time_ms_; + int run_time_ms_; + int64_t creation_time_ms_; + test::FrameGeneratorCapturer* capturer_; + bool rtp_start_timestamp_set_; + uint32_t rtp_start_timestamp_; + typedef std::map FrameCaptureTimeList; + FrameCaptureTimeList capture_time_list_; +}; + +void CallPerfTest::TestCaptureNtpTime(const FakeNetworkPipe::Config& net_config, + int threshold_ms, + int start_time_ms, + int run_time_ms) { + CaptureNtpTimeObserver observer(Clock::GetRealTimeClock(), + net_config, + threshold_ms, + start_time_ms, + run_time_ms); + + // Sender/receiver call. + Call::Config receiver_config(observer.ReceiveTransport()); + scoped_ptr receiver_call(Call::Create(receiver_config)); + scoped_ptr sender_call( + Call::Create(Call::Config(observer.SendTransport()))); + observer.SetReceivers(receiver_call->Receiver(), sender_call->Receiver()); + + // Configure send stream. + VideoSendStream::Config send_config = GetSendTestConfig(sender_call.get()); + VideoSendStream* send_stream = + sender_call->CreateVideoSendStream(send_config); + scoped_ptr capturer( + test::FrameGeneratorCapturer::Create( + send_stream->Input(), + send_config.encoder_settings.streams[0].width, + send_config.encoder_settings.streams[0].height, + 30, + Clock::GetRealTimeClock())); + observer.SetCapturer(capturer.get()); + + // Configure receive stream. + VideoReceiveStream::Config receive_config = + receiver_call->GetDefaultReceiveConfig(); + assert(receive_config.codecs.empty()); + VideoCodec codec = + test::CreateDecoderVideoCodec(send_config.encoder_settings); + receive_config.codecs.push_back(codec); + assert(receive_config.external_decoders.empty()); + ExternalVideoDecoder decoder; + test::FakeDecoder fake_decoder; + decoder.decoder = &fake_decoder; + decoder.payload_type = send_config.encoder_settings.payload_type; + receive_config.external_decoders.push_back(decoder); + receive_config.rtp.remote_ssrc = send_config.rtp.ssrcs[0]; + receive_config.rtp.local_ssrc = kReceiverLocalSsrc; + receive_config.renderer = &observer; + // Enable the receiver side rtt calculation. + receive_config.rtp.rtcp_xr.receiver_reference_time_report = true; + VideoReceiveStream* receive_stream = + receiver_call->CreateVideoReceiveStream(receive_config); + + // Start the test + receive_stream->Start(); + send_stream->Start(); + capturer->Start(); + + EXPECT_EQ(kEventSignaled, observer.Wait()) + << "Timed out while waiting for estimated capture ntp time to be " + << "within bounds."; + + capturer->Stop(); + send_stream->Stop(); + receive_stream->Stop(); + observer.StopSending(); + + sender_call->DestroyVideoSendStream(send_stream); + receiver_call->DestroyVideoReceiveStream(receive_stream); +} + +TEST_F(CallPerfTest, CaptureNtpTimeWithNetworkDelay) { + FakeNetworkPipe::Config net_config; + net_config.queue_delay_ms = 100; + // TODO(wu): lower the threshold as the calculation/estimatation becomes more + // accurate. + const int kThresholdMs = 30; + const int kStartTimeMs = 10000; + const int kRunTimeMs = 20000; + TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs); +} + +TEST_F(CallPerfTest, CaptureNtpTimeWithNetworkJitter) { + FakeNetworkPipe::Config net_config; + net_config.delay_standard_deviation_ms = 10; + // TODO(wu): lower the threshold as the calculation/estimatation becomes more + // accurate. + const int kThresholdMs = 30; + const int kStartTimeMs = 10000; + const int kRunTimeMs = 20000; + TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs); +} + TEST_F(CallPerfTest, RegisterCpuOveruseObserver) { // Verifies that either a normal or overuse callback is triggered. class OveruseCallbackObserver : public test::RtpRtcpObserver, diff --git a/webrtc/video_engine/vie_receiver.cc b/webrtc/video_engine/vie_receiver.cc index 75f53abb26..76228b3afe 100644 --- a/webrtc/video_engine/vie_receiver.cc +++ b/webrtc/video_engine/vie_receiver.cc @@ -21,7 +21,9 @@ #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h" #include "webrtc/modules/utility/interface/rtp_dump.h" #include "webrtc/modules/video_coding/main/interface/video_coding.h" +#include "webrtc/modules/video_coding/main/source/timestamp_extrapolator.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" +#include "webrtc/system_wrappers/interface/logging.h" #include "webrtc/system_wrappers/interface/tick_util.h" #include "webrtc/system_wrappers/interface/trace.h" @@ -45,6 +47,8 @@ ViEReceiver::ViEReceiver(const int32_t channel_id, rtp_rtcp_(NULL), vcm_(module_vcm), remote_bitrate_estimator_(remote_bitrate_estimator), + clock_(Clock::GetRealTimeClock()), + ts_extrapolator_(new VCMTimestampExtrapolator(clock_)), rtp_dump_(NULL), receiving_(false), restored_packet_in_use_(false), @@ -171,14 +175,37 @@ int ViEReceiver::ReceivedRTCPPacket(const void* rtcp_packet, int32_t ViEReceiver::OnReceivedPayloadData( const uint8_t* payload_data, const uint16_t payload_size, const WebRtcRTPHeader* rtp_header) { - // TODO(wu): Calculate ntp_time_ms - if (vcm_->IncomingPacket(payload_data, payload_size, *rtp_header) != 0) { + WebRtcRTPHeader rtp_header_with_ntp = *rtp_header; + CalculateCaptureNtpTime(&rtp_header_with_ntp); + if (vcm_->IncomingPacket(payload_data, + payload_size, + rtp_header_with_ntp) != 0) { // Check this... return -1; } return 0; } +void ViEReceiver::CalculateCaptureNtpTime(WebRtcRTPHeader* rtp_header) { + if (rtcp_list_.size() < 2) { + // We need two RTCP SR reports to calculate NTP. + return; + } + + int64_t sender_capture_ntp_ms = 0; + if (!synchronization::RtpToNtpMs(rtp_header->header.timestamp, + rtcp_list_, + &sender_capture_ntp_ms)) { + return; + } + uint32_t timestamp = sender_capture_ntp_ms * 90; + int64_t receiver_capture_ms = + ts_extrapolator_->ExtrapolateLocalTime(timestamp); + int64_t ntp_offset = + clock_->CurrentNtpInMilliseconds() - clock_->TimeInMilliseconds(); + rtp_header->ntp_time_ms = receiver_capture_ms + ntp_offset; +} + bool ViEReceiver::OnRecoveredPacket(const uint8_t* rtp_packet, int rtp_packet_length) { RTPHeader header; @@ -329,7 +356,56 @@ int ViEReceiver::InsertRTCPPacket(const uint8_t* rtcp_packet, } } assert(rtp_rtcp_); // Should be set by owner at construction time. - return rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length); + int ret = rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length); + if (ret != 0) { + return ret; + } + + if (!GetRtcpTimestamp()) { + LOG(LS_WARNING) << "Failed to retrieve timestamp information from RTCP SR."; + } + + return 0; +} + +bool ViEReceiver::GetRtcpTimestamp() { + uint16_t rtt = 0; + rtp_rtcp_->RTT(rtp_receiver_->SSRC(), &rtt, NULL, NULL, NULL); + if (rtt == 0) { + // Waiting for valid rtt. + return true; + } + + // Update RTCP list + uint32_t ntp_secs = 0; + uint32_t ntp_frac = 0; + uint32_t rtp_timestamp = 0; + if (0 != rtp_rtcp_->RemoteNTP(&ntp_secs, + &ntp_frac, + NULL, + NULL, + &rtp_timestamp)) { + return false; + } + + bool new_rtcp_sr = false; + if (!synchronization::UpdateRtcpList( + ntp_secs, ntp_frac, rtp_timestamp, &rtcp_list_, &new_rtcp_sr)) { + return false; + } + + if (!new_rtcp_sr) { + // No new RTCP SR since last time this function was called. + return true; + } + + // Update extrapolator with the new arrival time. + // The extrapolator assumes the TimeInMilliseconds time. + int64_t receiver_arrival_time = clock_->TimeInMilliseconds(); + int64_t sender_send_time_ms = Clock::NtpToMs(ntp_secs, ntp_frac); + int64_t sender_arrival_time_90k = (sender_send_time_ms + rtt / 2) * 90; + ts_extrapolator_->Update(receiver_arrival_time, sender_arrival_time_90k); + return true; } void ViEReceiver::StartReceive() { diff --git a/webrtc/video_engine/vie_receiver.h b/webrtc/video_engine/vie_receiver.h index b987c99cac..512a7b656b 100644 --- a/webrtc/video_engine/vie_receiver.h +++ b/webrtc/video_engine/vie_receiver.h @@ -14,6 +14,7 @@ #include #include "webrtc/engine_configurations.h" +#include "webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h" #include "webrtc/modules/rtp_rtcp/interface/receive_statistics.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h" #include "webrtc/system_wrappers/interface/scoped_ptr.h" @@ -21,6 +22,9 @@ #include "webrtc/video_engine/include/vie_network.h" #include "webrtc/video_engine/vie_defines.h" +// TODO(wu): Move rtp_to_ntp.h and timestamp_extrapolator.h to somewhere that +// can be shared between audio and video. + namespace webrtc { class CriticalSectionWrapper; @@ -32,6 +36,7 @@ class RtpHeaderParser; class RTPPayloadRegistry; class RtpReceiver; class RtpRtcp; +class VCMTimestampExtrapolator; class VideoCodingModule; struct ReceiveBandwidthEstimatorStats; @@ -105,6 +110,9 @@ class ViEReceiver : public RtpData { bool IsPacketInOrder(const RTPHeader& header) const; bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const; + bool GetRtcpTimestamp(); + void CalculateCaptureNtpTime(WebRtcRTPHeader* rtp_header); + scoped_ptr receive_cs_; const int32_t channel_id_; scoped_ptr rtp_header_parser_; @@ -117,6 +125,10 @@ class ViEReceiver : public RtpData { VideoCodingModule* vcm_; RemoteBitrateEstimator* remote_bitrate_estimator_; + Clock* clock_; + scoped_ptr ts_extrapolator_; + synchronization::RtcpList rtcp_list_; + RtpDump* rtp_dump_; bool receiving_; uint8_t restored_packet_[kViEMaxMtu]; diff --git a/webrtc/video_engine/vie_sync_module.cc b/webrtc/video_engine/vie_sync_module.cc index 89da022bf5..b7c74a75cf 100644 --- a/webrtc/video_engine/vie_sync_module.cc +++ b/webrtc/video_engine/vie_sync_module.cc @@ -30,31 +30,24 @@ int UpdateMeasurements(StreamSynchronization::Measurements* stream, return -1; if (!receiver.LastReceivedTimeMs(&stream->latest_receive_time_ms)) return -1; - synchronization::RtcpMeasurement measurement; - if (0 != rtp_rtcp.RemoteNTP(&measurement.ntp_secs, - &measurement.ntp_frac, + + uint32_t ntp_secs = 0; + uint32_t ntp_frac = 0; + uint32_t rtp_timestamp = 0; + if (0 != rtp_rtcp.RemoteNTP(&ntp_secs, + &ntp_frac, NULL, NULL, - &measurement.rtp_timestamp)) { + &rtp_timestamp)) { return -1; } - if (measurement.ntp_secs == 0 && measurement.ntp_frac == 0) { + + bool new_rtcp_sr = false; + if (!synchronization::UpdateRtcpList( + ntp_secs, ntp_frac, rtp_timestamp, &stream->rtcp, &new_rtcp_sr)) { return -1; } - for (synchronization::RtcpList::iterator it = stream->rtcp.begin(); - it != stream->rtcp.end(); ++it) { - if (measurement.ntp_secs == (*it).ntp_secs && - measurement.ntp_frac == (*it).ntp_frac) { - // This RTCP has already been added to the list. - return 0; - } - } - // We need two RTCP SR reports to map between RTP and NTP. More than two will - // not improve the mapping. - if (stream->rtcp.size() == 2) { - stream->rtcp.pop_back(); - } - stream->rtcp.push_front(measurement); + return 0; }