admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Calculate local/remote clock delta and capture ntp timestamp in receiver's timebase.

Browse Source 
BUG=3111
TEST=new performance tests
R=niklas.enbom@webrtc.org, stefan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/11689004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@5976 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
wu@webrtc.org 2014-04-24 22:10:24 +00:00
parent 93fd25c20c
commit cd70119a10
10 changed files with 355 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h
View File

@ -29,6 +29,15 @@ struct RtcpMeasurement {
typedef std::list<RtcpMeasurement> RtcpList;
// Updates |rtcp_list| with timestamps from the latest RTCP SR.
// |new_rtcp_sr| will be set to true if these are the timestamps which have
// never be added to |rtcp_list|.
bool UpdateRtcpList(uint32_t ntp_secs,
uint32_t ntp_frac,
uint32_t rtp_timestamp,
RtcpList* rtcp_list,
bool* new_rtcp_sr);
// Converts an RTP timestamp to the NTP domain in milliseconds using two
// (RTP timestamp, NTP timestamp) pairs.
bool RtpToNtpMs(int64_t rtp_timestamp, const RtcpList& rtcp,

34
webrtc/modules/remote_bitrate_estimator/rtp_to_ntp.cc
View File

@ -57,6 +57,40 @@ bool CompensateForWrapAround(uint32_t new_timestamp,
return true;
}
bool UpdateRtcpList(uint32_t ntp_secs,
uint32_t ntp_frac,
uint32_t rtp_timestamp,
RtcpList* rtcp_list,
bool* new_rtcp_sr) {
*new_rtcp_sr = false;
if (ntp_secs == 0 && ntp_frac == 0) {
return false;
}
RtcpMeasurement measurement;
measurement.ntp_secs = ntp_secs;
measurement.ntp_frac = ntp_frac;
measurement.rtp_timestamp = rtp_timestamp;
for (RtcpList::iterator it = rtcp_list->begin();
it != rtcp_list->end(); ++it) {
if (measurement.ntp_secs == (*it).ntp_secs &&
measurement.ntp_frac == (*it).ntp_frac) {
// This RTCP has already been added to the list.
return true;
}
}
// We need two RTCP SR reports to map between RTP and NTP. More than two will
// not improve the mapping.
if (rtcp_list->size() == 2) {
rtcp_list->pop_back();
}
rtcp_list->push_front(measurement);
*new_rtcp_sr = true;
return true;
}
// Converts |rtp_timestamp| to the NTP time base using the NTP and RTP timestamp
// pairs in |rtcp|. The converted timestamp is returned in
// |rtp_timestamp_in_ms|. This function compensates for wrap arounds in RTP

7
webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
View File

@ -733,7 +733,12 @@ int32_t ModuleRtpRtcpImpl::RTT(const uint32_t remote_ssrc,
uint16_t* avg_rtt,
uint16_t* min_rtt,
uint16_t* max_rtt) const {
return rtcp_receiver_.RTT(remote_ssrc, rtt, avg_rtt, min_rtt, max_rtt);
int32_t ret = rtcp_receiver_.RTT(remote_ssrc, rtt, avg_rtt, min_rtt, max_rtt);
if (rtt && *rtt == 0) {
// Try to get RTT from RtcpRttStats class.
*rtt = static_cast<uint16_t>(rtt_ms());
}
return ret;
}
// Reset RoundTripTime statistics.

1
webrtc/test/fake_decoder.cc
View File

@ -36,6 +36,7 @@ int32_t FakeDecoder::Decode(const EncodedImage& input,
const CodecSpecificInfo* codec_specific_info,
int64_t render_time_ms) {
frame_.set_timestamp(input._timeStamp);
frame_.set_ntp_time_ms(input.ntp_time_ms_);
frame_.set_render_time_ms(render_time_ms);
callback_->Decoded(frame_);

6
webrtc/test/frame_generator_capturer.cc
View File

@ -67,7 +67,8 @@ FrameGeneratorCapturer::FrameGeneratorCapturer(Clock* clock,
tick_(EventWrapper::Create()),
lock_(CriticalSectionWrapper::CreateCriticalSection()),
frame_generator_(frame_generator),
target_fps_(target_fps) {
target_fps_(target_fps),
first_frame_capture_time_(-1) {
assert(input != NULL);
assert(frame_generator != NULL);
assert(target_fps > 0);
@ -113,6 +114,9 @@ void FrameGeneratorCapturer::InsertFrame() {
if (sending_) {
I420VideoFrame* frame = frame_generator_->NextFrame();
frame->set_render_time_ms(clock_->CurrentNtpInMilliseconds());
if (first_frame_capture_time_ == -1) {
first_frame_capture_time_ = frame->render_time_ms();
}
input_->SwapFrame(frame);
}
}

4
webrtc/test/frame_generator_capturer.h
View File

@ -43,6 +43,8 @@ class FrameGeneratorCapturer : public VideoCapturer {
virtual void Start() OVERRIDE;
virtual void Stop() OVERRIDE;
int64_t first_frame_capture_time() const { return first_frame_capture_time_; }
private:
FrameGeneratorCapturer(Clock* clock,
VideoSendStreamInput* input,
@ -61,6 +63,8 @@ class FrameGeneratorCapturer : public VideoCapturer {
scoped_ptr<FrameGenerator> frame_generator_;
int target_fps_;
int64_t first_frame_capture_time_;
};
} // test
} // webrtc

193
webrtc/video/call_perf_tests.cc
View File

@ -80,6 +80,11 @@ class CallPerfTest : public ::testing::Test {
void TestMinTransmitBitrate(bool pad_to_min_bitrate);
void TestCaptureNtpTime(const FakeNetworkPipe::Config& net_config,
int threshold_ms,
int start_time_ms,
int run_time_ms);
VideoSendStream* send_stream_;
test::FakeEncoder fake_encoder_;
};
@ -343,6 +348,194 @@ TEST_F(CallPerfTest, PlaysOutAudioAndVideoInSync) {
VoiceEngine::Delete(voice_engine);
}
class CaptureNtpTimeObserver : public test::RtpRtcpObserver,
public VideoRenderer {
public:
CaptureNtpTimeObserver(Clock* clock,
const FakeNetworkPipe::Config& config,
int threshold_ms,
int start_time_ms,
int run_time_ms)
: RtpRtcpObserver(kLongTimeoutMs, config),
clock_(clock),
threshold_ms_(threshold_ms),
start_time_ms_(start_time_ms),
run_time_ms_(run_time_ms),
creation_time_ms_(clock_->TimeInMilliseconds()),
capturer_(NULL),
rtp_start_timestamp_set_(false),
rtp_start_timestamp_(0) {}
virtual void RenderFrame(const I420VideoFrame& video_frame,
int time_to_render_ms) OVERRIDE {
if (video_frame.ntp_time_ms() <= 0) {
// Haven't got enough RTCP SR in order to calculate the capture ntp time.
return;
}
int64_t now_ms = clock_->TimeInMilliseconds();
int64_t time_since_creation = now_ms - creation_time_ms_;
if (time_since_creation < start_time_ms_) {
// Wait for |start_time_ms_| before start measuring.
return;
}
if (time_since_creation > run_time_ms_) {
observation_complete_->Set();
}
FrameCaptureTimeList::iterator iter =
capture_time_list_.find(video_frame.timestamp());
EXPECT_TRUE(iter != capture_time_list_.end());
// The real capture time has been wrapped to uint32_t before converted
// to rtp timestamp in the sender side. So here we convert the estimated
// capture time to a uint32_t 90k timestamp also for comparing.
uint32_t estimated_capture_timestamp =
90 * static_cast<uint32_t>(video_frame.ntp_time_ms());
uint32_t real_capture_timestamp = iter->second;
int time_offset_ms = real_capture_timestamp - estimated_capture_timestamp;
time_offset_ms = time_offset_ms / 90;
std::stringstream ss;
ss << time_offset_ms;
webrtc::test::PrintResult("capture_ntp_time",
"",
"real - estimated",
ss.str(),
"ms",
true);
EXPECT_TRUE(std::abs(time_offset_ms) < threshold_ms_);
}
virtual Action OnSendRtp(const uint8_t* packet, size_t length) {
RTPHeader header;
EXPECT_TRUE(parser_->Parse(packet, static_cast<int>(length), &header));
if (!rtp_start_timestamp_set_) {
// Calculate the rtp timestamp offset in order to calculate the real
// capture time.
uint32_t first_capture_timestamp =
90 * static_cast<uint32_t>(capturer_->first_frame_capture_time());
rtp_start_timestamp_ = header.timestamp - first_capture_timestamp;
rtp_start_timestamp_set_ = true;
}
uint32_t capture_timestamp = header.timestamp - rtp_start_timestamp_;
capture_time_list_.insert(capture_time_list_.end(),
std::make_pair(header.timestamp,
capture_timestamp));
return SEND_PACKET;
}
void SetCapturer(test::FrameGeneratorCapturer* capturer) {
capturer_ = capturer;
}
private:
Clock* clock_;
int threshold_ms_;
int start_time_ms_;
int run_time_ms_;
int64_t creation_time_ms_;
test::FrameGeneratorCapturer* capturer_;
bool rtp_start_timestamp_set_;
uint32_t rtp_start_timestamp_;
typedef std::map<uint32_t, uint32_t> FrameCaptureTimeList;
FrameCaptureTimeList capture_time_list_;
};
void CallPerfTest::TestCaptureNtpTime(const FakeNetworkPipe::Config& net_config,
int threshold_ms,
int start_time_ms,
int run_time_ms) {
CaptureNtpTimeObserver observer(Clock::GetRealTimeClock(),
net_config,
threshold_ms,
start_time_ms,
run_time_ms);
// Sender/receiver call.
Call::Config receiver_config(observer.ReceiveTransport());
scoped_ptr<Call> receiver_call(Call::Create(receiver_config));
scoped_ptr<Call> sender_call(
Call::Create(Call::Config(observer.SendTransport())));
observer.SetReceivers(receiver_call->Receiver(), sender_call->Receiver());
// Configure send stream.
VideoSendStream::Config send_config = GetSendTestConfig(sender_call.get());
VideoSendStream* send_stream =
sender_call->CreateVideoSendStream(send_config);
scoped_ptr<test::FrameGeneratorCapturer> capturer(
test::FrameGeneratorCapturer::Create(
send_stream->Input(),
send_config.encoder_settings.streams[0].width,
send_config.encoder_settings.streams[0].height,
30,
Clock::GetRealTimeClock()));
observer.SetCapturer(capturer.get());
// Configure receive stream.
VideoReceiveStream::Config receive_config =
receiver_call->GetDefaultReceiveConfig();
assert(receive_config.codecs.empty());
VideoCodec codec =
test::CreateDecoderVideoCodec(send_config.encoder_settings);
receive_config.codecs.push_back(codec);
assert(receive_config.external_decoders.empty());
ExternalVideoDecoder decoder;
test::FakeDecoder fake_decoder;
decoder.decoder = &fake_decoder;
decoder.payload_type = send_config.encoder_settings.payload_type;
receive_config.external_decoders.push_back(decoder);
receive_config.rtp.remote_ssrc = send_config.rtp.ssrcs[0];
receive_config.rtp.local_ssrc = kReceiverLocalSsrc;
receive_config.renderer = &observer;
// Enable the receiver side rtt calculation.
receive_config.rtp.rtcp_xr.receiver_reference_time_report = true;
VideoReceiveStream* receive_stream =
receiver_call->CreateVideoReceiveStream(receive_config);
// Start the test
receive_stream->Start();
send_stream->Start();
capturer->Start();
EXPECT_EQ(kEventSignaled, observer.Wait())
<< "Timed out while waiting for estimated capture ntp time to be "
<< "within bounds.";
capturer->Stop();
send_stream->Stop();
receive_stream->Stop();
observer.StopSending();
sender_call->DestroyVideoSendStream(send_stream);
receiver_call->DestroyVideoReceiveStream(receive_stream);
}
TEST_F(CallPerfTest, CaptureNtpTimeWithNetworkDelay) {
FakeNetworkPipe::Config net_config;
net_config.queue_delay_ms = 100;
// TODO(wu): lower the threshold as the calculation/estimatation becomes more
// accurate.
const int kThresholdMs = 30;
const int kStartTimeMs = 10000;
const int kRunTimeMs = 20000;
TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs);
}
TEST_F(CallPerfTest, CaptureNtpTimeWithNetworkJitter) {
FakeNetworkPipe::Config net_config;
net_config.delay_standard_deviation_ms = 10;
// TODO(wu): lower the threshold as the calculation/estimatation becomes more
// accurate.
const int kThresholdMs = 30;
const int kStartTimeMs = 10000;
const int kRunTimeMs = 20000;
TestCaptureNtpTime(net_config, kThresholdMs, kStartTimeMs, kRunTimeMs);
}
TEST_F(CallPerfTest, RegisterCpuOveruseObserver) {
// Verifies that either a normal or overuse callback is triggered.
class OveruseCallbackObserver : public test::RtpRtcpObserver,

82
webrtc/video_engine/vie_receiver.cc
View File

@ -21,7 +21,9 @@
#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h"
#include "webrtc/modules/utility/interface/rtp_dump.h"
#include "webrtc/modules/video_coding/main/interface/video_coding.h"
#include "webrtc/modules/video_coding/main/source/timestamp_extrapolator.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/logging.h"
#include "webrtc/system_wrappers/interface/tick_util.h"
#include "webrtc/system_wrappers/interface/trace.h"
@ -45,6 +47,8 @@ ViEReceiver::ViEReceiver(const int32_t channel_id,
rtp_rtcp_(NULL),
vcm_(module_vcm),
remote_bitrate_estimator_(remote_bitrate_estimator),
clock_(Clock::GetRealTimeClock()),
ts_extrapolator_(new VCMTimestampExtrapolator(clock_)),
rtp_dump_(NULL),
receiving_(false),
restored_packet_in_use_(false),
@ -171,14 +175,37 @@ int ViEReceiver::ReceivedRTCPPacket(const void* rtcp_packet,
int32_t ViEReceiver::OnReceivedPayloadData(
const uint8_t* payload_data, const uint16_t payload_size,
const WebRtcRTPHeader* rtp_header) {
// TODO(wu): Calculate ntp_time_ms
if (vcm_->IncomingPacket(payload_data, payload_size, *rtp_header) != 0) {
WebRtcRTPHeader rtp_header_with_ntp = *rtp_header;
CalculateCaptureNtpTime(&rtp_header_with_ntp);
if (vcm_->IncomingPacket(payload_data,
payload_size,
rtp_header_with_ntp) != 0) {
// Check this...
return -1;
}
return 0;
}
void ViEReceiver::CalculateCaptureNtpTime(WebRtcRTPHeader* rtp_header) {
if (rtcp_list_.size() < 2) {
// We need two RTCP SR reports to calculate NTP.
return;
}
int64_t sender_capture_ntp_ms = 0;
if (!synchronization::RtpToNtpMs(rtp_header->header.timestamp,
rtcp_list_,
&sender_capture_ntp_ms)) {
return;
}
uint32_t timestamp = sender_capture_ntp_ms * 90;
int64_t receiver_capture_ms =
ts_extrapolator_->ExtrapolateLocalTime(timestamp);
int64_t ntp_offset =
clock_->CurrentNtpInMilliseconds() - clock_->TimeInMilliseconds();
rtp_header->ntp_time_ms = receiver_capture_ms + ntp_offset;
}
bool ViEReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
int rtp_packet_length) {
RTPHeader header;
@ -329,7 +356,56 @@ int ViEReceiver::InsertRTCPPacket(const uint8_t* rtcp_packet,
}
}
assert(rtp_rtcp_); // Should be set by owner at construction time.
return rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
int ret = rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
if (ret != 0) {
return ret;
}
if (!GetRtcpTimestamp()) {
LOG(LS_WARNING) << "Failed to retrieve timestamp information from RTCP SR.";
}
return 0;
}
bool ViEReceiver::GetRtcpTimestamp() {
uint16_t rtt = 0;
rtp_rtcp_->RTT(rtp_receiver_->SSRC(), &rtt, NULL, NULL, NULL);
if (rtt == 0) {
// Waiting for valid rtt.
return true;
}
// Update RTCP list
uint32_t ntp_secs = 0;
uint32_t ntp_frac = 0;
uint32_t rtp_timestamp = 0;
if (0 != rtp_rtcp_->RemoteNTP(&ntp_secs,
&ntp_frac,
NULL,
NULL,
&rtp_timestamp)) {
return false;
}
bool new_rtcp_sr = false;
if (!synchronization::UpdateRtcpList(
ntp_secs, ntp_frac, rtp_timestamp, &rtcp_list_, &new_rtcp_sr)) {
return false;
}
if (!new_rtcp_sr) {
// No new RTCP SR since last time this function was called.
return true;
}
// Update extrapolator with the new arrival time.
// The extrapolator assumes the TimeInMilliseconds time.
int64_t receiver_arrival_time = clock_->TimeInMilliseconds();
int64_t sender_send_time_ms = Clock::NtpToMs(ntp_secs, ntp_frac);
int64_t sender_arrival_time_90k = (sender_send_time_ms + rtt / 2) * 90;
ts_extrapolator_->Update(receiver_arrival_time, sender_arrival_time_90k);
return true;
}
void ViEReceiver::StartReceive() {

12
webrtc/video_engine/vie_receiver.h
View File

@ -14,6 +14,7 @@
#include <list>
#include "webrtc/engine_configurations.h"
#include "webrtc/modules/remote_bitrate_estimator/include/rtp_to_ntp.h"
#include "webrtc/modules/rtp_rtcp/interface/receive_statistics.h"
#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
@ -21,6 +22,9 @@
#include "webrtc/video_engine/include/vie_network.h"
#include "webrtc/video_engine/vie_defines.h"
// TODO(wu): Move rtp_to_ntp.h and timestamp_extrapolator.h to somewhere that
// can be shared between audio and video.
namespace webrtc {
class CriticalSectionWrapper;
@ -32,6 +36,7 @@ class RtpHeaderParser;
class RTPPayloadRegistry;
class RtpReceiver;
class RtpRtcp;
class VCMTimestampExtrapolator;
class VideoCodingModule;
struct ReceiveBandwidthEstimatorStats;
@ -105,6 +110,9 @@ class ViEReceiver : public RtpData {
bool IsPacketInOrder(const RTPHeader& header) const;
bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const;
bool GetRtcpTimestamp();
void CalculateCaptureNtpTime(WebRtcRTPHeader* rtp_header);
scoped_ptr<CriticalSectionWrapper> receive_cs_;
const int32_t channel_id_;
scoped_ptr<RtpHeaderParser> rtp_header_parser_;
@ -117,6 +125,10 @@ class ViEReceiver : public RtpData {
VideoCodingModule* vcm_;
RemoteBitrateEstimator* remote_bitrate_estimator_;
Clock* clock_;
scoped_ptr<VCMTimestampExtrapolator> ts_extrapolator_;
synchronization::RtcpList rtcp_list_;
RtpDump* rtp_dump_;
bool receiving_;
uint8_t restored_packet_[kViEMaxMtu];

31
webrtc/video_engine/vie_sync_module.cc
View File

@ -30,31 +30,24 @@ int UpdateMeasurements(StreamSynchronization::Measurements* stream,
return -1;
if (!receiver.LastReceivedTimeMs(&stream->latest_receive_time_ms))
return -1;
synchronization::RtcpMeasurement measurement;
if (0 != rtp_rtcp.RemoteNTP(&measurement.ntp_secs,
&measurement.ntp_frac,
uint32_t ntp_secs = 0;
uint32_t ntp_frac = 0;
uint32_t rtp_timestamp = 0;
if (0 != rtp_rtcp.RemoteNTP(&ntp_secs,
&ntp_frac,
NULL,
NULL,
&measurement.rtp_timestamp)) {
&rtp_timestamp)) {
return -1;
}
if (measurement.ntp_secs == 0 && measurement.ntp_frac == 0) {
bool new_rtcp_sr = false;
if (!synchronization::UpdateRtcpList(
ntp_secs, ntp_frac, rtp_timestamp, &stream->rtcp, &new_rtcp_sr)) {
return -1;
}
for (synchronization::RtcpList::iterator it = stream->rtcp.begin();
it != stream->rtcp.end(); ++it) {
if (measurement.ntp_secs == (*it).ntp_secs &&
measurement.ntp_frac == (*it).ntp_frac) {
// This RTCP has already been added to the list.
return 0;
}
}
// We need two RTCP SR reports to map between RTP and NTP. More than two will
// not improve the mapping.
if (stream->rtcp.size() == 2) {
stream->rtcp.pop_back();
}
stream->rtcp.push_front(measurement);
return 0;
}