admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First step in refactoring audio/video synchronization. Adds unittests.

Browse Source 
BUG=
TEST=stream_synchronization_unittest

Review URL: https://webrtc-codereview.appspot.com/669005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2455 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
stefan@webrtc.org 2012-06-28 07:51:16 +00:00
parent cee447a5bb
commit 5f28498149
6 changed files with 754 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

240
src/video_engine/stream_synchronization.cc Normal file
View File

@ -0,0 +1,240 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video_engine/stream_synchronization.h"
#include "system_wrappers/interface/trace.h"
namespace webrtc {
enum { kMaxVideoDiffMs = 80 };
enum { kMaxAudioDiffMs = 80 };
enum { kMaxDelay = 1500 };
const float FracMS = 4.294967296E6f;
struct ViESyncDelay {
ViESyncDelay() {
extra_video_delay_ms = 0;
last_video_delay_ms = 0;
extra_audio_delay_ms = 0;
last_sync_delay = 0;
network_delay = 120;
}
int extra_video_delay_ms;
int last_video_delay_ms;
int extra_audio_delay_ms;
int last_sync_delay;
int network_delay;
};
StreamSynchronization::StreamSynchronization(int audio_channel_id,
int video_channel_id)
: channel_delay_(new ViESyncDelay),
audio_channel_id_(audio_channel_id),
video_channel_id_(video_channel_id) {}
StreamSynchronization::~StreamSynchronization() {
delete channel_delay_;
}
int StreamSynchronization::ComputeDelays(const Measurements& audio,
int current_audio_delay_ms,
int* extra_audio_delay_ms,
const Measurements& video,
int* total_video_delay_target_ms) {
// ReceivedNTPxxx is NTP at sender side when sent.
// RTCPArrivalTimexxx is NTP at receiver side when received.
// can't use ConvertNTPTimeToMS since calculation can be
// negative
int NTPdiff = (audio.received_ntp_secs - video.received_ntp_secs)
* 1000; // ms
float ntp_diff_frac = audio.received_ntp_frac / FracMS -
video.received_ntp_frac / FracMS;
if (ntp_diff_frac > 0.0f)
NTPdiff += static_cast<int>(ntp_diff_frac + 0.5f);
else
NTPdiff += static_cast<int>(ntp_diff_frac - 0.5f);
int RTCPdiff = (audio.rtcp_arrivaltime_secs - video.rtcp_arrivaltime_secs)
* 1000; // ms
float rtcp_diff_frac = audio.rtcp_arrivaltime_frac / FracMS -
video.rtcp_arrivaltime_frac / FracMS;
if (rtcp_diff_frac > 0.0f)
RTCPdiff += static_cast<int>(rtcp_diff_frac + 0.5f);
else
RTCPdiff += static_cast<int>(rtcp_diff_frac - 0.5f);
int diff = NTPdiff - RTCPdiff;
// if diff is + video is behind
if (diff < -1000 || diff > 1000) {
// unresonable ignore value.
return -1;
}
channel_delay_->network_delay = diff;
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
"Audio delay is: %d for voice channel: %d",
current_audio_delay_ms, audio_channel_id_);
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
"Network delay diff is: %d for voice channel: %d",
channel_delay_->network_delay, audio_channel_id_);
// Calculate the difference between the lowest possible video delay and
// the current audio delay.
int current_diff_ms = *total_video_delay_target_ms - current_audio_delay_ms +
channel_delay_->network_delay;
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
"Current diff is: %d for audio channel: %d",
current_diff_ms, audio_channel_id_);
int video_delay_ms = 0;
if (current_diff_ms > 0) {
// The minimum video delay is longer than the current audio delay.
// We need to decrease extra video delay, if we have added extra delay
// earlier, or add extra audio delay.
if (channel_delay_->extra_video_delay_ms > 0) {
// We have extra delay added to ViE. Reduce this delay before adding
// extra delay to VoE.
// This is the desired delay, we can't reduce more than this.
video_delay_ms = *total_video_delay_target_ms;
// Check that we don't reduce the delay more than what is allowed.
if (video_delay_ms <
channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) {
video_delay_ms =
channel_delay_->last_video_delay_ms - kMaxVideoDiffMs;
channel_delay_->extra_video_delay_ms =
video_delay_ms - *total_video_delay_target_ms;
} else {
channel_delay_->extra_video_delay_ms = 0;
}
channel_delay_->last_video_delay_ms = video_delay_ms;
channel_delay_->last_sync_delay = -1;
channel_delay_->extra_audio_delay_ms = 0;
} else { // channel_delay_->extra_video_delay_ms > 0
// We have no extra video delay to remove, increase the audio delay.
if (channel_delay_->last_sync_delay >= 0) {
// We have increased the audio delay earlier, increase it even more.
int audio_diff_ms = current_diff_ms / 2;
if (audio_diff_ms > kMaxAudioDiffMs) {
// We only allow a maximum change of KMaxAudioDiffMS for audio
// due to NetEQ maximum changes.
audio_diff_ms = kMaxAudioDiffMs;
}
// Increase the audio delay
channel_delay_->extra_audio_delay_ms += audio_diff_ms;
// Don't set a too high delay.
if (channel_delay_->extra_audio_delay_ms > kMaxDelay) {
channel_delay_->extra_audio_delay_ms = kMaxDelay;
}
// Don't add any extra video delay.
video_delay_ms = *total_video_delay_target_ms;
channel_delay_->extra_video_delay_ms = 0;
channel_delay_->last_video_delay_ms = video_delay_ms;
channel_delay_->last_sync_delay = 1;
} else { // channel_delay_->last_sync_delay >= 0
// First time after a delay change, don't add any extra delay.
// This is to not toggle back and forth too much.
channel_delay_->extra_audio_delay_ms = 0;
// Set minimum video delay
video_delay_ms = *total_video_delay_target_ms;
channel_delay_->extra_video_delay_ms = 0;
channel_delay_->last_video_delay_ms = video_delay_ms;
channel_delay_->last_sync_delay = 0;
}
}
} else { // if (current_diffMS > 0)
// The minimum video delay is lower than the current audio delay.
// We need to decrease possible extra audio delay, or
// add extra video delay.
if (channel_delay_->extra_audio_delay_ms > 0) {
// We have extra delay in VoiceEngine
// Start with decreasing the voice delay
int audio_diff_ms = current_diff_ms / 2;
if (audio_diff_ms < -1 * kMaxAudioDiffMs) {
// Don't change the delay too much at once.
audio_diff_ms = -1 * kMaxAudioDiffMs;
}
// Add the negative difference.
channel_delay_->extra_audio_delay_ms += audio_diff_ms;
if (channel_delay_->extra_audio_delay_ms < 0) {
// Negative values not allowed.
channel_delay_->extra_audio_delay_ms = 0;
channel_delay_->last_sync_delay = 0;
} else {
// There is more audio delay to use for the next round.
channel_delay_->last_sync_delay = 1;
}
// Keep the video delay at the minimum values.
video_delay_ms = *total_video_delay_target_ms;
channel_delay_->extra_video_delay_ms = 0;
channel_delay_->last_video_delay_ms = video_delay_ms;
} else { // channel_delay_->extra_audio_delay_ms > 0
// We have no extra delay in VoiceEngine, increase the video delay.
channel_delay_->extra_audio_delay_ms = 0;
// Make the difference positive.
int video_diff_ms = -1 * current_diff_ms;
// This is the desired delay.
video_delay_ms = *total_video_delay_target_ms + video_diff_ms;
if (video_delay_ms > channel_delay_->last_video_delay_ms) {
if (video_delay_ms >
channel_delay_->last_video_delay_ms + kMaxVideoDiffMs) {
// Don't increase the delay too much at once
video_delay_ms =
channel_delay_->last_video_delay_ms + kMaxVideoDiffMs;
}
// Verify we don't go above the maximum allowed delay
if (video_delay_ms > kMaxDelay) {
video_delay_ms = kMaxDelay;
}
} else {
if (video_delay_ms <
channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) {
// Don't decrease the delay too much at once
video_delay_ms =
channel_delay_->last_video_delay_ms - kMaxVideoDiffMs;
}
// Verify we don't go below the minimum delay
if (video_delay_ms < *total_video_delay_target_ms) {
video_delay_ms = *total_video_delay_target_ms;
}
}
// Store the values
channel_delay_->extra_video_delay_ms =
video_delay_ms - *total_video_delay_target_ms;
channel_delay_->last_video_delay_ms = video_delay_ms;
channel_delay_->last_sync_delay = -1;
}
}
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
"Sync video delay %d ms for video channel and audio delay %d for audio "
"channel %d",
video_delay_ms, channel_delay_->extra_audio_delay_ms, audio_channel_id_);
*extra_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
if (video_delay_ms < 0) {
video_delay_ms = 0;
}
*total_video_delay_target_ms =
(*total_video_delay_target_ms > video_delay_ms) ?
*total_video_delay_target_ms : video_delay_ms;
return 0;
}
} // namespace webrtc

51
src/video_engine/stream_synchronization.h Normal file
View File

@ -0,0 +1,51 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_
#define WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_
#include "typedefs.h"
namespace webrtc {
struct ViESyncDelay;
class StreamSynchronization {
public:
struct Measurements {
Measurements()
: received_ntp_secs(0),
received_ntp_frac(0),
rtcp_arrivaltime_secs(0),
rtcp_arrivaltime_frac(0) {}
uint32_t received_ntp_secs;
uint32_t received_ntp_frac;
uint32_t rtcp_arrivaltime_secs;
uint32_t rtcp_arrivaltime_frac;
};
StreamSynchronization(int audio_channel_id, int video_channel_id);
~StreamSynchronization();
int ComputeDelays(const Measurements& audio,
int current_audio_delay_ms,
int* extra_audio_delay_ms,
const Measurements& video,
int* total_video_delay_target_ms);
private:
ViESyncDelay* channel_delay_;
int audio_channel_id_;
int video_channel_id_;
};
} // namespace webrtc
#endif // WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_

436
src/video_engine/stream_synchronization_unittest.cc Normal file
View File

@ -0,0 +1,436 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <math.h>
#include <algorithm>
#include "gtest/gtest.h"
#include "video_engine/stream_synchronization.h"
namespace webrtc {
// These correspond to the same constants defined in vie_sync_module.cc.
enum { kMaxVideoDiffMs = 80 };
enum { kMaxAudioDiffMs = 80 };
enum { kMaxDelay = 1500 };
class Time {
public:
explicit Time(int64_t offset)
: kNtpJan1970(2208988800UL),
time_now_ms_(offset) {}
void NowNtp(uint32_t* ntp_secs, uint32_t* ntp_frac) const {
*ntp_secs = time_now_ms_ / 1000 + kNtpJan1970;
int64_t remainder = time_now_ms_ % 1000;
*ntp_frac = static_cast<uint32_t>(
static_cast<double>(remainder) / 1000.0 * pow(2.0, 32.0) + 0.5);
}
void IncreaseTimeMs(int64_t inc) {
time_now_ms_ += inc;
}
int64_t time_now_ms() const {
return time_now_ms_;
}
private:
// January 1970, in NTP seconds.
const uint32_t kNtpJan1970;
int64_t time_now_ms_;
};
class StreamSynchronizationTest : public ::testing::Test {
protected:
virtual void SetUp() {
sync_ = new StreamSynchronization(0, 0);
send_time_ = new Time(kSendTimeOffsetMs);
receive_time_ = new Time(kReceiveTimeOffsetMs);
}
virtual void TearDown() {
delete sync_;
delete send_time_;
delete receive_time_;
}
int DelayedAudio(int delay_ms,
int current_audio_delay_ms,
int* extra_audio_delay_ms,
int* total_video_delay_ms) {
StreamSynchronization::Measurements audio;
StreamSynchronization::Measurements video;
send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac);
send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac);
receive_time_->NowNtp(&video.rtcp_arrivaltime_secs,
&video.rtcp_arrivaltime_frac);
// Audio later than video.
receive_time_->IncreaseTimeMs(delay_ms);
receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs,
&audio.rtcp_arrivaltime_frac);
return sync_->ComputeDelays(audio,
current_audio_delay_ms,
extra_audio_delay_ms,
video,
total_video_delay_ms);
}
int DelayedVideo(int delay_ms,
int current_audio_delay_ms,
int* extra_audio_delay_ms,
int* total_video_delay_ms) {
StreamSynchronization::Measurements audio;
StreamSynchronization::Measurements video;
send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac);
send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac);
receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs,
&audio.rtcp_arrivaltime_frac);
// Video later than audio.
receive_time_->IncreaseTimeMs(delay_ms);
receive_time_->NowNtp(&video.rtcp_arrivaltime_secs,
&video.rtcp_arrivaltime_frac);
return sync_->ComputeDelays(audio,
current_audio_delay_ms,
extra_audio_delay_ms,
video,
total_video_delay_ms);
}
int DelayedAudioAndVideo(int audio_delay_ms,
int video_delay_ms,
int current_audio_delay_ms,
int* extra_audio_delay_ms,
int* total_video_delay_ms) {
StreamSynchronization::Measurements audio;
StreamSynchronization::Measurements video;
send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac);
send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac);
if (audio_delay_ms > video_delay_ms) {
// Audio later than video.
receive_time_->IncreaseTimeMs(video_delay_ms);
receive_time_->NowNtp(&video.rtcp_arrivaltime_secs,
&video.rtcp_arrivaltime_frac);
receive_time_->IncreaseTimeMs(audio_delay_ms - video_delay_ms);
receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs,
&audio.rtcp_arrivaltime_frac);
} else {
// Video later than audio.
receive_time_->IncreaseTimeMs(audio_delay_ms);
receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs,
&audio.rtcp_arrivaltime_frac);
receive_time_->IncreaseTimeMs(video_delay_ms - audio_delay_ms);
receive_time_->NowNtp(&video.rtcp_arrivaltime_secs,
&video.rtcp_arrivaltime_frac);
}
return sync_->ComputeDelays(audio,
current_audio_delay_ms,
extra_audio_delay_ms,
video,
total_video_delay_ms);
}
int MaxAudioDelayIncrease(int current_audio_delay_ms, int delay_ms) {
return std::min((delay_ms - current_audio_delay_ms) / 2,
static_cast<int>(kMaxAudioDiffMs));
}
int MaxAudioDelayDecrease(int current_audio_delay_ms, int delay_ms) {
return std::max((delay_ms - current_audio_delay_ms) / 2, -kMaxAudioDiffMs);
}
enum { kSendTimeOffsetMs = 0 };
enum { kReceiveTimeOffsetMs = 123456 };
StreamSynchronization* sync_;
Time* send_time_;
Time* receive_time_;
};
TEST_F(StreamSynchronizationTest, NoDelay) {
uint32_t current_audio_delay_ms = 0;
int delay_ms = 0;
int extra_audio_delay_ms = 0;
int total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
EXPECT_EQ(0, total_video_delay_ms);
}
TEST_F(StreamSynchronizationTest, VideoDelay) {
uint32_t current_audio_delay_ms = 0;
int delay_ms = 200;
int extra_audio_delay_ms = 0;
int total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
// The video delay is not allowed to change more than this in 1 second.
EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms);
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
// Simulate 0 minimum delay in the VCM.
total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
// The video delay is not allowed to change more than this in 1 second.
EXPECT_EQ(2*kMaxVideoDiffMs, total_video_delay_ms);
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
// Simulate 0 minimum delay in the VCM.
total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
// The video delay is not allowed to change more than this in 1 second.
EXPECT_EQ(delay_ms, total_video_delay_ms);
}
TEST_F(StreamSynchronizationTest, AudioDelay) {
int current_audio_delay_ms = 0;
int delay_ms = 200;
int extra_audio_delay_ms = 0;
int current_extra_delay_ms = 0;
int total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than this in 1 second.
EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
current_extra_delay_ms = extra_audio_delay_ms;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than the half of the required
// change in delay.
EXPECT_EQ(current_extra_delay_ms +
MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
current_extra_delay_ms = extra_audio_delay_ms;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than the half of the required
// change in delay.
EXPECT_EQ(current_extra_delay_ms +
MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
extra_audio_delay_ms);
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason reduced the delay.
current_audio_delay_ms = 170;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// Since we only can ask NetEQ for a certain amount of extra delay, and
// we only measure the total NetEQ delay, we will ask for additional delay
// here to try to
EXPECT_EQ(current_extra_delay_ms +
MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms),
extra_audio_delay_ms);
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason significantly increased the delay.
current_audio_delay_ms = 250;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than the half of the required
// change in delay.
EXPECT_EQ(current_extra_delay_ms +
MaxAudioDelayDecrease(current_audio_delay_ms, delay_ms),
extra_audio_delay_ms);
}
TEST_F(StreamSynchronizationTest, BothDelayedVideoLater) {
int current_audio_delay_ms = 0;
int audio_delay_ms = 100;
int video_delay_ms = 300;
int extra_audio_delay_ms = 0;
int current_extra_delay_ms = 0;
int total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than this in 1 second.
EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
current_extra_delay_ms = extra_audio_delay_ms;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than the half of the required
// change in delay.
EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease(
current_audio_delay_ms, video_delay_ms - audio_delay_ms),
extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
current_extra_delay_ms = extra_audio_delay_ms;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than the half of the required
// change in delay.
EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease(
current_audio_delay_ms, video_delay_ms - audio_delay_ms),
extra_audio_delay_ms);
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason reduced the delay.
current_audio_delay_ms = 170;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// Since we only can ask NetEQ for a certain amount of extra delay, and
// we only measure the total NetEQ delay, we will ask for additional delay
// here to try to stay in sync.
EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease(
current_audio_delay_ms, video_delay_ms - audio_delay_ms),
extra_audio_delay_ms);
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason significantly increased the delay.
current_audio_delay_ms = 250;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than the half of the required
// change in delay.
EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease(
current_audio_delay_ms, video_delay_ms - audio_delay_ms),
extra_audio_delay_ms);
}
TEST_F(StreamSynchronizationTest, BothDelayedAudioLater) {
int current_audio_delay_ms = 0;
int audio_delay_ms = 300;
int video_delay_ms = 100;
int extra_audio_delay_ms = 0;
int current_extra_delay_ms = 0;
int total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms);
EXPECT_EQ(0, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
current_extra_delay_ms = extra_audio_delay_ms;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms,
video_delay_ms));
// Simulate 0 minimum delay in the VCM.
total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(2 * kMaxVideoDiffMs, total_video_delay_ms);
EXPECT_EQ(0, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
current_extra_delay_ms = extra_audio_delay_ms;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms,
video_delay_ms));
// Simulate 0 minimum delay in the VCM.
total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(audio_delay_ms - video_delay_ms, total_video_delay_ms);
EXPECT_EQ(0, extra_audio_delay_ms);
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ introduces some audio delay.
current_audio_delay_ms = 50;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms,
video_delay_ms));
// Simulate 0 minimum delay in the VCM.
total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms,
total_video_delay_ms);
EXPECT_EQ(0, extra_audio_delay_ms);
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ reduces its delay.
current_audio_delay_ms = 10;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms,
video_delay_ms));
// Simulate 0 minimum delay in the VCM.
total_video_delay_ms = 0;
EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms,
total_video_delay_ms);
EXPECT_EQ(0, extra_audio_delay_ms);
}
} // namespace webrtc

3
src/video_engine/video_engine_core.gypi
View File

@ -62,6 +62,7 @@
'include/vie_rtp_rtcp.h',
# headers
'stream_synchronization.h',
'vie_base_impl.h',
'vie_capture_impl.h',
'vie_codec_impl.h',
@ -96,6 +97,7 @@
'vie_sync_module.h',
# ViE
'stream_synchronization.cc',
'vie_base_impl.cc',
'vie_capture_impl.cc',
'vie_codec_impl.cc',
@ -148,6 +150,7 @@
'../modules/rtp_rtcp/interface',
],
'sources': [
'stream_synchronization_unittest.cc',
'vie_remb_unittest.cc',
],
},

221
src/video_engine/vie_sync_module.cc
View File

@ -15,15 +15,11 @@
#include "trace.h"
#include "video_coding.h"
#include "voe_video_sync.h"
#include "video_engine/stream_synchronization.h"
namespace webrtc {
enum { kSyncInterval = 1000};
enum { kMaxVideoDiffMs = 80 };
enum { kMaxAudioDiffMs = 80 };
enum { kMaxDelay = 1500 };
const float FracMS = 4.294967296E6f;
ViESyncModule::ViESyncModule(const int32_t channel_id, VideoCodingModule* vcm)
: data_cs_(CriticalSectionWrapper::CreateCriticalSection()),
@ -32,7 +28,8 @@ ViESyncModule::ViESyncModule(const int32_t channel_id, VideoCodingModule* vcm)
video_rtcp_module_(NULL),
voe_channel_id_(-1),
voe_sync_interface_(NULL),
last_sync_time_(TickTime::Now()) {
last_sync_time_(TickTime::Now()),
sync_() {
}
ViESyncModule::~ViESyncModule() {
@ -45,6 +42,7 @@ int ViESyncModule::ConfigureSync(int voe_channel_id,
voe_channel_id_ = voe_channel_id;
voe_sync_interface_ = voe_sync_interface;
video_rtcp_module_ = video_rtcp_module;
sync_.reset(new StreamSynchronization(voe_channel_id, channel_id_));
if (!voe_sync_interface) {
voe_channel_id_ = -1;
@ -62,7 +60,7 @@ int ViESyncModule::VoiceChannel() {
}
WebRtc_Word32 ViESyncModule::TimeUntilNextProcess() {
return (WebRtc_Word32)(kSyncInterval -
return static_cast<WebRtc_Word32>(kSyncInterval -
(TickTime::Now() - last_sync_time_).Milliseconds());
}
@ -79,6 +77,7 @@ WebRtc_Word32 ViESyncModule::Process() {
return 0;
}
assert(video_rtcp_module_ && voe_sync_interface_);
assert(sync_.get());
int current_audio_delay_ms = 0;
if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_,
@ -90,9 +89,6 @@ WebRtc_Word32 ViESyncModule::Process() {
return 0;
}
int current_diff_ms = 0;
// Total video delay.
int video_delay_ms = 0;
// VoiceEngine report delay estimates even when not started, ignore if the
// reported value is lower than 40 ms.
if (current_audio_delay_ms < 40) {
@ -108,211 +104,34 @@ WebRtc_Word32 ViESyncModule::Process() {
}
assert(voice_rtcp_module);
uint32_t video_received_ntp_secs = 0;
uint32_t video_received_ntp_frac = 0;
uint32_t video_rtcp_arrivaltime_secs = 0;
uint32_t video_rtcp_arrivaltime_frac = 0;
if (0 != video_rtcp_module_->RemoteNTP(&video_received_ntp_secs,
&video_received_ntp_frac,
&video_rtcp_arrivaltime_secs,
&video_rtcp_arrivaltime_frac)) {
StreamSynchronization::Measurements video;
if (0 != video_rtcp_module_->RemoteNTP(&video.received_ntp_secs,
&video.received_ntp_frac,
&video.rtcp_arrivaltime_secs,
&video.rtcp_arrivaltime_frac)) {
// Failed to get video NTP.
return 0;
}
uint32_t audio_received_ntp_secs = 0;
uint32_t audio_received_ntp_frac = 0;
uint32_t audio_rtcp_arrivaltime_secs = 0;
uint32_t audio_rtcp_arrivaltime_frac = 0;
if (0 != voice_rtcp_module->RemoteNTP(&audio_received_ntp_secs,
&audio_received_ntp_frac,
&audio_rtcp_arrivaltime_secs,
&audio_rtcp_arrivaltime_frac)) {
StreamSynchronization::Measurements audio;
if (0 != voice_rtcp_module->RemoteNTP(&audio.received_ntp_secs,
&audio.received_ntp_frac,
&audio.rtcp_arrivaltime_secs,
&audio.rtcp_arrivaltime_frac)) {
// Failed to get audio NTP.
return 0;
}
// ReceivedNTPxxx is NTP at sender side when sent.
// RTCPArrivalTimexxx is NTP at receiver side when received.
// can't use ConvertNTPTimeToMS since calculation can be
// negative
int NTPdiff = (audio_received_ntp_secs - video_received_ntp_secs)
* 1000; // ms
NTPdiff += static_cast<int>(audio_received_ntp_frac / FracMS -
video_received_ntp_frac / FracMS);
int RTCPdiff = (audio_rtcp_arrivaltime_secs - video_rtcp_arrivaltime_secs)
* 1000; // ms
RTCPdiff += static_cast<int>(audio_rtcp_arrivaltime_frac / FracMS -
video_rtcp_arrivaltime_frac / FracMS);
int diff = NTPdiff - RTCPdiff;
// if diff is + video is behind
if (diff < -1000 || diff > 1000) {
// unresonable ignore value.
int extra_audio_delay_ms = 0;
if (sync_->ComputeDelays(audio, current_audio_delay_ms, &extra_audio_delay_ms,
video, &total_video_delay_target_ms) != 0) {
return 0;
}
channel_delay_.network_delay = diff;
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
"Audio delay is: %d for voice channel: %d",
current_audio_delay_ms, voe_channel_id_);
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
"Network delay diff is: %d for voice channel: %d",
channel_delay_.network_delay, voe_channel_id_);
// Calculate the difference between the lowest possible video delay and
// the current audio delay.
current_diff_ms = total_video_delay_target_ms - current_audio_delay_ms +
channel_delay_.network_delay;
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
"Current diff is: %d for audio channel: %d",
current_diff_ms, voe_channel_id_);
if (current_diff_ms > 0) {
// The minimum video delay is longer than the current audio delay.
// We need to decrease extra video delay, if we have added extra delay
// earlier, or add extra audio delay.
if (channel_delay_.extra_video_delay_ms > 0) {
// We have extra delay added to ViE. Reduce this delay before adding
// extra delay to VoE.
// This is the desired delay, we can't reduce more than this.
video_delay_ms = total_video_delay_target_ms;
// Check that we don't reduce the delay more than what is allowed.
if (video_delay_ms <
channel_delay_.last_video_delay_ms - kMaxVideoDiffMs) {
video_delay_ms =
channel_delay_.last_video_delay_ms - kMaxVideoDiffMs;
channel_delay_.extra_video_delay_ms =
video_delay_ms - total_video_delay_target_ms;
} else {
channel_delay_.extra_video_delay_ms = 0;
}
channel_delay_.last_video_delay_ms = video_delay_ms;
channel_delay_.last_sync_delay = -1;
channel_delay_.extra_audio_delay_ms = 0;
} else { // channel_delay_.extra_video_delay_ms > 0
// We have no extra video delay to remove, increase the audio delay.
if (channel_delay_.last_sync_delay >= 0) {
// We have increased the audio delay earlier, increase it even more.
int audio_diff_ms = current_diff_ms / 2;
if (audio_diff_ms > kMaxAudioDiffMs) {
// We only allow a maximum change of KMaxAudioDiffMS for audio
// due to NetEQ maximum changes.
audio_diff_ms = kMaxAudioDiffMs;
}
// Increase the audio delay
channel_delay_.extra_audio_delay_ms += audio_diff_ms;
// Don't set a too high delay.
if (channel_delay_.extra_audio_delay_ms > kMaxDelay) {
channel_delay_.extra_audio_delay_ms = kMaxDelay;
}
// Don't add any extra video delay.
video_delay_ms = total_video_delay_target_ms;
channel_delay_.extra_video_delay_ms = 0;
channel_delay_.last_video_delay_ms = video_delay_ms;
channel_delay_.last_sync_delay = 1;
} else { // channel_delay_.last_sync_delay >= 0
// First time after a delay change, don't add any extra delay.
// This is to not toggle back and forth too much.
channel_delay_.extra_audio_delay_ms = 0;
// Set minimum video delay
video_delay_ms = total_video_delay_target_ms;
channel_delay_.extra_video_delay_ms = 0;
channel_delay_.last_video_delay_ms = video_delay_ms;
channel_delay_.last_sync_delay = 0;
}
}
} else { // if (current_diffMS > 0)
// The minimum video delay is lower than the current audio delay.
// We need to decrease possible extra audio delay, or
// add extra video delay.
if (channel_delay_.extra_audio_delay_ms > 0) {
// We have extra delay in VoiceEngine
// Start with decreasing the voice delay
int audio_diff_ms = current_diff_ms / 2;
if (audio_diff_ms < -1 * kMaxAudioDiffMs) {
// Don't change the delay too much at once.
audio_diff_ms = -1 * kMaxAudioDiffMs;
}
// Add the negative difference.
channel_delay_.extra_audio_delay_ms += audio_diff_ms;
if (channel_delay_.extra_audio_delay_ms < 0) {
// Negative values not allowed.
channel_delay_.extra_audio_delay_ms = 0;
channel_delay_.last_sync_delay = 0;
} else {
// There is more audio delay to use for the next round.
channel_delay_.last_sync_delay = 1;
}
// Keep the video delay at the minimum values.
video_delay_ms = total_video_delay_target_ms;
channel_delay_.extra_video_delay_ms = 0;
channel_delay_.last_video_delay_ms = video_delay_ms;
} else { // channel_delay_.extra_audio_delay_ms > 0
// We have no extra delay in VoiceEngine, increase the video delay.
channel_delay_.extra_audio_delay_ms = 0;
// Make the difference positive.
int video_diff_ms = -1 * current_diff_ms;
// This is the desired delay.
video_delay_ms = total_video_delay_target_ms + video_diff_ms;
if (video_delay_ms > channel_delay_.last_video_delay_ms) {
if (video_delay_ms >
channel_delay_.last_video_delay_ms + kMaxVideoDiffMs) {
// Don't increase the delay too much at once
video_delay_ms =
channel_delay_.last_video_delay_ms + kMaxVideoDiffMs;
}
// Verify we don't go above the maximum allowed delay
if (video_delay_ms > kMaxDelay) {
video_delay_ms = kMaxDelay;
}
} else {
if (video_delay_ms <
channel_delay_.last_video_delay_ms - kMaxVideoDiffMs) {
// Don't decrease the delay too much at once
video_delay_ms =
channel_delay_.last_video_delay_ms - kMaxVideoDiffMs;
}
// Verify we don't go below the minimum delay
if (video_delay_ms < total_video_delay_target_ms) {
video_delay_ms = total_video_delay_target_ms;
}
}
// Store the values
channel_delay_.extra_video_delay_ms =
video_delay_ms - total_video_delay_target_ms;
channel_delay_.last_video_delay_ms = video_delay_ms;
channel_delay_.last_sync_delay = -1;
}
}
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
"Sync video delay %d ms for video channel and audio delay %d for audio "
"channel %d",
video_delay_ms, channel_delay_.extra_audio_delay_ms, voe_channel_id_);
// Set the extra audio delay.synchronization
if (voe_sync_interface_->SetMinimumPlayoutDelay(
voe_channel_id_, channel_delay_.extra_audio_delay_ms) == -1) {
voe_channel_id_, extra_audio_delay_ms) == -1) {
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideo, channel_id_,
"Error setting voice delay");
}
if (video_delay_ms < 0) {
video_delay_ms = 0;
}
total_video_delay_target_ms =
(total_video_delay_target_ms > video_delay_ms) ?
total_video_delay_target_ms : video_delay_ms;
vcm_->SetMinimumPlayoutDelay(total_video_delay_target_ms);
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_,
"New Video delay target is: %d", total_video_delay_target_ms);

22
src/video_engine/vie_sync_module.h
View File

@ -14,14 +14,15 @@
#ifndef WEBRTC_VIDEO_ENGINE_VIE_SYNC_MODULE_H_
#define WEBRTC_VIDEO_ENGINE_VIE_SYNC_MODULE_H_
#include "module.h"
#include "modules/interface/module.h"
#include "system_wrappers/interface/scoped_ptr.h"
#include "tick_util.h"
#include "system_wrappers/interface/tick_util.h"
namespace webrtc {
class CriticalSectionWrapper;
class RtpRtcp;
class StreamSynchronization;
class VideoCodingModule;
class VoEVideoSync;
@ -48,22 +49,7 @@ class ViESyncModule : public Module {
int voe_channel_id_;
VoEVideoSync* voe_sync_interface_;
TickTime last_sync_time_;
struct ViESyncDelay {
ViESyncDelay() {
extra_video_delay_ms = 0;
last_video_delay_ms = 0;
extra_audio_delay_ms = 0;
last_sync_delay = 0;
network_delay = 120;
}
int extra_video_delay_ms;
int last_video_delay_ms;
int extra_audio_delay_ms;
int last_sync_delay;
int network_delay;
};
ViESyncDelay channel_delay_;
scoped_ptr<StreamSynchronization> sync_;
};
} // namespace webrtc