From 5f284981492db810026be90a3cd01a9346fe109c Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Thu, 28 Jun 2012 07:51:16 +0000 Subject: [PATCH] First step in refactoring audio/video synchronization. Adds unittests. BUG= TEST=stream_synchronization_unittest Review URL: https://webrtc-codereview.appspot.com/669005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@2455 4adac7df-926f-26a2-2b94-8c16560cd09d --- src/video_engine/stream_synchronization.cc | 240 ++++++++++ src/video_engine/stream_synchronization.h | 51 ++ .../stream_synchronization_unittest.cc | 436 ++++++++++++++++++ src/video_engine/video_engine_core.gypi | 3 + src/video_engine/vie_sync_module.cc | 221 +-------- src/video_engine/vie_sync_module.h | 22 +- 6 files changed, 754 insertions(+), 219 deletions(-) create mode 100644 src/video_engine/stream_synchronization.cc create mode 100644 src/video_engine/stream_synchronization.h create mode 100644 src/video_engine/stream_synchronization_unittest.cc diff --git a/src/video_engine/stream_synchronization.cc b/src/video_engine/stream_synchronization.cc new file mode 100644 index 0000000000..1ba1f09c1b --- /dev/null +++ b/src/video_engine/stream_synchronization.cc @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video_engine/stream_synchronization.h" +#include "system_wrappers/interface/trace.h" + +namespace webrtc { + +enum { kMaxVideoDiffMs = 80 }; +enum { kMaxAudioDiffMs = 80 }; +enum { kMaxDelay = 1500 }; + +const float FracMS = 4.294967296E6f; + +struct ViESyncDelay { + ViESyncDelay() { + extra_video_delay_ms = 0; + last_video_delay_ms = 0; + extra_audio_delay_ms = 0; + last_sync_delay = 0; + network_delay = 120; + } + + int extra_video_delay_ms; + int last_video_delay_ms; + int extra_audio_delay_ms; + int last_sync_delay; + int network_delay; +}; + +StreamSynchronization::StreamSynchronization(int audio_channel_id, + int video_channel_id) + : channel_delay_(new ViESyncDelay), + audio_channel_id_(audio_channel_id), + video_channel_id_(video_channel_id) {} + +StreamSynchronization::~StreamSynchronization() { + delete channel_delay_; +} + +int StreamSynchronization::ComputeDelays(const Measurements& audio, + int current_audio_delay_ms, + int* extra_audio_delay_ms, + const Measurements& video, + int* total_video_delay_target_ms) { + // ReceivedNTPxxx is NTP at sender side when sent. + // RTCPArrivalTimexxx is NTP at receiver side when received. + // can't use ConvertNTPTimeToMS since calculation can be + // negative + int NTPdiff = (audio.received_ntp_secs - video.received_ntp_secs) + * 1000; // ms + float ntp_diff_frac = audio.received_ntp_frac / FracMS - + video.received_ntp_frac / FracMS; + if (ntp_diff_frac > 0.0f) + NTPdiff += static_cast(ntp_diff_frac + 0.5f); + else + NTPdiff += static_cast(ntp_diff_frac - 0.5f); + + int RTCPdiff = (audio.rtcp_arrivaltime_secs - video.rtcp_arrivaltime_secs) + * 1000; // ms + float rtcp_diff_frac = audio.rtcp_arrivaltime_frac / FracMS - + video.rtcp_arrivaltime_frac / FracMS; + if (rtcp_diff_frac > 0.0f) + RTCPdiff += static_cast(rtcp_diff_frac + 0.5f); + else + RTCPdiff += static_cast(rtcp_diff_frac - 0.5f); + + int diff = NTPdiff - RTCPdiff; + // if diff is + video is behind + if (diff < -1000 || diff > 1000) { + // unresonable ignore value. + return -1; + } + channel_delay_->network_delay = diff; + + WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, + "Audio delay is: %d for voice channel: %d", + current_audio_delay_ms, audio_channel_id_); + WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, + "Network delay diff is: %d for voice channel: %d", + channel_delay_->network_delay, audio_channel_id_); + // Calculate the difference between the lowest possible video delay and + // the current audio delay. + int current_diff_ms = *total_video_delay_target_ms - current_audio_delay_ms + + channel_delay_->network_delay; + WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, + "Current diff is: %d for audio channel: %d", + current_diff_ms, audio_channel_id_); + + int video_delay_ms = 0; + if (current_diff_ms > 0) { + // The minimum video delay is longer than the current audio delay. + // We need to decrease extra video delay, if we have added extra delay + // earlier, or add extra audio delay. + if (channel_delay_->extra_video_delay_ms > 0) { + // We have extra delay added to ViE. Reduce this delay before adding + // extra delay to VoE. + + // This is the desired delay, we can't reduce more than this. + video_delay_ms = *total_video_delay_target_ms; + + // Check that we don't reduce the delay more than what is allowed. + if (video_delay_ms < + channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) { + video_delay_ms = + channel_delay_->last_video_delay_ms - kMaxVideoDiffMs; + channel_delay_->extra_video_delay_ms = + video_delay_ms - *total_video_delay_target_ms; + } else { + channel_delay_->extra_video_delay_ms = 0; + } + channel_delay_->last_video_delay_ms = video_delay_ms; + channel_delay_->last_sync_delay = -1; + channel_delay_->extra_audio_delay_ms = 0; + } else { // channel_delay_->extra_video_delay_ms > 0 + // We have no extra video delay to remove, increase the audio delay. + if (channel_delay_->last_sync_delay >= 0) { + // We have increased the audio delay earlier, increase it even more. + int audio_diff_ms = current_diff_ms / 2; + if (audio_diff_ms > kMaxAudioDiffMs) { + // We only allow a maximum change of KMaxAudioDiffMS for audio + // due to NetEQ maximum changes. + audio_diff_ms = kMaxAudioDiffMs; + } + // Increase the audio delay + channel_delay_->extra_audio_delay_ms += audio_diff_ms; + + // Don't set a too high delay. + if (channel_delay_->extra_audio_delay_ms > kMaxDelay) { + channel_delay_->extra_audio_delay_ms = kMaxDelay; + } + + // Don't add any extra video delay. + video_delay_ms = *total_video_delay_target_ms; + channel_delay_->extra_video_delay_ms = 0; + channel_delay_->last_video_delay_ms = video_delay_ms; + channel_delay_->last_sync_delay = 1; + } else { // channel_delay_->last_sync_delay >= 0 + // First time after a delay change, don't add any extra delay. + // This is to not toggle back and forth too much. + channel_delay_->extra_audio_delay_ms = 0; + // Set minimum video delay + video_delay_ms = *total_video_delay_target_ms; + channel_delay_->extra_video_delay_ms = 0; + channel_delay_->last_video_delay_ms = video_delay_ms; + channel_delay_->last_sync_delay = 0; + } + } + } else { // if (current_diffMS > 0) + // The minimum video delay is lower than the current audio delay. + // We need to decrease possible extra audio delay, or + // add extra video delay. + + if (channel_delay_->extra_audio_delay_ms > 0) { + // We have extra delay in VoiceEngine + // Start with decreasing the voice delay + int audio_diff_ms = current_diff_ms / 2; + if (audio_diff_ms < -1 * kMaxAudioDiffMs) { + // Don't change the delay too much at once. + audio_diff_ms = -1 * kMaxAudioDiffMs; + } + // Add the negative difference. + channel_delay_->extra_audio_delay_ms += audio_diff_ms; + + if (channel_delay_->extra_audio_delay_ms < 0) { + // Negative values not allowed. + channel_delay_->extra_audio_delay_ms = 0; + channel_delay_->last_sync_delay = 0; + } else { + // There is more audio delay to use for the next round. + channel_delay_->last_sync_delay = 1; + } + + // Keep the video delay at the minimum values. + video_delay_ms = *total_video_delay_target_ms; + channel_delay_->extra_video_delay_ms = 0; + channel_delay_->last_video_delay_ms = video_delay_ms; + } else { // channel_delay_->extra_audio_delay_ms > 0 + // We have no extra delay in VoiceEngine, increase the video delay. + channel_delay_->extra_audio_delay_ms = 0; + + // Make the difference positive. + int video_diff_ms = -1 * current_diff_ms; + + // This is the desired delay. + video_delay_ms = *total_video_delay_target_ms + video_diff_ms; + if (video_delay_ms > channel_delay_->last_video_delay_ms) { + if (video_delay_ms > + channel_delay_->last_video_delay_ms + kMaxVideoDiffMs) { + // Don't increase the delay too much at once + video_delay_ms = + channel_delay_->last_video_delay_ms + kMaxVideoDiffMs; + } + // Verify we don't go above the maximum allowed delay + if (video_delay_ms > kMaxDelay) { + video_delay_ms = kMaxDelay; + } + } else { + if (video_delay_ms < + channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) { + // Don't decrease the delay too much at once + video_delay_ms = + channel_delay_->last_video_delay_ms - kMaxVideoDiffMs; + } + // Verify we don't go below the minimum delay + if (video_delay_ms < *total_video_delay_target_ms) { + video_delay_ms = *total_video_delay_target_ms; + } + } + // Store the values + channel_delay_->extra_video_delay_ms = + video_delay_ms - *total_video_delay_target_ms; + channel_delay_->last_video_delay_ms = video_delay_ms; + channel_delay_->last_sync_delay = -1; + } + } + + WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, + "Sync video delay %d ms for video channel and audio delay %d for audio " + "channel %d", + video_delay_ms, channel_delay_->extra_audio_delay_ms, audio_channel_id_); + + *extra_audio_delay_ms = channel_delay_->extra_audio_delay_ms; + + if (video_delay_ms < 0) { + video_delay_ms = 0; + } + *total_video_delay_target_ms = + (*total_video_delay_target_ms > video_delay_ms) ? + *total_video_delay_target_ms : video_delay_ms; + return 0; +} +} // namespace webrtc diff --git a/src/video_engine/stream_synchronization.h b/src/video_engine/stream_synchronization.h new file mode 100644 index 0000000000..5a88383ab1 --- /dev/null +++ b/src/video_engine/stream_synchronization.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_ +#define WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_ + +#include "typedefs.h" + +namespace webrtc { + +struct ViESyncDelay; + +class StreamSynchronization { + public: + struct Measurements { + Measurements() + : received_ntp_secs(0), + received_ntp_frac(0), + rtcp_arrivaltime_secs(0), + rtcp_arrivaltime_frac(0) {} + uint32_t received_ntp_secs; + uint32_t received_ntp_frac; + uint32_t rtcp_arrivaltime_secs; + uint32_t rtcp_arrivaltime_frac; + }; + + StreamSynchronization(int audio_channel_id, int video_channel_id); + ~StreamSynchronization(); + + int ComputeDelays(const Measurements& audio, + int current_audio_delay_ms, + int* extra_audio_delay_ms, + const Measurements& video, + int* total_video_delay_target_ms); + + private: + ViESyncDelay* channel_delay_; + int audio_channel_id_; + int video_channel_id_; +}; + +} // namespace webrtc + +#endif // WEBRTC_VIDEO_ENGINE_STREAM_SYNCHRONIZATION_H_ diff --git a/src/video_engine/stream_synchronization_unittest.cc b/src/video_engine/stream_synchronization_unittest.cc new file mode 100644 index 0000000000..d4b002c35e --- /dev/null +++ b/src/video_engine/stream_synchronization_unittest.cc @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "gtest/gtest.h" +#include "video_engine/stream_synchronization.h" + +namespace webrtc { + +// These correspond to the same constants defined in vie_sync_module.cc. +enum { kMaxVideoDiffMs = 80 }; +enum { kMaxAudioDiffMs = 80 }; +enum { kMaxDelay = 1500 }; + +class Time { + public: + explicit Time(int64_t offset) + : kNtpJan1970(2208988800UL), + time_now_ms_(offset) {} + + void NowNtp(uint32_t* ntp_secs, uint32_t* ntp_frac) const { + *ntp_secs = time_now_ms_ / 1000 + kNtpJan1970; + int64_t remainder = time_now_ms_ % 1000; + *ntp_frac = static_cast( + static_cast(remainder) / 1000.0 * pow(2.0, 32.0) + 0.5); + } + + void IncreaseTimeMs(int64_t inc) { + time_now_ms_ += inc; + } + + int64_t time_now_ms() const { + return time_now_ms_; + } + private: + // January 1970, in NTP seconds. + const uint32_t kNtpJan1970; + int64_t time_now_ms_; +}; + +class StreamSynchronizationTest : public ::testing::Test { + protected: + virtual void SetUp() { + sync_ = new StreamSynchronization(0, 0); + send_time_ = new Time(kSendTimeOffsetMs); + receive_time_ = new Time(kReceiveTimeOffsetMs); + } + + virtual void TearDown() { + delete sync_; + delete send_time_; + delete receive_time_; + } + + int DelayedAudio(int delay_ms, + int current_audio_delay_ms, + int* extra_audio_delay_ms, + int* total_video_delay_ms) { + StreamSynchronization::Measurements audio; + StreamSynchronization::Measurements video; + send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac); + send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac); + receive_time_->NowNtp(&video.rtcp_arrivaltime_secs, + &video.rtcp_arrivaltime_frac); + // Audio later than video. + receive_time_->IncreaseTimeMs(delay_ms); + receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs, + &audio.rtcp_arrivaltime_frac); + return sync_->ComputeDelays(audio, + current_audio_delay_ms, + extra_audio_delay_ms, + video, + total_video_delay_ms); + } + + int DelayedVideo(int delay_ms, + int current_audio_delay_ms, + int* extra_audio_delay_ms, + int* total_video_delay_ms) { + StreamSynchronization::Measurements audio; + StreamSynchronization::Measurements video; + send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac); + send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac); + receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs, + &audio.rtcp_arrivaltime_frac); + // Video later than audio. + receive_time_->IncreaseTimeMs(delay_ms); + receive_time_->NowNtp(&video.rtcp_arrivaltime_secs, + &video.rtcp_arrivaltime_frac); + return sync_->ComputeDelays(audio, + current_audio_delay_ms, + extra_audio_delay_ms, + video, + total_video_delay_ms); + } + + int DelayedAudioAndVideo(int audio_delay_ms, + int video_delay_ms, + int current_audio_delay_ms, + int* extra_audio_delay_ms, + int* total_video_delay_ms) { + StreamSynchronization::Measurements audio; + StreamSynchronization::Measurements video; + send_time_->NowNtp(&audio.received_ntp_secs, &audio.received_ntp_frac); + send_time_->NowNtp(&video.received_ntp_secs, &video.received_ntp_frac); + + if (audio_delay_ms > video_delay_ms) { + // Audio later than video. + receive_time_->IncreaseTimeMs(video_delay_ms); + receive_time_->NowNtp(&video.rtcp_arrivaltime_secs, + &video.rtcp_arrivaltime_frac); + receive_time_->IncreaseTimeMs(audio_delay_ms - video_delay_ms); + receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs, + &audio.rtcp_arrivaltime_frac); + } else { + // Video later than audio. + receive_time_->IncreaseTimeMs(audio_delay_ms); + receive_time_->NowNtp(&audio.rtcp_arrivaltime_secs, + &audio.rtcp_arrivaltime_frac); + receive_time_->IncreaseTimeMs(video_delay_ms - audio_delay_ms); + receive_time_->NowNtp(&video.rtcp_arrivaltime_secs, + &video.rtcp_arrivaltime_frac); + } + return sync_->ComputeDelays(audio, + current_audio_delay_ms, + extra_audio_delay_ms, + video, + total_video_delay_ms); + } + + int MaxAudioDelayIncrease(int current_audio_delay_ms, int delay_ms) { + return std::min((delay_ms - current_audio_delay_ms) / 2, + static_cast(kMaxAudioDiffMs)); + } + + int MaxAudioDelayDecrease(int current_audio_delay_ms, int delay_ms) { + return std::max((delay_ms - current_audio_delay_ms) / 2, -kMaxAudioDiffMs); + } + + enum { kSendTimeOffsetMs = 0 }; + enum { kReceiveTimeOffsetMs = 123456 }; + + StreamSynchronization* sync_; + Time* send_time_; + Time* receive_time_; +}; + +TEST_F(StreamSynchronizationTest, NoDelay) { + uint32_t current_audio_delay_ms = 0; + int delay_ms = 0; + int extra_audio_delay_ms = 0; + int total_video_delay_ms = 0; + + EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, extra_audio_delay_ms); + EXPECT_EQ(0, total_video_delay_ms); +} + +TEST_F(StreamSynchronizationTest, VideoDelay) { + uint32_t current_audio_delay_ms = 0; + int delay_ms = 200; + int extra_audio_delay_ms = 0; + int total_video_delay_ms = 0; + + EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, extra_audio_delay_ms); + // The video delay is not allowed to change more than this in 1 second. + EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms); + + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + // Simulate 0 minimum delay in the VCM. + total_video_delay_ms = 0; + EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, extra_audio_delay_ms); + // The video delay is not allowed to change more than this in 1 second. + EXPECT_EQ(2*kMaxVideoDiffMs, total_video_delay_ms); + + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + // Simulate 0 minimum delay in the VCM. + total_video_delay_ms = 0; + EXPECT_EQ(0, DelayedAudio(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, extra_audio_delay_ms); + // The video delay is not allowed to change more than this in 1 second. + EXPECT_EQ(delay_ms, total_video_delay_ms); +} + +TEST_F(StreamSynchronizationTest, AudioDelay) { + int current_audio_delay_ms = 0; + int delay_ms = 200; + int extra_audio_delay_ms = 0; + int current_extra_delay_ms = 0; + int total_video_delay_ms = 0; + + EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // The audio delay is not allowed to change more than this in 1 second. + EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms); + current_audio_delay_ms = extra_audio_delay_ms; + current_extra_delay_ms = extra_audio_delay_ms; + + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // The audio delay is not allowed to change more than the half of the required + // change in delay. + EXPECT_EQ(current_extra_delay_ms + + MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms), + extra_audio_delay_ms); + current_audio_delay_ms = extra_audio_delay_ms; + current_extra_delay_ms = extra_audio_delay_ms; + + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // The audio delay is not allowed to change more than the half of the required + // change in delay. + EXPECT_EQ(current_extra_delay_ms + + MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms), + extra_audio_delay_ms); + current_extra_delay_ms = extra_audio_delay_ms; + + // Simulate that NetEQ for some reason reduced the delay. + current_audio_delay_ms = 170; + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // Since we only can ask NetEQ for a certain amount of extra delay, and + // we only measure the total NetEQ delay, we will ask for additional delay + // here to try to + EXPECT_EQ(current_extra_delay_ms + + MaxAudioDelayIncrease(current_audio_delay_ms, delay_ms), + extra_audio_delay_ms); + current_extra_delay_ms = extra_audio_delay_ms; + + // Simulate that NetEQ for some reason significantly increased the delay. + current_audio_delay_ms = 250; + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + EXPECT_EQ(0, DelayedVideo(delay_ms, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // The audio delay is not allowed to change more than the half of the required + // change in delay. + EXPECT_EQ(current_extra_delay_ms + + MaxAudioDelayDecrease(current_audio_delay_ms, delay_ms), + extra_audio_delay_ms); +} + +TEST_F(StreamSynchronizationTest, BothDelayedVideoLater) { + int current_audio_delay_ms = 0; + int audio_delay_ms = 100; + int video_delay_ms = 300; + int extra_audio_delay_ms = 0; + int current_extra_delay_ms = 0; + int total_video_delay_ms = 0; + + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // The audio delay is not allowed to change more than this in 1 second. + EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms); + current_audio_delay_ms = extra_audio_delay_ms; + current_extra_delay_ms = extra_audio_delay_ms; + + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // The audio delay is not allowed to change more than the half of the required + // change in delay. + EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease( + current_audio_delay_ms, video_delay_ms - audio_delay_ms), + extra_audio_delay_ms); + current_audio_delay_ms = extra_audio_delay_ms; + current_extra_delay_ms = extra_audio_delay_ms; + + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // The audio delay is not allowed to change more than the half of the required + // change in delay. + EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease( + current_audio_delay_ms, video_delay_ms - audio_delay_ms), + extra_audio_delay_ms); + current_extra_delay_ms = extra_audio_delay_ms; + + // Simulate that NetEQ for some reason reduced the delay. + current_audio_delay_ms = 170; + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // Since we only can ask NetEQ for a certain amount of extra delay, and + // we only measure the total NetEQ delay, we will ask for additional delay + // here to try to stay in sync. + EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease( + current_audio_delay_ms, video_delay_ms - audio_delay_ms), + extra_audio_delay_ms); + current_extra_delay_ms = extra_audio_delay_ms; + + // Simulate that NetEQ for some reason significantly increased the delay. + current_audio_delay_ms = 250; + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(800); + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(0, total_video_delay_ms); + // The audio delay is not allowed to change more than the half of the required + // change in delay. + EXPECT_EQ(current_extra_delay_ms + MaxAudioDelayIncrease( + current_audio_delay_ms, video_delay_ms - audio_delay_ms), + extra_audio_delay_ms); +} + +TEST_F(StreamSynchronizationTest, BothDelayedAudioLater) { + int current_audio_delay_ms = 0; + int audio_delay_ms = 300; + int video_delay_ms = 100; + int extra_audio_delay_ms = 0; + int current_extra_delay_ms = 0; + int total_video_delay_ms = 0; + + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms); + EXPECT_EQ(0, extra_audio_delay_ms); + current_audio_delay_ms = extra_audio_delay_ms; + current_extra_delay_ms = extra_audio_delay_ms; + + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms, + video_delay_ms)); + // Simulate 0 minimum delay in the VCM. + total_video_delay_ms = 0; + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(2 * kMaxVideoDiffMs, total_video_delay_ms); + EXPECT_EQ(0, extra_audio_delay_ms); + current_audio_delay_ms = extra_audio_delay_ms; + current_extra_delay_ms = extra_audio_delay_ms; + + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms, + video_delay_ms)); + // Simulate 0 minimum delay in the VCM. + total_video_delay_ms = 0; + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(audio_delay_ms - video_delay_ms, total_video_delay_ms); + EXPECT_EQ(0, extra_audio_delay_ms); + current_extra_delay_ms = extra_audio_delay_ms; + + // Simulate that NetEQ introduces some audio delay. + current_audio_delay_ms = 50; + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms, + video_delay_ms)); + // Simulate 0 minimum delay in the VCM. + total_video_delay_ms = 0; + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms, + total_video_delay_ms); + EXPECT_EQ(0, extra_audio_delay_ms); + current_extra_delay_ms = extra_audio_delay_ms; + + // Simulate that NetEQ reduces its delay. + current_audio_delay_ms = 10; + send_time_->IncreaseTimeMs(1000); + receive_time_->IncreaseTimeMs(1000 - std::max(audio_delay_ms, + video_delay_ms)); + // Simulate 0 minimum delay in the VCM. + total_video_delay_ms = 0; + EXPECT_EQ(0, DelayedAudioAndVideo(audio_delay_ms, + video_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, + &total_video_delay_ms)); + EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms, + total_video_delay_ms); + EXPECT_EQ(0, extra_audio_delay_ms); +} +} // namespace webrtc diff --git a/src/video_engine/video_engine_core.gypi b/src/video_engine/video_engine_core.gypi index cf4a1ecfcf..10c44fcb30 100644 --- a/src/video_engine/video_engine_core.gypi +++ b/src/video_engine/video_engine_core.gypi @@ -62,6 +62,7 @@ 'include/vie_rtp_rtcp.h', # headers + 'stream_synchronization.h', 'vie_base_impl.h', 'vie_capture_impl.h', 'vie_codec_impl.h', @@ -96,6 +97,7 @@ 'vie_sync_module.h', # ViE + 'stream_synchronization.cc', 'vie_base_impl.cc', 'vie_capture_impl.cc', 'vie_codec_impl.cc', @@ -148,6 +150,7 @@ '../modules/rtp_rtcp/interface', ], 'sources': [ + 'stream_synchronization_unittest.cc', 'vie_remb_unittest.cc', ], }, diff --git a/src/video_engine/vie_sync_module.cc b/src/video_engine/vie_sync_module.cc index 5aa5595b5a..01aa5d22e3 100644 --- a/src/video_engine/vie_sync_module.cc +++ b/src/video_engine/vie_sync_module.cc @@ -15,15 +15,11 @@ #include "trace.h" #include "video_coding.h" #include "voe_video_sync.h" +#include "video_engine/stream_synchronization.h" namespace webrtc { enum { kSyncInterval = 1000}; -enum { kMaxVideoDiffMs = 80 }; -enum { kMaxAudioDiffMs = 80 }; -enum { kMaxDelay = 1500 }; - -const float FracMS = 4.294967296E6f; ViESyncModule::ViESyncModule(const int32_t channel_id, VideoCodingModule* vcm) : data_cs_(CriticalSectionWrapper::CreateCriticalSection()), @@ -32,7 +28,8 @@ ViESyncModule::ViESyncModule(const int32_t channel_id, VideoCodingModule* vcm) video_rtcp_module_(NULL), voe_channel_id_(-1), voe_sync_interface_(NULL), - last_sync_time_(TickTime::Now()) { + last_sync_time_(TickTime::Now()), + sync_() { } ViESyncModule::~ViESyncModule() { @@ -45,6 +42,7 @@ int ViESyncModule::ConfigureSync(int voe_channel_id, voe_channel_id_ = voe_channel_id; voe_sync_interface_ = voe_sync_interface; video_rtcp_module_ = video_rtcp_module; + sync_.reset(new StreamSynchronization(voe_channel_id, channel_id_)); if (!voe_sync_interface) { voe_channel_id_ = -1; @@ -62,7 +60,7 @@ int ViESyncModule::VoiceChannel() { } WebRtc_Word32 ViESyncModule::TimeUntilNextProcess() { - return (WebRtc_Word32)(kSyncInterval - + return static_cast(kSyncInterval - (TickTime::Now() - last_sync_time_).Milliseconds()); } @@ -79,6 +77,7 @@ WebRtc_Word32 ViESyncModule::Process() { return 0; } assert(video_rtcp_module_ && voe_sync_interface_); + assert(sync_.get()); int current_audio_delay_ms = 0; if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_, @@ -90,9 +89,6 @@ WebRtc_Word32 ViESyncModule::Process() { return 0; } - int current_diff_ms = 0; - // Total video delay. - int video_delay_ms = 0; // VoiceEngine report delay estimates even when not started, ignore if the // reported value is lower than 40 ms. if (current_audio_delay_ms < 40) { @@ -108,211 +104,34 @@ WebRtc_Word32 ViESyncModule::Process() { } assert(voice_rtcp_module); - uint32_t video_received_ntp_secs = 0; - uint32_t video_received_ntp_frac = 0; - uint32_t video_rtcp_arrivaltime_secs = 0; - uint32_t video_rtcp_arrivaltime_frac = 0; - - if (0 != video_rtcp_module_->RemoteNTP(&video_received_ntp_secs, - &video_received_ntp_frac, - &video_rtcp_arrivaltime_secs, - &video_rtcp_arrivaltime_frac)) { + StreamSynchronization::Measurements video; + if (0 != video_rtcp_module_->RemoteNTP(&video.received_ntp_secs, + &video.received_ntp_frac, + &video.rtcp_arrivaltime_secs, + &video.rtcp_arrivaltime_frac)) { // Failed to get video NTP. return 0; } - uint32_t audio_received_ntp_secs = 0; - uint32_t audio_received_ntp_frac = 0; - uint32_t audio_rtcp_arrivaltime_secs = 0; - uint32_t audio_rtcp_arrivaltime_frac = 0; - if (0 != voice_rtcp_module->RemoteNTP(&audio_received_ntp_secs, - &audio_received_ntp_frac, - &audio_rtcp_arrivaltime_secs, - &audio_rtcp_arrivaltime_frac)) { + StreamSynchronization::Measurements audio; + if (0 != voice_rtcp_module->RemoteNTP(&audio.received_ntp_secs, + &audio.received_ntp_frac, + &audio.rtcp_arrivaltime_secs, + &audio.rtcp_arrivaltime_frac)) { // Failed to get audio NTP. return 0; } - // ReceivedNTPxxx is NTP at sender side when sent. - // RTCPArrivalTimexxx is NTP at receiver side when received. - // can't use ConvertNTPTimeToMS since calculation can be - // negative - int NTPdiff = (audio_received_ntp_secs - video_received_ntp_secs) - * 1000; // ms - NTPdiff += static_cast(audio_received_ntp_frac / FracMS - - video_received_ntp_frac / FracMS); - - int RTCPdiff = (audio_rtcp_arrivaltime_secs - video_rtcp_arrivaltime_secs) - * 1000; // ms - RTCPdiff += static_cast(audio_rtcp_arrivaltime_frac / FracMS - - video_rtcp_arrivaltime_frac / FracMS); - - int diff = NTPdiff - RTCPdiff; - // if diff is + video is behind - if (diff < -1000 || diff > 1000) { - // unresonable ignore value. + int extra_audio_delay_ms = 0; + if (sync_->ComputeDelays(audio, current_audio_delay_ms, &extra_audio_delay_ms, + video, &total_video_delay_target_ms) != 0) { return 0; } - channel_delay_.network_delay = diff; - - WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_, - "Audio delay is: %d for voice channel: %d", - current_audio_delay_ms, voe_channel_id_); - WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_, - "Network delay diff is: %d for voice channel: %d", - channel_delay_.network_delay, voe_channel_id_); - // Calculate the difference between the lowest possible video delay and - // the current audio delay. - current_diff_ms = total_video_delay_target_ms - current_audio_delay_ms + - channel_delay_.network_delay; - WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_, - "Current diff is: %d for audio channel: %d", - current_diff_ms, voe_channel_id_); - - if (current_diff_ms > 0) { - // The minimum video delay is longer than the current audio delay. - // We need to decrease extra video delay, if we have added extra delay - // earlier, or add extra audio delay. - if (channel_delay_.extra_video_delay_ms > 0) { - // We have extra delay added to ViE. Reduce this delay before adding - // extra delay to VoE. - - // This is the desired delay, we can't reduce more than this. - video_delay_ms = total_video_delay_target_ms; - - // Check that we don't reduce the delay more than what is allowed. - if (video_delay_ms < - channel_delay_.last_video_delay_ms - kMaxVideoDiffMs) { - video_delay_ms = - channel_delay_.last_video_delay_ms - kMaxVideoDiffMs; - channel_delay_.extra_video_delay_ms = - video_delay_ms - total_video_delay_target_ms; - } else { - channel_delay_.extra_video_delay_ms = 0; - } - channel_delay_.last_video_delay_ms = video_delay_ms; - channel_delay_.last_sync_delay = -1; - channel_delay_.extra_audio_delay_ms = 0; - } else { // channel_delay_.extra_video_delay_ms > 0 - // We have no extra video delay to remove, increase the audio delay. - if (channel_delay_.last_sync_delay >= 0) { - // We have increased the audio delay earlier, increase it even more. - int audio_diff_ms = current_diff_ms / 2; - if (audio_diff_ms > kMaxAudioDiffMs) { - // We only allow a maximum change of KMaxAudioDiffMS for audio - // due to NetEQ maximum changes. - audio_diff_ms = kMaxAudioDiffMs; - } - // Increase the audio delay - channel_delay_.extra_audio_delay_ms += audio_diff_ms; - - // Don't set a too high delay. - if (channel_delay_.extra_audio_delay_ms > kMaxDelay) { - channel_delay_.extra_audio_delay_ms = kMaxDelay; - } - - // Don't add any extra video delay. - video_delay_ms = total_video_delay_target_ms; - channel_delay_.extra_video_delay_ms = 0; - channel_delay_.last_video_delay_ms = video_delay_ms; - channel_delay_.last_sync_delay = 1; - } else { // channel_delay_.last_sync_delay >= 0 - // First time after a delay change, don't add any extra delay. - // This is to not toggle back and forth too much. - channel_delay_.extra_audio_delay_ms = 0; - // Set minimum video delay - video_delay_ms = total_video_delay_target_ms; - channel_delay_.extra_video_delay_ms = 0; - channel_delay_.last_video_delay_ms = video_delay_ms; - channel_delay_.last_sync_delay = 0; - } - } - } else { // if (current_diffMS > 0) - // The minimum video delay is lower than the current audio delay. - // We need to decrease possible extra audio delay, or - // add extra video delay. - - if (channel_delay_.extra_audio_delay_ms > 0) { - // We have extra delay in VoiceEngine - // Start with decreasing the voice delay - int audio_diff_ms = current_diff_ms / 2; - if (audio_diff_ms < -1 * kMaxAudioDiffMs) { - // Don't change the delay too much at once. - audio_diff_ms = -1 * kMaxAudioDiffMs; - } - // Add the negative difference. - channel_delay_.extra_audio_delay_ms += audio_diff_ms; - - if (channel_delay_.extra_audio_delay_ms < 0) { - // Negative values not allowed. - channel_delay_.extra_audio_delay_ms = 0; - channel_delay_.last_sync_delay = 0; - } else { - // There is more audio delay to use for the next round. - channel_delay_.last_sync_delay = 1; - } - - // Keep the video delay at the minimum values. - video_delay_ms = total_video_delay_target_ms; - channel_delay_.extra_video_delay_ms = 0; - channel_delay_.last_video_delay_ms = video_delay_ms; - } else { // channel_delay_.extra_audio_delay_ms > 0 - // We have no extra delay in VoiceEngine, increase the video delay. - channel_delay_.extra_audio_delay_ms = 0; - - // Make the difference positive. - int video_diff_ms = -1 * current_diff_ms; - - // This is the desired delay. - video_delay_ms = total_video_delay_target_ms + video_diff_ms; - if (video_delay_ms > channel_delay_.last_video_delay_ms) { - if (video_delay_ms > - channel_delay_.last_video_delay_ms + kMaxVideoDiffMs) { - // Don't increase the delay too much at once - video_delay_ms = - channel_delay_.last_video_delay_ms + kMaxVideoDiffMs; - } - // Verify we don't go above the maximum allowed delay - if (video_delay_ms > kMaxDelay) { - video_delay_ms = kMaxDelay; - } - } else { - if (video_delay_ms < - channel_delay_.last_video_delay_ms - kMaxVideoDiffMs) { - // Don't decrease the delay too much at once - video_delay_ms = - channel_delay_.last_video_delay_ms - kMaxVideoDiffMs; - } - // Verify we don't go below the minimum delay - if (video_delay_ms < total_video_delay_target_ms) { - video_delay_ms = total_video_delay_target_ms; - } - } - // Store the values - channel_delay_.extra_video_delay_ms = - video_delay_ms - total_video_delay_target_ms; - channel_delay_.last_video_delay_ms = video_delay_ms; - channel_delay_.last_sync_delay = -1; - } - } - - WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_, - "Sync video delay %d ms for video channel and audio delay %d for audio " - "channel %d", - video_delay_ms, channel_delay_.extra_audio_delay_ms, voe_channel_id_); - // Set the extra audio delay.synchronization if (voe_sync_interface_->SetMinimumPlayoutDelay( - voe_channel_id_, channel_delay_.extra_audio_delay_ms) == -1) { + voe_channel_id_, extra_audio_delay_ms) == -1) { WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideo, channel_id_, "Error setting voice delay"); } - - if (video_delay_ms < 0) { - video_delay_ms = 0; - } - total_video_delay_target_ms = - (total_video_delay_target_ms > video_delay_ms) ? - total_video_delay_target_ms : video_delay_ms; vcm_->SetMinimumPlayoutDelay(total_video_delay_target_ms); WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, channel_id_, "New Video delay target is: %d", total_video_delay_target_ms); diff --git a/src/video_engine/vie_sync_module.h b/src/video_engine/vie_sync_module.h index 052f7e3888..c93d58621b 100644 --- a/src/video_engine/vie_sync_module.h +++ b/src/video_engine/vie_sync_module.h @@ -14,14 +14,15 @@ #ifndef WEBRTC_VIDEO_ENGINE_VIE_SYNC_MODULE_H_ #define WEBRTC_VIDEO_ENGINE_VIE_SYNC_MODULE_H_ -#include "module.h" +#include "modules/interface/module.h" #include "system_wrappers/interface/scoped_ptr.h" -#include "tick_util.h" +#include "system_wrappers/interface/tick_util.h" namespace webrtc { class CriticalSectionWrapper; class RtpRtcp; +class StreamSynchronization; class VideoCodingModule; class VoEVideoSync; @@ -48,22 +49,7 @@ class ViESyncModule : public Module { int voe_channel_id_; VoEVideoSync* voe_sync_interface_; TickTime last_sync_time_; - - struct ViESyncDelay { - ViESyncDelay() { - extra_video_delay_ms = 0; - last_video_delay_ms = 0; - extra_audio_delay_ms = 0; - last_sync_delay = 0; - network_delay = 120; - } - int extra_video_delay_ms; - int last_video_delay_ms; - int extra_audio_delay_ms; - int last_sync_delay; - int network_delay; - }; - ViESyncDelay channel_delay_; + scoped_ptr sync_; }; } // namespace webrtc