diff --git a/webrtc/video_engine/stream_synchronization.cc b/webrtc/video_engine/stream_synchronization.cc index 48eb36a5c4..3e8d0eaafa 100644 --- a/webrtc/video_engine/stream_synchronization.cc +++ b/webrtc/video_engine/stream_synchronization.cc @@ -18,9 +18,11 @@ namespace webrtc { -const int kMaxVideoDiffMs = 80; -const int kMaxAudioDiffMs = 80; -const int kMaxDeltaDelayMs = 1500; +static const int kMaxChangeMs = 80; +static const int kMaxDeltaDelayMs = 10000; +static const int kFilterLength = 4; +// Minimum difference between audio and video to warrant a change. +static const int kMinDeltaMs = 30; struct ViESyncDelay { ViESyncDelay() { @@ -43,7 +45,8 @@ StreamSynchronization::StreamSynchronization(int audio_channel_id, : channel_delay_(new ViESyncDelay), audio_channel_id_(audio_channel_id), video_channel_id_(video_channel_id), - base_target_delay_ms_(0) {} + base_target_delay_ms_(0), + avg_diff_ms_(0) {} StreamSynchronization::~StreamSynchronization() { delete channel_delay_; @@ -100,11 +103,24 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms, WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, "Current diff is: %d for audio channel: %d", relative_delay_ms, audio_channel_id_); + int current_diff_ms = *total_video_delay_target_ms - current_audio_delay_ms + relative_delay_ms; + avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ + + current_diff_ms) / kFilterLength; + if (abs(avg_diff_ms_) < kMinDeltaMs) { + // Don't adjust if the diff is within our margin. + return false; + } + + // Make sure we don't move too fast. + int diff_ms = avg_diff_ms_ / 2; + diff_ms = std::min(diff_ms, kMaxChangeMs); + diff_ms = std::max(diff_ms, -kMaxChangeMs); + int video_delay_ms = base_target_delay_ms_; - if (current_diff_ms > 0) { + if (diff_ms > 0) { // The minimum video delay is longer than the current audio delay. // We need to decrease extra video delay, if we have added extra delay // earlier, or add extra audio delay. @@ -116,10 +132,8 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms, video_delay_ms = *total_video_delay_target_ms; // Check that we don't reduce the delay more than what is allowed. - if (video_delay_ms < - channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) { - video_delay_ms = - channel_delay_->last_video_delay_ms - kMaxVideoDiffMs; + if (video_delay_ms < channel_delay_->last_video_delay_ms - diff_ms) { + video_delay_ms = channel_delay_->last_video_delay_ms - diff_ms; channel_delay_->extra_video_delay_ms = video_delay_ms - *total_video_delay_target_ms; } else { @@ -132,21 +146,13 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms, // We have no extra video delay to remove, increase the audio delay. if (channel_delay_->last_sync_delay >= 0) { // We have increased the audio delay earlier, increase it even more. - int audio_diff_ms = current_diff_ms / 2; - if (audio_diff_ms > kMaxAudioDiffMs) { - // We only allow a maximum change of KMaxAudioDiffMS for audio - // due to NetEQ maximum changes. - audio_diff_ms = kMaxAudioDiffMs; - } // Increase the audio delay. - channel_delay_->extra_audio_delay_ms += audio_diff_ms; + channel_delay_->extra_audio_delay_ms += diff_ms; // Don't set a too high delay. - if (channel_delay_->extra_audio_delay_ms > - base_target_delay_ms_ + kMaxDeltaDelayMs) { - channel_delay_->extra_audio_delay_ms = - base_target_delay_ms_ + kMaxDeltaDelayMs; - } + channel_delay_->extra_audio_delay_ms = std::min( + channel_delay_->extra_audio_delay_ms, + base_target_delay_ms_ + kMaxDeltaDelayMs); // Don't add any extra video delay. video_delay_ms = *total_video_delay_target_ms; @@ -164,20 +170,15 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms, channel_delay_->last_sync_delay = 0; } } - } else { // if (current_diff_ms > 0) + } else { // if (diff_ms > 0) // The minimum video delay is lower than the current audio delay. // We need to decrease possible extra audio delay, or // add extra video delay. if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { // We have extra delay in VoiceEngine. // Start with decreasing the voice delay. - int audio_diff_ms = current_diff_ms / 2; - if (audio_diff_ms < -1 * kMaxAudioDiffMs) { - // Don't change the delay too much at once. - audio_diff_ms = -1 * kMaxAudioDiffMs; - } - // Add the negative difference. - channel_delay_->extra_audio_delay_ms += audio_diff_ms; + // Note: diff_ms is negative; add the negative difference. + channel_delay_->extra_audio_delay_ms += diff_ms; if (channel_delay_->extra_audio_delay_ms < 0) { // Negative values not allowed. @@ -196,34 +197,15 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms, // We have no extra delay in VoiceEngine, increase the video delay. channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; - // Make the difference positive. - int video_diff_ms = -1 * current_diff_ms; - // This is the desired delay. - video_delay_ms = *total_video_delay_target_ms + video_diff_ms; - if (video_delay_ms > channel_delay_->last_video_delay_ms) { - if (video_delay_ms > - channel_delay_->last_video_delay_ms + kMaxVideoDiffMs) { - // Don't increase the delay too much at once. - video_delay_ms = - channel_delay_->last_video_delay_ms + kMaxVideoDiffMs; - } - // Verify we don't go above the maximum allowed delay. - if (video_delay_ms > base_target_delay_ms_ + kMaxDeltaDelayMs) { - video_delay_ms = base_target_delay_ms_ + kMaxDeltaDelayMs; - } - } else { - if (video_delay_ms < - channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) { - // Don't decrease the delay too much at once. - video_delay_ms = - channel_delay_->last_video_delay_ms - kMaxVideoDiffMs; - } - // Verify we don't go below the minimum delay. - if (video_delay_ms < *total_video_delay_target_ms) { - video_delay_ms = *total_video_delay_target_ms; - } - } + // Note: diff_ms is negative. + video_delay_ms = std::max(*total_video_delay_target_ms, + channel_delay_->last_video_delay_ms) - diff_ms; + + // Verify we don't go above the maximum allowed delay. + video_delay_ms = std::min(video_delay_ms, + base_target_delay_ms_ + kMaxDeltaDelayMs); + // Store the values. channel_delay_->extra_video_delay_ms = video_delay_ms - *total_video_delay_target_ms; @@ -231,20 +213,16 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms, channel_delay_->last_sync_delay = -1; } } - + avg_diff_ms_ = 0; WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_, "Sync video delay %d ms for video channel and audio delay %d for audio " "channel %d", video_delay_ms, channel_delay_->extra_audio_delay_ms, audio_channel_id_); *extra_audio_delay_ms = channel_delay_->extra_audio_delay_ms; - - if (video_delay_ms < 0) { - video_delay_ms = 0; - } - *total_video_delay_target_ms = - (*total_video_delay_target_ms > video_delay_ms) ? - *total_video_delay_target_ms : video_delay_ms; + video_delay_ms = std::max(video_delay_ms, 0); + *total_video_delay_target_ms = std::max(*total_video_delay_target_ms, + video_delay_ms); return true; } diff --git a/webrtc/video_engine/stream_synchronization.h b/webrtc/video_engine/stream_synchronization.h index 9b7780c33a..f057b76d09 100644 --- a/webrtc/video_engine/stream_synchronization.h +++ b/webrtc/video_engine/stream_synchronization.h @@ -52,6 +52,7 @@ class StreamSynchronization { int audio_channel_id_; int video_channel_id_; int base_target_delay_ms_; + int avg_diff_ms_; }; } // namespace webrtc diff --git a/webrtc/video_engine/stream_synchronization_unittest.cc b/webrtc/video_engine/stream_synchronization_unittest.cc index 44cb146d20..395556b5e0 100644 --- a/webrtc/video_engine/stream_synchronization_unittest.cc +++ b/webrtc/video_engine/stream_synchronization_unittest.cc @@ -25,6 +25,7 @@ enum { kMaxDelay = 1500 }; enum { kDefaultAudioFrequency = 8000 }; enum { kDefaultVideoFrequency = 90000 }; const double kNtpFracPerMs = 4.294967296E6; +static const int kSmoothingFilter = 4 * 2; class Time { public: @@ -160,13 +161,14 @@ class StreamSynchronizationTest : public ::testing::Test { int video_delay_ms = base_target_delay + 100; int extra_audio_delay_ms = 0; int total_video_delay_ms = base_target_delay; + int filtered_move = (audio_delay_ms - video_delay_ms) / kSmoothingFilter; EXPECT_TRUE(DelayedStreams(audio_delay_ms, video_delay_ms, current_audio_delay_ms, &extra_audio_delay_ms, &total_video_delay_ms)); - EXPECT_EQ(base_target_delay + kMaxVideoDiffMs, total_video_delay_ms); + EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms); EXPECT_EQ(base_target_delay, extra_audio_delay_ms); current_audio_delay_ms = extra_audio_delay_ms; @@ -180,7 +182,7 @@ class StreamSynchronizationTest : public ::testing::Test { current_audio_delay_ms, &extra_audio_delay_ms, &total_video_delay_ms)); - EXPECT_EQ(base_target_delay + 2 * kMaxVideoDiffMs, total_video_delay_ms); + EXPECT_EQ(base_target_delay + 2 * filtered_move, total_video_delay_ms); EXPECT_EQ(base_target_delay, extra_audio_delay_ms); current_audio_delay_ms = extra_audio_delay_ms; @@ -194,8 +196,7 @@ class StreamSynchronizationTest : public ::testing::Test { current_audio_delay_ms, &extra_audio_delay_ms, &total_video_delay_ms)); - EXPECT_EQ(base_target_delay + audio_delay_ms - video_delay_ms, - total_video_delay_ms); + EXPECT_EQ(base_target_delay + 3 * filtered_move, total_video_delay_ms); EXPECT_EQ(base_target_delay, extra_audio_delay_ms); // Simulate that NetEQ introduces some audio delay. @@ -210,8 +211,9 @@ class StreamSynchronizationTest : public ::testing::Test { current_audio_delay_ms, &extra_audio_delay_ms, &total_video_delay_ms)); - EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms, - total_video_delay_ms); + filtered_move = 3 * filtered_move + + (50 + audio_delay_ms - video_delay_ms) / kSmoothingFilter; + EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms); EXPECT_EQ(base_target_delay, extra_audio_delay_ms); // Simulate that NetEQ reduces its delay. @@ -226,8 +228,11 @@ class StreamSynchronizationTest : public ::testing::Test { current_audio_delay_ms, &extra_audio_delay_ms, &total_video_delay_ms)); - EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms, - total_video_delay_ms); + + filtered_move = filtered_move + + (10 + audio_delay_ms - video_delay_ms) / kSmoothingFilter; + + EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms); EXPECT_EQ(base_target_delay, extra_audio_delay_ms); } @@ -245,7 +250,7 @@ class StreamSynchronizationTest : public ::testing::Test { &total_video_delay_ms)); EXPECT_EQ(base_target_delay, total_video_delay_ms); // The audio delay is not allowed to change more than this in 1 second. - EXPECT_EQ(base_target_delay + kMaxAudioDiffMs, extra_audio_delay_ms); + EXPECT_GE(base_target_delay + kMaxAudioDiffMs, extra_audio_delay_ms); current_audio_delay_ms = extra_audio_delay_ms; int current_extra_delay_ms = extra_audio_delay_ms; @@ -283,7 +288,7 @@ class StreamSynchronizationTest : public ::testing::Test { current_extra_delay_ms = extra_audio_delay_ms; // Simulate that NetEQ for some reason reduced the delay. - current_audio_delay_ms = base_target_delay + 170; + current_audio_delay_ms = base_target_delay + 10; send_time_->IncreaseTimeMs(1000); receive_time_->IncreaseTimeMs(800); EXPECT_TRUE(DelayedStreams(audio_delay_ms, @@ -302,7 +307,7 @@ class StreamSynchronizationTest : public ::testing::Test { current_extra_delay_ms = extra_audio_delay_ms; // Simulate that NetEQ for some reason significantly increased the delay. - current_audio_delay_ms = base_target_delay + 250; + current_audio_delay_ms = base_target_delay + 350; send_time_->IncreaseTimeMs(1000); receive_time_->IncreaseTimeMs(800); EXPECT_TRUE(DelayedStreams(audio_delay_ms, @@ -320,12 +325,13 @@ class StreamSynchronizationTest : public ::testing::Test { } int MaxAudioDelayIncrease(int current_audio_delay_ms, int delay_ms) { - return std::min((delay_ms - current_audio_delay_ms) / 2, + return std::min((delay_ms - current_audio_delay_ms) / kSmoothingFilter, static_cast(kMaxAudioDiffMs)); } int MaxAudioDelayDecrease(int current_audio_delay_ms, int delay_ms) { - return std::max((delay_ms - current_audio_delay_ms) / 2, -kMaxAudioDiffMs); + return std::max((delay_ms - current_audio_delay_ms) / kSmoothingFilter, + -kMaxAudioDiffMs); } enum { kSendTimeOffsetMs = 98765 }; @@ -343,8 +349,8 @@ TEST_F(StreamSynchronizationTest, NoDelay) { int extra_audio_delay_ms = 0; int total_video_delay_ms = 0; - EXPECT_TRUE(DelayedStreams(0, 0, current_audio_delay_ms, - &extra_audio_delay_ms, &total_video_delay_ms)); + EXPECT_FALSE(DelayedStreams(0, 0, current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); EXPECT_EQ(0, extra_audio_delay_ms); EXPECT_EQ(0, total_video_delay_ms); } @@ -359,7 +365,7 @@ TEST_F(StreamSynchronizationTest, VideoDelay) { &extra_audio_delay_ms, &total_video_delay_ms)); EXPECT_EQ(0, extra_audio_delay_ms); // The video delay is not allowed to change more than this in 1 second. - EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms); + EXPECT_EQ(delay_ms / kSmoothingFilter, total_video_delay_ms); send_time_->IncreaseTimeMs(1000); receive_time_->IncreaseTimeMs(800); @@ -369,7 +375,7 @@ TEST_F(StreamSynchronizationTest, VideoDelay) { &extra_audio_delay_ms, &total_video_delay_ms)); EXPECT_EQ(0, extra_audio_delay_ms); // The video delay is not allowed to change more than this in 1 second. - EXPECT_EQ(2*kMaxVideoDiffMs, total_video_delay_ms); + EXPECT_EQ(2 * delay_ms / kSmoothingFilter, total_video_delay_ms); send_time_->IncreaseTimeMs(1000); receive_time_->IncreaseTimeMs(800); @@ -380,7 +386,7 @@ TEST_F(StreamSynchronizationTest, VideoDelay) { EXPECT_EQ(0, extra_audio_delay_ms); // Enough time should have elapsed for the requested total video delay to be // equal to the relative delay between audio and video, i.e., we are in sync. - EXPECT_EQ(delay_ms, total_video_delay_ms); + EXPECT_EQ(3 * delay_ms / kSmoothingFilter, total_video_delay_ms); } TEST_F(StreamSynchronizationTest, AudioDelay) { @@ -393,7 +399,7 @@ TEST_F(StreamSynchronizationTest, AudioDelay) { &extra_audio_delay_ms, &total_video_delay_ms)); EXPECT_EQ(0, total_video_delay_ms); // The audio delay is not allowed to change more than this in 1 second. - EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms); + EXPECT_EQ(delay_ms / kSmoothingFilter, extra_audio_delay_ms); current_audio_delay_ms = extra_audio_delay_ms; int current_extra_delay_ms = extra_audio_delay_ms; @@ -423,7 +429,7 @@ TEST_F(StreamSynchronizationTest, AudioDelay) { current_extra_delay_ms = extra_audio_delay_ms; // Simulate that NetEQ for some reason reduced the delay. - current_audio_delay_ms = 170; + current_audio_delay_ms = 10; send_time_->IncreaseTimeMs(1000); receive_time_->IncreaseTimeMs(800); EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms, @@ -438,7 +444,7 @@ TEST_F(StreamSynchronizationTest, AudioDelay) { current_extra_delay_ms = extra_audio_delay_ms; // Simulate that NetEQ for some reason significantly increased the delay. - current_audio_delay_ms = 250; + current_audio_delay_ms = 350; send_time_->IncreaseTimeMs(1000); receive_time_->IncreaseTimeMs(800); EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms, @@ -485,32 +491,29 @@ TEST_F(StreamSynchronizationTest, BaseDelay) { int extra_audio_delay_ms = 0; int total_video_delay_ms = base_target_delay_ms; sync_->SetTargetBufferingDelay(base_target_delay_ms); - EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms, - current_audio_delay_ms, - &extra_audio_delay_ms, &total_video_delay_ms)); - EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms); - EXPECT_EQ(base_target_delay_ms, total_video_delay_ms); + // We are in sync don't change. + EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); // Triggering another call with the same values. Delay should not be modified. base_target_delay_ms = 2000; current_audio_delay_ms = base_target_delay_ms; total_video_delay_ms = base_target_delay_ms; sync_->SetTargetBufferingDelay(base_target_delay_ms); - EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms, - current_audio_delay_ms, - &extra_audio_delay_ms, &total_video_delay_ms)); - EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms); - EXPECT_EQ(base_target_delay_ms, total_video_delay_ms); + // We are in sync don't change. + EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); // Changing delay value - intended to test this module only. In practice it // would take VoE time to adapt. base_target_delay_ms = 5000; current_audio_delay_ms = base_target_delay_ms; total_video_delay_ms = base_target_delay_ms; sync_->SetTargetBufferingDelay(base_target_delay_ms); - EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms, - current_audio_delay_ms, - &extra_audio_delay_ms, &total_video_delay_ms)); - EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms); - EXPECT_EQ(base_target_delay_ms, total_video_delay_ms); + // We are in sync don't change. + EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms, + current_audio_delay_ms, + &extra_audio_delay_ms, &total_video_delay_ms)); } TEST_F(StreamSynchronizationTest, BothDelayedAudioLaterWithBaseDelay) {