diff --git a/webrtc/api/androidvideotracksource.cc b/webrtc/api/androidvideotracksource.cc index 000337d34d..f49cc30c8c 100644 --- a/webrtc/api/androidvideotracksource.cc +++ b/webrtc/api/androidvideotracksource.cc @@ -237,9 +237,8 @@ bool AndroidVideoTrackSource::AdaptFrame(int width, RTC_DCHECK(camera_thread_checker_.CalledOnValidThread()); int64_t system_time_us = rtc::TimeMicros(); - - int64_t offset_us = - timestamp_aligner_.UpdateOffset(camera_time_us, system_time_us); + *translated_camera_time_us = + timestamp_aligner_.TranslateTimestamp(camera_time_us, system_time_us); if (!broadcaster_.frame_wanted()) { return false; @@ -254,8 +253,6 @@ bool AndroidVideoTrackSource::AdaptFrame(int width, *crop_x = (width - *crop_width) / 2; *crop_y = (height - *crop_height) / 2; - *translated_camera_time_us = timestamp_aligner_.ClipTimestamp( - camera_time_us + offset_us, system_time_us); return true; } diff --git a/webrtc/base/timestampaligner.cc b/webrtc/base/timestampaligner.cc index 41cac9bde0..281da8f862 100644 --- a/webrtc/base/timestampaligner.cc +++ b/webrtc/base/timestampaligner.cc @@ -8,15 +8,30 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + +#include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/timestampaligner.h" +#include "webrtc/base/timeutils.h" namespace rtc { -TimestampAligner::TimestampAligner() : frames_seen_(0), offset_us_(0) {} +TimestampAligner::TimestampAligner() + : frames_seen_(0), + offset_us_(0), + clip_bias_us_(0), + prev_translated_time_us_(std::numeric_limits::min()) {} TimestampAligner::~TimestampAligner() {} +int64_t TimestampAligner::TranslateTimestamp(int64_t camera_time_us, + int64_t system_time_us) { + return ClipTimestamp( + camera_time_us + UpdateOffset(camera_time_us, system_time_us), + system_time_us); +} + int64_t TimestampAligner::UpdateOffset(int64_t camera_time_us, int64_t system_time_us) { // Estimate the offset between system monotonic time and the capture @@ -63,18 +78,19 @@ int64_t TimestampAligner::UpdateOffset(int64_t camera_time_us, // If the current difference is far from the currently estimated // offset, the filter is reset. This could happen, e.g., if the // camera clock is reset, or cameras are plugged in and out, or if - // the application process is temporarily suspended. The limit of - // 300 ms should make this unlikely in normal operation, and at the - // same time, converging gradually rather than resetting the filter - // should be tolerable for jumps in camera time below this - // threshold. - static const int64_t kResetLimitUs = 300000; - if (std::abs(error_us) > kResetLimitUs) { + // the application process is temporarily suspended. Expected to + // happen for the very first timestamp (|frames_seen_| = 0). The + // threshold of 300 ms should make this unlikely in normal + // operation, and at the same time, converging gradually rather than + // resetting the filter should be tolerable for jumps in camera time + // below this threshold. + static const int64_t kResetThresholdUs = 300000; + if (std::abs(error_us) > kResetThresholdUs) { LOG(LS_INFO) << "Resetting timestamp translation after averaging " << frames_seen_ << " frames. Old offset: " << offset_us_ << ", new offset: " << diff_us; frames_seen_ = 0; - prev_translated_time_us_ = rtc::Optional(); + clip_bias_us_ = 0; } static const int kWindowSize = 100; @@ -85,23 +101,34 @@ int64_t TimestampAligner::UpdateOffset(int64_t camera_time_us, return offset_us_; } -int64_t TimestampAligner::ClipTimestamp(int64_t time_us, +int64_t TimestampAligner::ClipTimestamp(int64_t filtered_time_us, int64_t system_time_us) { - // Make timestamps monotonic. - if (!prev_translated_time_us_) { - // Initialize. - clip_bias_us_ = 0; - } else if (time_us < *prev_translated_time_us_) { - time_us = *prev_translated_time_us_; - } - - // Clip to make sure we don't produce time stamps in the future. - time_us -= clip_bias_us_; + const int64_t kMinFrameIntervalUs = rtc::kNumMicrosecsPerMillisec; + // Clip to make sure we don't produce timestamps in the future. + int64_t time_us = filtered_time_us - clip_bias_us_; if (time_us > system_time_us) { clip_bias_us_ += time_us - system_time_us; time_us = system_time_us; } - prev_translated_time_us_ = rtc::Optional(time_us); + // Make timestamps monotonic, with a minimum inter-frame interval of 1 ms. + else if (time_us < prev_translated_time_us_ + kMinFrameIntervalUs) { + time_us = prev_translated_time_us_ + kMinFrameIntervalUs; + if (time_us > system_time_us) { + // In the anomalous case that this function is called with values of + // |system_time_us| less than |kMinFrameIntervalUs| apart, we may output + // timestamps with with too short inter-frame interval. We may even return + // duplicate timestamps in case this function is called several times with + // exactly the same |system_time_us|. + LOG(LS_WARNING) << "too short translated timestamp interval: " + << "system time (us) = " << system_time_us + << ", interval (us) = " + << system_time_us - prev_translated_time_us_; + time_us = system_time_us; + } + } + RTC_DCHECK_GE(time_us, prev_translated_time_us_); + RTC_DCHECK_LE(time_us, system_time_us); + prev_translated_time_us_ = time_us; return time_us; } diff --git a/webrtc/base/timestampaligner.h b/webrtc/base/timestampaligner.h index 00431f355b..590499dab3 100644 --- a/webrtc/base/timestampaligner.h +++ b/webrtc/base/timestampaligner.h @@ -13,10 +13,18 @@ #include "webrtc/base/basictypes.h" #include "webrtc/base/constructormagic.h" -#include "webrtc/base/optional.h" namespace rtc { +// The TimestampAligner class helps translating camera timestamps into +// the same timescale as is used by rtc::TimeMicros(). Some cameras +// have built in timestamping which is more accurate than reading the +// system clock, but using a different epoch and unknown clock drift. +// Frame timestamps in webrtc should use rtc::TimeMicros (system monotonic +// time), and this class provides a filter which lets us use the +// rtc::TimeMicros timescale, and at the same time take advantage of +// higher accuracy of the camera clock. + // This class is not thread safe, so all calls to it must be synchronized // externally. class TimestampAligner { @@ -25,9 +33,23 @@ class TimestampAligner { ~TimestampAligner(); public: + // Translates camera timestamps to the same timescale as is used by + // rtc::TimeMicros(). |camera_time_us| is assumed to be accurate, but + // with an unknown epoch and clock drift. |system_time_us| is + // time according to rtc::TimeMicros(), preferably read as soon as + // possible when the frame is captured. It may have poor accuracy + // due to poor resolution or scheduling delays. Returns the + // translated timestamp. + int64_t TranslateTimestamp(int64_t camera_time_us, int64_t system_time_us); + + protected: // Update the estimated offset between camera time and system monotonic time. int64_t UpdateOffset(int64_t camera_time_us, int64_t system_time_us); + // Clip timestamp, return value is always + // <= |system_time_us|, and + // >= min(|prev_translated_time_us_| + |kMinFrameIntervalUs|, + // |system_time_us|). int64_t ClipTimestamp(int64_t filtered_time_us, int64_t system_time_us); private: @@ -36,11 +58,13 @@ class TimestampAligner { // Estimated offset between camera time and system monotonic time. int64_t offset_us_; - // State for timestamp clipping, applied after the filter, to ensure - // that translated timestamps are monotonic and not in the future. - // Subtracted from the translated timestamps. + // State for the ClipTimestamp method, applied after the filter. + // A large negative camera clock drift tends to push translated + // timestamps into the future. |clip_bias_us_| is subtracted from the + // translated timestamps, to get them back from the future. int64_t clip_bias_us_; - rtc::Optional prev_translated_time_us_; + // Used to ensure that translated timestamps are monotonous. + int64_t prev_translated_time_us_; RTC_DISALLOW_COPY_AND_ASSIGN(TimestampAligner); }; diff --git a/webrtc/base/timestampaligner_unittest.cc b/webrtc/base/timestampaligner_unittest.cc index ae96de0a88..a4c0e5a41f 100644 --- a/webrtc/base/timestampaligner_unittest.cc +++ b/webrtc/base/timestampaligner_unittest.cc @@ -11,6 +11,7 @@ #include #include +#include #include "webrtc/base/gunit.h" #include "webrtc/base/random.h" @@ -39,95 +40,148 @@ double MeanTimeDifference(int nsamples, int window_size) { } } -} // Anonymous namespace - -class TimestampAlignerTest : public testing::Test { - protected: - void TestTimestampFilter(double rel_freq_error) { - const int64_t kEpoch = 10000; - const int64_t kJitterUs = 5000; - const int64_t kIntervalUs = 33333; // 30 FPS - const int kWindowSize = 100; - const int kNumFrames = 3 * kWindowSize; - - int64_t interval_error_us = kIntervalUs * rel_freq_error; - int64_t system_start_us = rtc::TimeMicros(); - webrtc::Random random(17); - - int64_t prev_translated_time_us = system_start_us; - - for (int i = 0; i < kNumFrames; i++) { - // Camera time subject to drift. - int64_t camera_time_us = kEpoch + i * (kIntervalUs + interval_error_us); - int64_t system_time_us = system_start_us + i * kIntervalUs; - // And system time readings are subject to jitter. - int64_t system_measured_us = system_time_us + random.Rand(kJitterUs); - - int64_t offset_us = - timestamp_aligner_.UpdateOffset(camera_time_us, system_measured_us); - - int64_t filtered_time_us = camera_time_us + offset_us; - int64_t translated_time_us = timestamp_aligner_.ClipTimestamp( - filtered_time_us, system_measured_us); - - EXPECT_LE(translated_time_us, system_measured_us); - EXPECT_GE(translated_time_us, prev_translated_time_us); - - // The relative frequency error contributes to the expected error - // by a factor which is the difference between the current time - // and the average of earlier sample times. - int64_t expected_error_us = - kJitterUs / 2 + - rel_freq_error * kIntervalUs * MeanTimeDifference(i, kWindowSize); - - int64_t bias_us = filtered_time_us - translated_time_us; - EXPECT_GE(bias_us, 0); - - if (i == 0) { - EXPECT_EQ(translated_time_us, system_measured_us); - } else { - EXPECT_NEAR(filtered_time_us, system_time_us + expected_error_us, - 2.0 * kJitterUs / sqrt(std::max(i, kWindowSize))); - } - // If the camera clock runs too fast (rel_freq_error > 0.0), The - // bias is expected to roughly cancel the expected error from the - // clock drift, as this grows. Otherwise, it reflects the - // measurement noise. The tolerances here were selected after some - // trial and error. - if (i < 10 || rel_freq_error <= 0.0) { - EXPECT_LE(bias_us, 3000); - } else { - EXPECT_NEAR(bias_us, expected_error_us, 1500); - } - prev_translated_time_us = translated_time_us; - } - } - - private: - TimestampAligner timestamp_aligner_; +class TimestampAlignerForTest : public TimestampAligner { + // Make internal methods accessible to testing. + public: + using TimestampAligner::UpdateOffset; + using TimestampAligner::ClipTimestamp; }; -TEST_F(TimestampAlignerTest, AttenuateTimestampJitterNoDrift) { +void TestTimestampFilter(double rel_freq_error) { + TimestampAlignerForTest timestamp_aligner_for_test; + TimestampAligner timestamp_aligner; + const int64_t kEpoch = 10000; + const int64_t kJitterUs = 5000; + const int64_t kIntervalUs = 33333; // 30 FPS + const int kWindowSize = 100; + const int kNumFrames = 3 * kWindowSize; + + int64_t interval_error_us = kIntervalUs * rel_freq_error; + int64_t system_start_us = rtc::TimeMicros(); + webrtc::Random random(17); + + int64_t prev_translated_time_us = system_start_us; + + for (int i = 0; i < kNumFrames; i++) { + // Camera time subject to drift. + int64_t camera_time_us = kEpoch + i * (kIntervalUs + interval_error_us); + int64_t system_time_us = system_start_us + i * kIntervalUs; + // And system time readings are subject to jitter. + int64_t system_measured_us = system_time_us + random.Rand(kJitterUs); + + int64_t offset_us = timestamp_aligner_for_test.UpdateOffset( + camera_time_us, system_measured_us); + + int64_t filtered_time_us = camera_time_us + offset_us; + int64_t translated_time_us = timestamp_aligner_for_test.ClipTimestamp( + filtered_time_us, system_measured_us); + + // Check that we get identical result from the all-in-one helper method. + ASSERT_EQ(translated_time_us, timestamp_aligner.TranslateTimestamp( + camera_time_us, system_measured_us)); + + EXPECT_LE(translated_time_us, system_measured_us); + EXPECT_GE(translated_time_us, + prev_translated_time_us + rtc::kNumMicrosecsPerMillisec); + + // The relative frequency error contributes to the expected error + // by a factor which is the difference between the current time + // and the average of earlier sample times. + int64_t expected_error_us = + kJitterUs / 2 + + rel_freq_error * kIntervalUs * MeanTimeDifference(i, kWindowSize); + + int64_t bias_us = filtered_time_us - translated_time_us; + EXPECT_GE(bias_us, 0); + + if (i == 0) { + EXPECT_EQ(translated_time_us, system_measured_us); + } else { + EXPECT_NEAR(filtered_time_us, system_time_us + expected_error_us, + 2.0 * kJitterUs / sqrt(std::max(i, kWindowSize))); + } + // If the camera clock runs too fast (rel_freq_error > 0.0), The + // bias is expected to roughly cancel the expected error from the + // clock drift, as this grows. Otherwise, it reflects the + // measurement noise. The tolerances here were selected after some + // trial and error. + if (i < 10 || rel_freq_error <= 0.0) { + EXPECT_LE(bias_us, 3000); + } else { + EXPECT_NEAR(bias_us, expected_error_us, 1500); + } + prev_translated_time_us = translated_time_us; + } +} + +} // Anonymous namespace + +TEST(TimestampAlignerTest, AttenuateTimestampJitterNoDrift) { TestTimestampFilter(0.0); } // 100 ppm is a worst case for a reasonable crystal. -TEST_F(TimestampAlignerTest, AttenuateTimestampJitterSmallPosDrift) { +TEST(TimestampAlignerTest, AttenuateTimestampJitterSmallPosDrift) { TestTimestampFilter(0.0001); } -TEST_F(TimestampAlignerTest, AttenuateTimestampJitterSmallNegDrift) { +TEST(TimestampAlignerTest, AttenuateTimestampJitterSmallNegDrift) { TestTimestampFilter(-0.0001); } // 3000 ppm, 3 ms / s, is the worst observed drift, see // https://bugs.chromium.org/p/webrtc/issues/detail?id=5456 -TEST_F(TimestampAlignerTest, AttenuateTimestampJitterLargePosDrift) { +TEST(TimestampAlignerTest, AttenuateTimestampJitterLargePosDrift) { TestTimestampFilter(0.003); } -TEST_F(TimestampAlignerTest, AttenuateTimestampJitterLargeNegDrift) { +TEST(TimestampAlignerTest, AttenuateTimestampJitterLargeNegDrift) { TestTimestampFilter(-0.003); } +// Exhibits a mostly hypothetical problem, where certain inputs to the +// TimestampAligner.UpdateOffset filter result in non-monotonous +// translated timestamps. This test verifies that the ClipTimestamp +// logic handles this case correctly. +TEST(TimestampAlignerTest, ClipToMonotonous) { + TimestampAlignerForTest timestamp_aligner; + + // For system time stamps { 0, s1, s1 + s2 }, and camera timestamps + // {0, c1, c1 + c2}, we exhibit non-monotonous behaviour if and only + // if c1 > s1 + 2 s2 + 4 c2. + const int kNumSamples = 3; + const int64_t camera_time_us[kNumSamples] = {0, 80000, 90001}; + const int64_t system_time_us[kNumSamples] = {0, 10000, 20000}; + const int64_t expected_offset_us[kNumSamples] = {0, -35000, -46667}; + + // Non-monotonic translated timestamps can happen when only for + // translated timestamps in the future. Which is tolerated if + // |timestamp_aligner.clip_bias_us| is large enough. Instead of + // changing that private member for this test, just add the bias to + // |system_time_us| when calling ClipTimestamp. + const int64_t kClipBiasUs = 100000; + + bool did_clip = false; + int64_t prev_timestamp_us = std::numeric_limits::min(); + for (int i = 0; i < kNumSamples; i++) { + int64_t offset_us = + timestamp_aligner.UpdateOffset(camera_time_us[i], system_time_us[i]); + EXPECT_EQ(offset_us, expected_offset_us[i]); + + int64_t translated_timestamp_us = camera_time_us[i] + offset_us; + int64_t clip_timestamp_us = timestamp_aligner.ClipTimestamp( + translated_timestamp_us, system_time_us[i] + kClipBiasUs); + if (translated_timestamp_us <= prev_timestamp_us) { + did_clip = true; + EXPECT_EQ(clip_timestamp_us, + prev_timestamp_us + rtc::kNumMicrosecsPerMillisec); + } else { + // No change from clipping. + EXPECT_EQ(clip_timestamp_us, translated_timestamp_us); + } + prev_timestamp_us = clip_timestamp_us; + } + EXPECT_TRUE(did_clip); +} + } // namespace rtc diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc index 9d297bf914..c340760468 100644 --- a/webrtc/media/base/videocapturer.cc +++ b/webrtc/media/base/videocapturer.cc @@ -225,11 +225,10 @@ bool VideoCapturer::AdaptFrame(int width, int* crop_x, int* crop_y, int64_t* translated_camera_time_us) { - int64_t offset_us = - translated_camera_time_us - ? timestamp_aligner_.UpdateOffset(camera_time_us, system_time_us) - : 0; - + if (translated_camera_time_us) { + *translated_camera_time_us = + timestamp_aligner_.TranslateTimestamp(camera_time_us, system_time_us); + } if (!broadcaster_.frame_wanted()) { return false; } @@ -252,10 +251,6 @@ bool VideoCapturer::AdaptFrame(int width, *crop_y = 0; } - if (translated_camera_time_us) { - *translated_camera_time_us = timestamp_aligner_.ClipTimestamp( - camera_time_us + offset_us, system_time_us); - } return true; }