admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New method TimestampAligner::TranslateTimestamp

Browse Source 
Also enforce a minimum inter-frame interval of 1 ms,
fix a bug in the clipping logic, and improve comments.

BUG=webrtc:5740

Review-Url: https://codereview.webrtc.org/2325563002
Cr-Commit-Position: refs/heads/master@{#14206}
This commit is contained in:
nisse 2016-09-14 00:37:00 -07:00 committed by Commit bot
parent f8a4ecc4a1
commit a075848ebd
5 changed files with 208 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
webrtc/api/androidvideotracksource.cc
View File

@ -237,9 +237,8 @@ bool AndroidVideoTrackSource::AdaptFrame(int width,
RTC_DCHECK(camera_thread_checker_.CalledOnValidThread());
int64_t system_time_us = rtc::TimeMicros();
int64_t offset_us =
timestamp_aligner_.UpdateOffset(camera_time_us, system_time_us);
*translated_camera_time_us =
timestamp_aligner_.TranslateTimestamp(camera_time_us, system_time_us);
if (!broadcaster_.frame_wanted()) {
return false;
@ -254,8 +253,6 @@ bool AndroidVideoTrackSource::AdaptFrame(int width,
*crop_x = (width - *crop_width) / 2;
*crop_y = (height - *crop_height) / 2;
*translated_camera_time_us = timestamp_aligner_.ClipTimestamp(
camera_time_us + offset_us, system_time_us);
return true;
}

69
webrtc/base/timestampaligner.cc
View File

@ -8,15 +8,30 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include <limits>
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/timestampaligner.h"
#include "webrtc/base/timeutils.h"
namespace rtc {
TimestampAligner::TimestampAligner() : frames_seen_(0), offset_us_(0) {}
TimestampAligner::TimestampAligner()
: frames_seen_(0),
offset_us_(0),
clip_bias_us_(0),
prev_translated_time_us_(std::numeric_limits<int64_t>::min()) {}
TimestampAligner::~TimestampAligner() {}
int64_t TimestampAligner::TranslateTimestamp(int64_t camera_time_us,
int64_t system_time_us) {
return ClipTimestamp(
camera_time_us + UpdateOffset(camera_time_us, system_time_us),
system_time_us);
}
int64_t TimestampAligner::UpdateOffset(int64_t camera_time_us,
int64_t system_time_us) {
// Estimate the offset between system monotonic time and the capture
@ -63,18 +78,19 @@ int64_t TimestampAligner::UpdateOffset(int64_t camera_time_us,
// If the current difference is far from the currently estimated
// offset, the filter is reset. This could happen, e.g., if the
// camera clock is reset, or cameras are plugged in and out, or if
// the application process is temporarily suspended. The limit of
// 300 ms should make this unlikely in normal operation, and at the
// same time, converging gradually rather than resetting the filter
// should be tolerable for jumps in camera time below this
// threshold.
static const int64_t kResetLimitUs = 300000;
if (std::abs(error_us) > kResetLimitUs) {
// the application process is temporarily suspended. Expected to
// happen for the very first timestamp (|frames_seen_| = 0). The
// threshold of 300 ms should make this unlikely in normal
// operation, and at the same time, converging gradually rather than
// resetting the filter should be tolerable for jumps in camera time
// below this threshold.
static const int64_t kResetThresholdUs = 300000;
if (std::abs(error_us) > kResetThresholdUs) {
LOG(LS_INFO) << "Resetting timestamp translation after averaging "
<< frames_seen_ << " frames. Old offset: " << offset_us_
<< ", new offset: " << diff_us;
frames_seen_ = 0;
prev_translated_time_us_ = rtc::Optional<int64_t>();
clip_bias_us_ = 0;
}
static const int kWindowSize = 100;
@ -85,23 +101,34 @@ int64_t TimestampAligner::UpdateOffset(int64_t camera_time_us,
return offset_us_;
}
int64_t TimestampAligner::ClipTimestamp(int64_t time_us,
int64_t TimestampAligner::ClipTimestamp(int64_t filtered_time_us,
int64_t system_time_us) {
// Make timestamps monotonic.
if (!prev_translated_time_us_) {
// Initialize.
clip_bias_us_ = 0;
} else if (time_us < *prev_translated_time_us_) {
time_us = *prev_translated_time_us_;
}
// Clip to make sure we don't produce time stamps in the future.
time_us -= clip_bias_us_;
const int64_t kMinFrameIntervalUs = rtc::kNumMicrosecsPerMillisec;
// Clip to make sure we don't produce timestamps in the future.
int64_t time_us = filtered_time_us - clip_bias_us_;
if (time_us > system_time_us) {
clip_bias_us_ += time_us - system_time_us;
time_us = system_time_us;
}
prev_translated_time_us_ = rtc::Optional<int64_t>(time_us);
// Make timestamps monotonic, with a minimum inter-frame interval of 1 ms.
else if (time_us < prev_translated_time_us_ + kMinFrameIntervalUs) {
time_us = prev_translated_time_us_ + kMinFrameIntervalUs;
if (time_us > system_time_us) {
// In the anomalous case that this function is called with values of
// |system_time_us| less than |kMinFrameIntervalUs| apart, we may output
// timestamps with with too short inter-frame interval. We may even return
// duplicate timestamps in case this function is called several times with
// exactly the same |system_time_us|.
LOG(LS_WARNING) << "too short translated timestamp interval: "
<< "system time (us) = " << system_time_us
<< ", interval (us) = "
<< system_time_us - prev_translated_time_us_;
time_us = system_time_us;
}
}
RTC_DCHECK_GE(time_us, prev_translated_time_us_);
RTC_DCHECK_LE(time_us, system_time_us);
prev_translated_time_us_ = time_us;
return time_us;
}

34
webrtc/base/timestampaligner.h
View File

@ -13,10 +13,18 @@
#include "webrtc/base/basictypes.h"
#include "webrtc/base/constructormagic.h"
#include "webrtc/base/optional.h"
namespace rtc {
// The TimestampAligner class helps translating camera timestamps into
// the same timescale as is used by rtc::TimeMicros(). Some cameras
// have built in timestamping which is more accurate than reading the
// system clock, but using a different epoch and unknown clock drift.
// Frame timestamps in webrtc should use rtc::TimeMicros (system monotonic
// time), and this class provides a filter which lets us use the
// rtc::TimeMicros timescale, and at the same time take advantage of
// higher accuracy of the camera clock.
// This class is not thread safe, so all calls to it must be synchronized
// externally.
class TimestampAligner {
@ -25,9 +33,23 @@ class TimestampAligner {
~TimestampAligner();
public:
// Translates camera timestamps to the same timescale as is used by
// rtc::TimeMicros(). |camera_time_us| is assumed to be accurate, but
// with an unknown epoch and clock drift. |system_time_us| is
// time according to rtc::TimeMicros(), preferably read as soon as
// possible when the frame is captured. It may have poor accuracy
// due to poor resolution or scheduling delays. Returns the
// translated timestamp.
int64_t TranslateTimestamp(int64_t camera_time_us, int64_t system_time_us);
protected:
// Update the estimated offset between camera time and system monotonic time.
int64_t UpdateOffset(int64_t camera_time_us, int64_t system_time_us);
// Clip timestamp, return value is always
// <= |system_time_us|, and
// >= min(|prev_translated_time_us_| + |kMinFrameIntervalUs|,
// |system_time_us|).
int64_t ClipTimestamp(int64_t filtered_time_us, int64_t system_time_us);
private:
@ -36,11 +58,13 @@ class TimestampAligner {
// Estimated offset between camera time and system monotonic time.
int64_t offset_us_;
// State for timestamp clipping, applied after the filter, to ensure
// that translated timestamps are monotonic and not in the future.
// Subtracted from the translated timestamps.
// State for the ClipTimestamp method, applied after the filter.
// A large negative camera clock drift tends to push translated
// timestamps into the future. |clip_bias_us_| is subtracted from the
// translated timestamps, to get them back from the future.
int64_t clip_bias_us_;
rtc::Optional<int64_t> prev_translated_time_us_;
// Used to ensure that translated timestamps are monotonous.
int64_t prev_translated_time_us_;
RTC_DISALLOW_COPY_AND_ASSIGN(TimestampAligner);
};

196
webrtc/base/timestampaligner_unittest.cc
View File

@ -11,6 +11,7 @@
#include <math.h>
#include <algorithm>
#include <limits>
#include "webrtc/base/gunit.h"
#include "webrtc/base/random.h"
@ -39,95 +40,148 @@ double MeanTimeDifference(int nsamples, int window_size) {
}
}
} // Anonymous namespace
class TimestampAlignerTest : public testing::Test {
protected:
void TestTimestampFilter(double rel_freq_error) {
const int64_t kEpoch = 10000;
const int64_t kJitterUs = 5000;
const int64_t kIntervalUs = 33333; // 30 FPS
const int kWindowSize = 100;
const int kNumFrames = 3 * kWindowSize;
int64_t interval_error_us = kIntervalUs * rel_freq_error;
int64_t system_start_us = rtc::TimeMicros();
webrtc::Random random(17);
int64_t prev_translated_time_us = system_start_us;
for (int i = 0; i < kNumFrames; i++) {
// Camera time subject to drift.
int64_t camera_time_us = kEpoch + i * (kIntervalUs + interval_error_us);
int64_t system_time_us = system_start_us + i * kIntervalUs;
// And system time readings are subject to jitter.
int64_t system_measured_us = system_time_us + random.Rand(kJitterUs);
int64_t offset_us =
timestamp_aligner_.UpdateOffset(camera_time_us, system_measured_us);
int64_t filtered_time_us = camera_time_us + offset_us;
int64_t translated_time_us = timestamp_aligner_.ClipTimestamp(
filtered_time_us, system_measured_us);
EXPECT_LE(translated_time_us, system_measured_us);
EXPECT_GE(translated_time_us, prev_translated_time_us);
// The relative frequency error contributes to the expected error
// by a factor which is the difference between the current time
// and the average of earlier sample times.
int64_t expected_error_us =
kJitterUs / 2 +
rel_freq_error * kIntervalUs * MeanTimeDifference(i, kWindowSize);
int64_t bias_us = filtered_time_us - translated_time_us;
EXPECT_GE(bias_us, 0);
if (i == 0) {
EXPECT_EQ(translated_time_us, system_measured_us);
} else {
EXPECT_NEAR(filtered_time_us, system_time_us + expected_error_us,
2.0 * kJitterUs / sqrt(std::max(i, kWindowSize)));
}
// If the camera clock runs too fast (rel_freq_error > 0.0), The
// bias is expected to roughly cancel the expected error from the
// clock drift, as this grows. Otherwise, it reflects the
// measurement noise. The tolerances here were selected after some
// trial and error.
if (i < 10 || rel_freq_error <= 0.0) {
EXPECT_LE(bias_us, 3000);
} else {
EXPECT_NEAR(bias_us, expected_error_us, 1500);
}
prev_translated_time_us = translated_time_us;
}
}
private:
TimestampAligner timestamp_aligner_;
class TimestampAlignerForTest : public TimestampAligner {
// Make internal methods accessible to testing.
public:
using TimestampAligner::UpdateOffset;
using TimestampAligner::ClipTimestamp;
};
TEST_F(TimestampAlignerTest, AttenuateTimestampJitterNoDrift) {
void TestTimestampFilter(double rel_freq_error) {
TimestampAlignerForTest timestamp_aligner_for_test;
TimestampAligner timestamp_aligner;
const int64_t kEpoch = 10000;
const int64_t kJitterUs = 5000;
const int64_t kIntervalUs = 33333; // 30 FPS
const int kWindowSize = 100;
const int kNumFrames = 3 * kWindowSize;
int64_t interval_error_us = kIntervalUs * rel_freq_error;
int64_t system_start_us = rtc::TimeMicros();
webrtc::Random random(17);
int64_t prev_translated_time_us = system_start_us;
for (int i = 0; i < kNumFrames; i++) {
// Camera time subject to drift.
int64_t camera_time_us = kEpoch + i * (kIntervalUs + interval_error_us);
int64_t system_time_us = system_start_us + i * kIntervalUs;
// And system time readings are subject to jitter.
int64_t system_measured_us = system_time_us + random.Rand(kJitterUs);
int64_t offset_us = timestamp_aligner_for_test.UpdateOffset(
camera_time_us, system_measured_us);
int64_t filtered_time_us = camera_time_us + offset_us;
int64_t translated_time_us = timestamp_aligner_for_test.ClipTimestamp(
filtered_time_us, system_measured_us);
// Check that we get identical result from the all-in-one helper method.
ASSERT_EQ(translated_time_us, timestamp_aligner.TranslateTimestamp(
camera_time_us, system_measured_us));
EXPECT_LE(translated_time_us, system_measured_us);
EXPECT_GE(translated_time_us,
prev_translated_time_us + rtc::kNumMicrosecsPerMillisec);
// The relative frequency error contributes to the expected error
// by a factor which is the difference between the current time
// and the average of earlier sample times.
int64_t expected_error_us =
kJitterUs / 2 +
rel_freq_error * kIntervalUs * MeanTimeDifference(i, kWindowSize);
int64_t bias_us = filtered_time_us - translated_time_us;
EXPECT_GE(bias_us, 0);
if (i == 0) {
EXPECT_EQ(translated_time_us, system_measured_us);
} else {
EXPECT_NEAR(filtered_time_us, system_time_us + expected_error_us,
2.0 * kJitterUs / sqrt(std::max(i, kWindowSize)));
}
// If the camera clock runs too fast (rel_freq_error > 0.0), The
// bias is expected to roughly cancel the expected error from the
// clock drift, as this grows. Otherwise, it reflects the
// measurement noise. The tolerances here were selected after some
// trial and error.
if (i < 10 || rel_freq_error <= 0.0) {
EXPECT_LE(bias_us, 3000);
} else {
EXPECT_NEAR(bias_us, expected_error_us, 1500);
}
prev_translated_time_us = translated_time_us;
}
}
} // Anonymous namespace
TEST(TimestampAlignerTest, AttenuateTimestampJitterNoDrift) {
TestTimestampFilter(0.0);
}
// 100 ppm is a worst case for a reasonable crystal.
TEST_F(TimestampAlignerTest, AttenuateTimestampJitterSmallPosDrift) {
TEST(TimestampAlignerTest, AttenuateTimestampJitterSmallPosDrift) {
TestTimestampFilter(0.0001);
}
TEST_F(TimestampAlignerTest, AttenuateTimestampJitterSmallNegDrift) {
TEST(TimestampAlignerTest, AttenuateTimestampJitterSmallNegDrift) {
TestTimestampFilter(-0.0001);
}
// 3000 ppm, 3 ms / s, is the worst observed drift, see
// https://bugs.chromium.org/p/webrtc/issues/detail?id=5456
TEST_F(TimestampAlignerTest, AttenuateTimestampJitterLargePosDrift) {
TEST(TimestampAlignerTest, AttenuateTimestampJitterLargePosDrift) {
TestTimestampFilter(0.003);
}
TEST_F(TimestampAlignerTest, AttenuateTimestampJitterLargeNegDrift) {
TEST(TimestampAlignerTest, AttenuateTimestampJitterLargeNegDrift) {
TestTimestampFilter(-0.003);
}
// Exhibits a mostly hypothetical problem, where certain inputs to the
// TimestampAligner.UpdateOffset filter result in non-monotonous
// translated timestamps. This test verifies that the ClipTimestamp
// logic handles this case correctly.
TEST(TimestampAlignerTest, ClipToMonotonous) {
TimestampAlignerForTest timestamp_aligner;
// For system time stamps { 0, s1, s1 + s2 }, and camera timestamps
// {0, c1, c1 + c2}, we exhibit non-monotonous behaviour if and only
// if c1 > s1 + 2 s2 + 4 c2.
const int kNumSamples = 3;
const int64_t camera_time_us[kNumSamples] = {0, 80000, 90001};
const int64_t system_time_us[kNumSamples] = {0, 10000, 20000};
const int64_t expected_offset_us[kNumSamples] = {0, -35000, -46667};
// Non-monotonic translated timestamps can happen when only for
// translated timestamps in the future. Which is tolerated if
// |timestamp_aligner.clip_bias_us| is large enough. Instead of
// changing that private member for this test, just add the bias to
// |system_time_us| when calling ClipTimestamp.
const int64_t kClipBiasUs = 100000;
bool did_clip = false;
int64_t prev_timestamp_us = std::numeric_limits<int64_t>::min();
for (int i = 0; i < kNumSamples; i++) {
int64_t offset_us =
timestamp_aligner.UpdateOffset(camera_time_us[i], system_time_us[i]);
EXPECT_EQ(offset_us, expected_offset_us[i]);
int64_t translated_timestamp_us = camera_time_us[i] + offset_us;
int64_t clip_timestamp_us = timestamp_aligner.ClipTimestamp(
translated_timestamp_us, system_time_us[i] + kClipBiasUs);
if (translated_timestamp_us <= prev_timestamp_us) {
did_clip = true;
EXPECT_EQ(clip_timestamp_us,
prev_timestamp_us + rtc::kNumMicrosecsPerMillisec);
} else {
// No change from clipping.
EXPECT_EQ(clip_timestamp_us, translated_timestamp_us);
}
prev_timestamp_us = clip_timestamp_us;
}
EXPECT_TRUE(did_clip);
}
} // namespace rtc

13
webrtc/media/base/videocapturer.cc
View File

@ -225,11 +225,10 @@ bool VideoCapturer::AdaptFrame(int width,
int* crop_x,
int* crop_y,
int64_t* translated_camera_time_us) {
int64_t offset_us =
translated_camera_time_us
? timestamp_aligner_.UpdateOffset(camera_time_us, system_time_us)
: 0;
if (translated_camera_time_us) {
*translated_camera_time_us =
timestamp_aligner_.TranslateTimestamp(camera_time_us, system_time_us);
}
if (!broadcaster_.frame_wanted()) {
return false;
}
@ -252,10 +251,6 @@ bool VideoCapturer::AdaptFrame(int width,
*crop_y = 0;
}
if (translated_camera_time_us) {
*translated_camera_time_us = timestamp_aligner_.ClipTimestamp(
camera_time_us + offset_us, system_time_us);
}
return true;
}