nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
nisse | a075848 | 2016-09-14 00:37:00 -0700 | [diff] [blame] | 11 | #include <limits> |
| 12 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 13 | #include "rtc_base/checks.h" |
| 14 | #include "rtc_base/logging.h" |
| 15 | #include "rtc_base/timestampaligner.h" |
| 16 | #include "rtc_base/timeutils.h" |
nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 17 | |
| 18 | namespace rtc { |
| 19 | |
nisse | a075848 | 2016-09-14 00:37:00 -0700 | [diff] [blame] | 20 | TimestampAligner::TimestampAligner() |
| 21 | : frames_seen_(0), |
| 22 | offset_us_(0), |
| 23 | clip_bias_us_(0), |
| 24 | prev_translated_time_us_(std::numeric_limits<int64_t>::min()) {} |
nisse | 76f91cd | 2016-08-24 01:58:42 -0700 | [diff] [blame] | 25 | |
nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 26 | TimestampAligner::~TimestampAligner() {} |
| 27 | |
nisse | a075848 | 2016-09-14 00:37:00 -0700 | [diff] [blame] | 28 | int64_t TimestampAligner::TranslateTimestamp(int64_t camera_time_us, |
| 29 | int64_t system_time_us) { |
| 30 | return ClipTimestamp( |
| 31 | camera_time_us + UpdateOffset(camera_time_us, system_time_us), |
| 32 | system_time_us); |
| 33 | } |
| 34 | |
nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 35 | int64_t TimestampAligner::UpdateOffset(int64_t camera_time_us, |
| 36 | int64_t system_time_us) { |
| 37 | // Estimate the offset between system monotonic time and the capture |
| 38 | // time from the camera. The camera is assumed to provide more |
| 39 | // accurate timestamps than we get from the system time. But the |
| 40 | // camera may use its own free-running clock with a large offset and |
| 41 | // a small drift compared to the system clock. So the model is |
| 42 | // basically |
| 43 | // |
| 44 | // y_k = c_0 + c_1 * x_k + v_k |
| 45 | // |
| 46 | // where x_k is the camera timestamp, believed to be accurate in its |
| 47 | // own scale. y_k is our reading of the system clock. v_k is the |
| 48 | // measurement noise, i.e., the delay from frame capture until the |
| 49 | // system clock was read. |
| 50 | // |
| 51 | // It's possible to do (weighted) least-squares estimation of both |
| 52 | // c_0 and c_1. Then we get the constants as c_1 = Cov(x,y) / |
| 53 | // Var(x), and c_0 = mean(y) - c_1 * mean(x). Substituting this c_0, |
| 54 | // we can rearrange the model as |
| 55 | // |
| 56 | // y_k = mean(y) + (x_k - mean(x)) + (c_1 - 1) * (x_k - mean(x)) + v_k |
| 57 | // |
| 58 | // Now if we use a weighted average which gradually forgets old |
| 59 | // values, x_k - mean(x) is bounded, of the same order as the time |
| 60 | // constant (and close to constant for a steady frame rate). In |
| 61 | // addition, the frequency error |c_1 - 1| should be small. Cameras |
| 62 | // with a frequency error up to 3000 ppm (3 ms drift per second) |
| 63 | // have been observed, but frequency errors below 100 ppm could be |
| 64 | // expected of any cheap crystal. |
| 65 | // |
| 66 | // Bottom line is that we ignore the c_1 term, and use only the estimator |
| 67 | // |
| 68 | // x_k + mean(y-x) |
| 69 | // |
| 70 | // where mean is plain averaging for initial samples, followed by |
| 71 | // exponential averaging. |
| 72 | |
| 73 | // The input for averaging, y_k - x_k in the above notation. |
| 74 | int64_t diff_us = system_time_us - camera_time_us; |
| 75 | // The deviation from the current average. |
| 76 | int64_t error_us = diff_us - offset_us_; |
| 77 | |
| 78 | // If the current difference is far from the currently estimated |
| 79 | // offset, the filter is reset. This could happen, e.g., if the |
| 80 | // camera clock is reset, or cameras are plugged in and out, or if |
nisse | a075848 | 2016-09-14 00:37:00 -0700 | [diff] [blame] | 81 | // the application process is temporarily suspended. Expected to |
| 82 | // happen for the very first timestamp (|frames_seen_| = 0). The |
| 83 | // threshold of 300 ms should make this unlikely in normal |
| 84 | // operation, and at the same time, converging gradually rather than |
| 85 | // resetting the filter should be tolerable for jumps in camera time |
| 86 | // below this threshold. |
| 87 | static const int64_t kResetThresholdUs = 300000; |
| 88 | if (std::abs(error_us) > kResetThresholdUs) { |
nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 89 | LOG(LS_INFO) << "Resetting timestamp translation after averaging " |
| 90 | << frames_seen_ << " frames. Old offset: " << offset_us_ |
| 91 | << ", new offset: " << diff_us; |
| 92 | frames_seen_ = 0; |
nisse | a075848 | 2016-09-14 00:37:00 -0700 | [diff] [blame] | 93 | clip_bias_us_ = 0; |
nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 94 | } |
| 95 | |
| 96 | static const int kWindowSize = 100; |
| 97 | if (frames_seen_ < kWindowSize) { |
| 98 | ++frames_seen_; |
| 99 | } |
| 100 | offset_us_ += error_us / frames_seen_; |
| 101 | return offset_us_; |
| 102 | } |
| 103 | |
nisse | a075848 | 2016-09-14 00:37:00 -0700 | [diff] [blame] | 104 | int64_t TimestampAligner::ClipTimestamp(int64_t filtered_time_us, |
nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 105 | int64_t system_time_us) { |
nisse | a075848 | 2016-09-14 00:37:00 -0700 | [diff] [blame] | 106 | const int64_t kMinFrameIntervalUs = rtc::kNumMicrosecsPerMillisec; |
| 107 | // Clip to make sure we don't produce timestamps in the future. |
| 108 | int64_t time_us = filtered_time_us - clip_bias_us_; |
nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 109 | if (time_us > system_time_us) { |
| 110 | clip_bias_us_ += time_us - system_time_us; |
| 111 | time_us = system_time_us; |
| 112 | } |
nisse | a075848 | 2016-09-14 00:37:00 -0700 | [diff] [blame] | 113 | // Make timestamps monotonic, with a minimum inter-frame interval of 1 ms. |
| 114 | else if (time_us < prev_translated_time_us_ + kMinFrameIntervalUs) { |
| 115 | time_us = prev_translated_time_us_ + kMinFrameIntervalUs; |
| 116 | if (time_us > system_time_us) { |
| 117 | // In the anomalous case that this function is called with values of |
| 118 | // |system_time_us| less than |kMinFrameIntervalUs| apart, we may output |
| 119 | // timestamps with with too short inter-frame interval. We may even return |
| 120 | // duplicate timestamps in case this function is called several times with |
| 121 | // exactly the same |system_time_us|. |
| 122 | LOG(LS_WARNING) << "too short translated timestamp interval: " |
| 123 | << "system time (us) = " << system_time_us |
| 124 | << ", interval (us) = " |
| 125 | << system_time_us - prev_translated_time_us_; |
| 126 | time_us = system_time_us; |
| 127 | } |
| 128 | } |
| 129 | RTC_DCHECK_GE(time_us, prev_translated_time_us_); |
| 130 | RTC_DCHECK_LE(time_us, system_time_us); |
| 131 | prev_translated_time_us_ = time_us; |
nisse | 191b359 | 2016-06-22 08:36:53 -0700 | [diff] [blame] | 132 | return time_us; |
| 133 | } |
| 134 | |
| 135 | } // namespace rtc |