blob: d1c8eb970c3cb4125022508b2cb43b3f36571843 [file] [log] [blame]
Jens Axboe02bcaa82006-11-24 10:42:00 +01001/*
Jens Axboef5cc0242006-11-24 10:47:40 +01002 * Clock functions
Jens Axboe02bcaa82006-11-24 10:42:00 +01003 */
Jens Axboef5cc0242006-11-24 10:47:40 +01004
Jens Axboe02bcaa82006-11-24 10:42:00 +01005#include <unistd.h>
Jens Axboec223da82010-03-24 13:23:53 +01006#include <math.h>
Jens Axboe02bcaa82006-11-24 10:42:00 +01007#include <sys/time.h>
Bruce Cran03e20d62011-01-02 20:14:54 +01008#include <time.h>
Jens Axboe02bcaa82006-11-24 10:42:00 +01009
10#include "fio.h"
Jens Axboebe4ecfd2008-12-08 14:10:52 +010011#include "smalloc.h"
Jens Axboe02bcaa82006-11-24 10:42:00 +010012
13#include "hash.h"
Jens Axboe7d11f872012-12-17 12:03:29 +010014#include "os/os.h"
Jens Axboe02bcaa82006-11-24 10:42:00 +010015
Christian Ehrhardt919e7892014-04-08 17:52:00 +020016#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
Jens Axboec223da82010-03-24 13:23:53 +010017static unsigned long cycles_per_usec;
Jens Axboe71339112012-12-21 22:54:56 +010018static unsigned long inv_cycles_per_usec;
Jens Axboe400531d2014-12-16 22:40:37 -070019static uint64_t max_cycles_for_mult;
Christian Ehrhardtbbc1f062015-02-27 08:20:46 -070020#endif
21#ifdef ARCH_CPU_CLOCK_WRAPS
Jens Axboe1e62c3f2014-12-17 08:51:23 -070022static unsigned long long cycles_start, cycles_wrap;
Jens Axboe09a32402010-08-15 15:01:51 -040023#endif
Jens Axboe4de98eb2013-01-01 10:59:04 +010024int tsc_reliable = 0;
Jens Axboe5d879392012-12-18 09:12:27 +010025
26struct tv_valid {
Jens Axboeba458c22013-01-21 05:38:22 -070027 uint64_t last_cycles;
Jens Axboe02c7e292014-12-16 15:37:25 -070028 int last_tv_valid;
29 int warned;
Jens Axboe5d879392012-12-18 09:12:27 +010030};
Jens Axboed8fc0cd2014-12-17 09:08:01 -070031#ifdef ARCH_HAVE_CPU_CLOCK
Jens Axboe67bf9822013-01-10 11:23:19 +010032#ifdef CONFIG_TLS_THREAD
Jens Axboeb4ea84d2013-04-11 14:20:33 +020033static __thread struct tv_valid static_tv_valid;
Jens Axboe67bf9822013-01-10 11:23:19 +010034#else
Jens Axboe5d879392012-12-18 09:12:27 +010035static pthread_key_t tv_tls_key;
Jens Axboe67bf9822013-01-10 11:23:19 +010036#endif
Jens Axboed8fc0cd2014-12-17 09:08:01 -070037#endif
Jens Axboe02bcaa82006-11-24 10:42:00 +010038
Bruce Cran16de1bf2012-02-20 17:07:32 +000039enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
Jens Axboefa80fea2012-12-09 20:29:00 +010040int fio_clock_source_set = 0;
Jens Axboe10aa1362014-04-01 21:10:36 -060041static enum fio_cs fio_clock_source_inited = CS_INVAL;
Jens Axboec223da82010-03-24 13:23:53 +010042
Jens Axboe02bcaa82006-11-24 10:42:00 +010043#ifdef FIO_DEBUG_TIME
44
45#define HASH_BITS 8
46#define HASH_SIZE (1 << HASH_BITS)
47
Jens Axboe01743ee2008-06-02 12:19:19 +020048static struct flist_head hash[HASH_SIZE];
Jens Axboe02bcaa82006-11-24 10:42:00 +010049static int gtod_inited;
50
51struct gtod_log {
Jens Axboe01743ee2008-06-02 12:19:19 +020052 struct flist_head list;
Jens Axboe02bcaa82006-11-24 10:42:00 +010053 void *caller;
54 unsigned long calls;
55};
56
57static struct gtod_log *find_hash(void *caller)
58{
59 unsigned long h = hash_ptr(caller, HASH_BITS);
Jens Axboe01743ee2008-06-02 12:19:19 +020060 struct flist_head *entry;
Jens Axboe02bcaa82006-11-24 10:42:00 +010061
Jens Axboe01743ee2008-06-02 12:19:19 +020062 flist_for_each(entry, &hash[h]) {
63 struct gtod_log *log = flist_entry(entry, struct gtod_log,
64 list);
Jens Axboe02bcaa82006-11-24 10:42:00 +010065
66 if (log->caller == caller)
67 return log;
68 }
69
70 return NULL;
71}
72
Jens Axboe5194e342014-12-16 23:03:54 -070073static void inc_caller(void *caller)
Jens Axboe02bcaa82006-11-24 10:42:00 +010074{
75 struct gtod_log *log = find_hash(caller);
76
77 if (!log) {
78 unsigned long h;
79
80 log = malloc(sizeof(*log));
Jens Axboe01743ee2008-06-02 12:19:19 +020081 INIT_FLIST_HEAD(&log->list);
Jens Axboe02bcaa82006-11-24 10:42:00 +010082 log->caller = caller;
83 log->calls = 0;
84
85 h = hash_ptr(caller, HASH_BITS);
Jens Axboe01743ee2008-06-02 12:19:19 +020086 flist_add_tail(&log->list, &hash[h]);
Jens Axboe02bcaa82006-11-24 10:42:00 +010087 }
88
Jens Axboe5194e342014-12-16 23:03:54 -070089 log->calls++;
Jens Axboe02bcaa82006-11-24 10:42:00 +010090}
91
92static void gtod_log_caller(void *caller)
93{
Jens Axboe5194e342014-12-16 23:03:54 -070094 if (gtod_inited)
95 inc_caller(caller);
Jens Axboe02bcaa82006-11-24 10:42:00 +010096}
97
98static void fio_exit fio_dump_gtod(void)
99{
100 unsigned long total_calls = 0;
101 int i;
102
103 for (i = 0; i < HASH_SIZE; i++) {
Jens Axboe01743ee2008-06-02 12:19:19 +0200104 struct flist_head *entry;
Jens Axboe02bcaa82006-11-24 10:42:00 +0100105 struct gtod_log *log;
106
Jens Axboe01743ee2008-06-02 12:19:19 +0200107 flist_for_each(entry, &hash[i]) {
108 log = flist_entry(entry, struct gtod_log, list);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100109
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100110 printf("function %p, calls %lu\n", log->caller,
111 log->calls);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100112 total_calls += log->calls;
113 }
114 }
115
116 printf("Total %lu gettimeofday\n", total_calls);
117}
118
119static void fio_init gtod_init(void)
120{
121 int i;
122
123 for (i = 0; i < HASH_SIZE; i++)
Jens Axboe01743ee2008-06-02 12:19:19 +0200124 INIT_FLIST_HEAD(&hash[i]);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100125
126 gtod_inited = 1;
127}
128
129#endif /* FIO_DEBUG_TIME */
130
Jens Axboe67bf9822013-01-10 11:23:19 +0100131#ifdef CONFIG_CLOCK_GETTIME
Jens Axboe9ff1c072012-12-20 15:17:57 +0100132static int fill_clock_gettime(struct timespec *ts)
133{
Jens Axboe67bf9822013-01-10 11:23:19 +0100134#ifdef CONFIG_CLOCK_MONOTONIC
Jens Axboe9ff1c072012-12-20 15:17:57 +0100135 return clock_gettime(CLOCK_MONOTONIC, ts);
136#else
137 return clock_gettime(CLOCK_REALTIME, ts);
138#endif
139}
Jens Axboe1e97cce2006-12-05 11:44:16 +0100140#endif
Jens Axboe67bf9822013-01-10 11:23:19 +0100141
Jens Axboea7d78f22014-12-16 15:13:45 -0700142static void __fio_gettime(struct timeval *tp)
Jens Axboe02bcaa82006-11-24 10:42:00 +0100143{
Jens Axboec223da82010-03-24 13:23:53 +0100144 switch (fio_clock_source) {
Jens Axboe67bf9822013-01-10 11:23:19 +0100145#ifdef CONFIG_GETTIMEOFDAY
Jens Axboec223da82010-03-24 13:23:53 +0100146 case CS_GTOD:
Jens Axboe02bcaa82006-11-24 10:42:00 +0100147 gettimeofday(tp, NULL);
Jens Axboec223da82010-03-24 13:23:53 +0100148 break;
Jens Axboe67bf9822013-01-10 11:23:19 +0100149#endif
150#ifdef CONFIG_CLOCK_GETTIME
Jens Axboec223da82010-03-24 13:23:53 +0100151 case CS_CGETTIME: {
Jens Axboe02bcaa82006-11-24 10:42:00 +0100152 struct timespec ts;
153
Jens Axboe9ff1c072012-12-20 15:17:57 +0100154 if (fill_clock_gettime(&ts) < 0) {
Jens Axboec223da82010-03-24 13:23:53 +0100155 log_err("fio: clock_gettime fails\n");
156 assert(0);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100157 }
158
159 tp->tv_sec = ts.tv_sec;
160 tp->tv_usec = ts.tv_nsec / 1000;
Jens Axboec223da82010-03-24 13:23:53 +0100161 break;
162 }
Jens Axboe67bf9822013-01-10 11:23:19 +0100163#endif
Jens Axboec223da82010-03-24 13:23:53 +0100164#ifdef ARCH_HAVE_CPU_CLOCK
165 case CS_CPUCLOCK: {
Jens Axboeba458c22013-01-21 05:38:22 -0700166 uint64_t usecs, t;
Jens Axboed8fc0cd2014-12-17 09:08:01 -0700167 struct tv_valid *tv;
168
169#ifdef CONFIG_TLS_THREAD
170 tv = &static_tv_valid;
171#else
172 tv = pthread_getspecific(tv_tls_key);
173#endif
Jens Axboec223da82010-03-24 13:23:53 +0100174
175 t = get_cpu_clock();
Christian Ehrhardtbbc1f062015-02-27 08:20:46 -0700176#ifdef ARCH_CPU_CLOCK_WRAPS
Jens Axboe1e62c3f2014-12-17 08:51:23 -0700177 if (t < cycles_start && !cycles_wrap)
178 cycles_wrap = 1;
Jens Axboed8fc0cd2014-12-17 09:08:01 -0700179 else if (cycles_wrap && t >= cycles_start && !tv->warned) {
180 log_err("fio: double CPU clock wrap\n");
181 tv->warned = 1;
Jens Axboe02c7e292014-12-16 15:37:25 -0700182 }
Jens Axboec223da82010-03-24 13:23:53 +0100183
Jens Axboe1e62c3f2014-12-17 08:51:23 -0700184 t -= cycles_start;
Christian Ehrhardtbbc1f062015-02-27 08:20:46 -0700185#endif
Jens Axboea7d78f22014-12-16 15:13:45 -0700186 tv->last_cycles = t;
187 tv->last_tv_valid = 1;
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200188#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
189 usecs = t / ARCH_CPU_CLOCK_CYCLES_PER_USEC;
190#else
Jens Axboe400531d2014-12-16 22:40:37 -0700191 if (t < max_cycles_for_mult)
192 usecs = (t * inv_cycles_per_usec) / 16777216UL;
193 else
194 usecs = t / cycles_per_usec;
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200195#endif
Jens Axboec223da82010-03-24 13:23:53 +0100196 tp->tv_sec = usecs / 1000000;
197 tp->tv_usec = usecs % 1000000;
Jens Axboec223da82010-03-24 13:23:53 +0100198 break;
199 }
200#endif
201 default:
202 log_err("fio: invalid clock source %d\n", fio_clock_source);
203 break;
Jens Axboe02bcaa82006-11-24 10:42:00 +0100204 }
Jens Axboe67bf9822013-01-10 11:23:19 +0100205}
206
207#ifdef FIO_DEBUG_TIME
208void fio_gettime(struct timeval *tp, void *caller)
209#else
210void fio_gettime(struct timeval *tp, void fio_unused *caller)
211#endif
212{
Jens Axboe67bf9822013-01-10 11:23:19 +0100213#ifdef FIO_DEBUG_TIME
214 if (!caller)
215 caller = __builtin_return_address(0);
216
217 gtod_log_caller(caller);
218#endif
Jens Axboe04194192014-12-16 19:43:55 -0700219 if (fio_unlikely(fio_gettime_offload(tp)))
Jens Axboe67bf9822013-01-10 11:23:19 +0100220 return;
Jens Axboe67bf9822013-01-10 11:23:19 +0100221
Jens Axboea7d78f22014-12-16 15:13:45 -0700222 __fio_gettime(tp);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100223}
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100224
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200225#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
Jens Axboec223da82010-03-24 13:23:53 +0100226static unsigned long get_cycles_per_usec(void)
227{
228 struct timeval s, e;
Jens Axboeba458c22013-01-21 05:38:22 -0700229 uint64_t c_s, c_e;
Jens Axboe67bf9822013-01-10 11:23:19 +0100230 enum fio_cs old_cs = fio_clock_source;
Jens Axboec223da82010-03-24 13:23:53 +0100231
Jens Axboe67bf9822013-01-10 11:23:19 +0100232#ifdef CONFIG_CLOCK_GETTIME
233 fio_clock_source = CS_CGETTIME;
234#else
235 fio_clock_source = CS_GTOD;
236#endif
237 __fio_gettime(&s);
Jens Axboe9ff1c072012-12-20 15:17:57 +0100238
Jens Axboec223da82010-03-24 13:23:53 +0100239 c_s = get_cpu_clock();
240 do {
Jens Axboeba458c22013-01-21 05:38:22 -0700241 uint64_t elapsed;
Jens Axboec223da82010-03-24 13:23:53 +0100242
Jens Axboe67bf9822013-01-10 11:23:19 +0100243 __fio_gettime(&e);
Jens Axboe9ff1c072012-12-20 15:17:57 +0100244
Jens Axboec223da82010-03-24 13:23:53 +0100245 elapsed = utime_since(&s, &e);
Jens Axboe486332e2012-12-10 08:07:14 +0100246 if (elapsed >= 1280) {
Jens Axboec223da82010-03-24 13:23:53 +0100247 c_e = get_cpu_clock();
248 break;
249 }
250 } while (1);
251
Jens Axboe67bf9822013-01-10 11:23:19 +0100252 fio_clock_source = old_cs;
Jens Axboec0110002012-12-10 08:29:03 +0100253 return (c_e - c_s + 127) >> 7;
Jens Axboec223da82010-03-24 13:23:53 +0100254}
255
Jens Axboefa80fea2012-12-09 20:29:00 +0100256#define NR_TIME_ITERS 50
257
Jens Axboee2598792013-02-24 21:29:35 +0100258static int calibrate_cpu_clock(void)
Jens Axboec223da82010-03-24 13:23:53 +0100259{
260 double delta, mean, S;
Jens Axboefa529152015-01-06 09:36:25 -0700261 uint64_t minc, maxc, avg, cycles[NR_TIME_ITERS];
Jens Axboec223da82010-03-24 13:23:53 +0100262 int i, samples;
263
Jens Axboec223da82010-03-24 13:23:53 +0100264 cycles[0] = get_cycles_per_usec();
265 S = delta = mean = 0.0;
Jens Axboefa80fea2012-12-09 20:29:00 +0100266 for (i = 0; i < NR_TIME_ITERS; i++) {
Jens Axboec223da82010-03-24 13:23:53 +0100267 cycles[i] = get_cycles_per_usec();
268 delta = cycles[i] - mean;
269 if (delta) {
270 mean += delta / (i + 1.0);
271 S += delta * (cycles[i] - mean);
272 }
273 }
274
Jens Axboee2598792013-02-24 21:29:35 +0100275 /*
276 * The most common platform clock breakage is returning zero
277 * indefinitely. Check for that and return failure.
278 */
279 if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
280 return 1;
281
Jens Axboefa80fea2012-12-09 20:29:00 +0100282 S = sqrt(S / (NR_TIME_ITERS - 1.0));
Jens Axboec223da82010-03-24 13:23:53 +0100283
Jens Axboefa529152015-01-06 09:36:25 -0700284 minc = -1ULL;
285 maxc = samples = avg = 0;
Jens Axboefa80fea2012-12-09 20:29:00 +0100286 for (i = 0; i < NR_TIME_ITERS; i++) {
Jens Axboec223da82010-03-24 13:23:53 +0100287 double this = cycles[i];
288
Jens Axboefa529152015-01-06 09:36:25 -0700289 minc = min(cycles[i], minc);
290 maxc = max(cycles[i], maxc);
291
Bruce Cran03e20d62011-01-02 20:14:54 +0100292 if ((fmax(this, mean) - fmin(this, mean)) > S)
Jens Axboec223da82010-03-24 13:23:53 +0100293 continue;
294 samples++;
295 avg += this;
296 }
297
Jens Axboefa80fea2012-12-09 20:29:00 +0100298 S /= (double) NR_TIME_ITERS;
Jens Axboe89db7272012-12-10 08:29:56 +0100299 mean /= 10.0;
Jens Axboec223da82010-03-24 13:23:53 +0100300
Jens Axboefa80fea2012-12-09 20:29:00 +0100301 for (i = 0; i < NR_TIME_ITERS; i++)
Jens Axboe4b91ee82013-02-25 10:18:33 +0100302 dprint(FD_TIME, "cycles[%d]=%llu\n", i,
303 (unsigned long long) cycles[i] / 10);
Jens Axboec223da82010-03-24 13:23:53 +0100304
Jens Axboed7abad32012-12-10 10:15:59 +0100305 avg /= samples;
Jens Axboeb0ff22d2013-01-01 13:38:18 +0100306 avg = (avg + 5) / 10;
Jens Axboefa529152015-01-06 09:36:25 -0700307 minc /= 10;
308 maxc /= 10;
Jens Axboe4b91ee82013-02-25 10:18:33 +0100309 dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
Jens Axboefa529152015-01-06 09:36:25 -0700310 dprint(FD_TIME, "min=%llu, max=%llu, mean=%f, S=%f\n",
311 (unsigned long long) minc,
312 (unsigned long long) maxc, mean, S);
Jens Axboec223da82010-03-24 13:23:53 +0100313
314 cycles_per_usec = avg;
Jens Axboe71339112012-12-21 22:54:56 +0100315 inv_cycles_per_usec = 16777216UL / cycles_per_usec;
Jens Axboe400531d2014-12-16 22:40:37 -0700316 max_cycles_for_mult = ~0ULL / inv_cycles_per_usec;
Jens Axboe71339112012-12-21 22:54:56 +0100317 dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
Christian Ehrhardtbbc1f062015-02-27 08:20:46 -0700318#ifdef ARCH_CPU_CLOCK_WRAPS
Jens Axboe1e62c3f2014-12-17 08:51:23 -0700319 cycles_start = get_cpu_clock();
320 dprint(FD_TIME, "cycles_start=%llu\n", cycles_start);
Christian Ehrhardtbbc1f062015-02-27 08:20:46 -0700321#endif
Jens Axboee2598792013-02-24 21:29:35 +0100322 return 0;
Jens Axboe09a32402010-08-15 15:01:51 -0400323}
324#else
Jens Axboee2598792013-02-24 21:29:35 +0100325static int calibrate_cpu_clock(void)
Jens Axboe09a32402010-08-15 15:01:51 -0400326{
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200327#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
328 return 0;
329#else
Jens Axboee2598792013-02-24 21:29:35 +0100330 return 1;
Jens Axboe09a32402010-08-15 15:01:51 -0400331#endif
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200332}
333#endif // ARCH_HAVE_CPU_CLOCK
Jens Axboe09a32402010-08-15 15:01:51 -0400334
Jens Axboe67bf9822013-01-10 11:23:19 +0100335#ifndef CONFIG_TLS_THREAD
Jens Axboe5d879392012-12-18 09:12:27 +0100336void fio_local_clock_init(int is_thread)
337{
338 struct tv_valid *t;
339
Jens Axboe572cfb32013-12-06 12:02:10 -0700340 t = calloc(1, sizeof(*t));
Jens Axboea7d78f22014-12-16 15:13:45 -0700341 if (pthread_setspecific(tv_tls_key, t)) {
Jens Axboe5d879392012-12-18 09:12:27 +0100342 log_err("fio: can't set TLS key\n");
Jens Axboea7d78f22014-12-16 15:13:45 -0700343 assert(0);
344 }
Jens Axboe5d879392012-12-18 09:12:27 +0100345}
346
347static void kill_tv_tls_key(void *data)
348{
349 free(data);
350}
Jens Axboe67bf9822013-01-10 11:23:19 +0100351#else
352void fio_local_clock_init(int is_thread)
353{
354}
355#endif
Jens Axboe5d879392012-12-18 09:12:27 +0100356
Jens Axboe09a32402010-08-15 15:01:51 -0400357void fio_clock_init(void)
358{
Jens Axboe01423ea2012-12-14 20:37:06 +0100359 if (fio_clock_source == fio_clock_source_inited)
360 return;
361
Jens Axboe67bf9822013-01-10 11:23:19 +0100362#ifndef CONFIG_TLS_THREAD
Jens Axboe5d879392012-12-18 09:12:27 +0100363 if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
364 log_err("fio: can't create TLS key\n");
Jens Axboe67bf9822013-01-10 11:23:19 +0100365#endif
Jens Axboe5d879392012-12-18 09:12:27 +0100366
Jens Axboe01423ea2012-12-14 20:37:06 +0100367 fio_clock_source_inited = fio_clock_source;
Jens Axboee2598792013-02-24 21:29:35 +0100368
369 if (calibrate_cpu_clock())
370 tsc_reliable = 0;
Jens Axboefa80fea2012-12-09 20:29:00 +0100371
372 /*
373 * If the arch sets tsc_reliable != 0, then it must be good enough
374 * to use as THE clock source. For x86 CPUs, this means the TSC
375 * runs at a constant rate and is synced across CPU cores.
376 */
377 if (tsc_reliable) {
Jens Axboe10564132014-12-23 10:41:34 -0700378 if (!fio_clock_source_set && !fio_monotonic_clocktest(0))
Jens Axboefa80fea2012-12-09 20:29:00 +0100379 fio_clock_source = CS_CPUCLOCK;
380 } else if (fio_clock_source == CS_CPUCLOCK)
381 log_info("fio: clocksource=cpu may not be reliable\n");
Jens Axboec223da82010-03-24 13:23:53 +0100382}
383
Jens Axboe0cbbc392014-09-30 16:04:12 -0600384uint64_t utime_since(const struct timeval *s, const struct timeval *e)
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100385{
Jens Axboe39ab7da2012-11-06 22:10:43 +0100386 long sec, usec;
Jens Axboeaa60bc52013-01-04 13:24:52 +0100387 uint64_t ret;
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100388
Jens Axboe39ab7da2012-11-06 22:10:43 +0100389 sec = e->tv_sec - s->tv_sec;
390 usec = e->tv_usec - s->tv_usec;
391 if (sec > 0 && usec < 0) {
392 sec--;
393 usec += 1000000;
394 }
Jens Axboe783a3eb2012-02-09 10:27:10 +0100395
396 /*
Jens Axboe39ab7da2012-11-06 22:10:43 +0100397 * time warp bug on some kernels?
Jens Axboe783a3eb2012-02-09 10:27:10 +0100398 */
Jens Axboe39ab7da2012-11-06 22:10:43 +0100399 if (sec < 0 || (sec == 0 && usec < 0))
400 return 0;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100401
Jens Axboe39ab7da2012-11-06 22:10:43 +0100402 ret = sec * 1000000ULL + usec;
403
404 return ret;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100405}
406
Jens Axboe0cbbc392014-09-30 16:04:12 -0600407uint64_t utime_since_now(const struct timeval *s)
Jens Axboe783a3eb2012-02-09 10:27:10 +0100408{
Jens Axboe39ab7da2012-11-06 22:10:43 +0100409 struct timeval t;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100410
Jens Axboe39ab7da2012-11-06 22:10:43 +0100411 fio_gettime(&t, NULL);
412 return utime_since(s, &t);
413}
Jens Axboe783a3eb2012-02-09 10:27:10 +0100414
Jens Axboe0cbbc392014-09-30 16:04:12 -0600415uint64_t mtime_since(const struct timeval *s, const struct timeval *e)
Jens Axboe39ab7da2012-11-06 22:10:43 +0100416{
417 long sec, usec, ret;
418
419 sec = e->tv_sec - s->tv_sec;
420 usec = e->tv_usec - s->tv_usec;
421 if (sec > 0 && usec < 0) {
422 sec--;
423 usec += 1000000;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100424 }
425
Jens Axboe39ab7da2012-11-06 22:10:43 +0100426 if (sec < 0 || (sec == 0 && usec < 0))
427 return 0;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100428
Jens Axboe39ab7da2012-11-06 22:10:43 +0100429 sec *= 1000UL;
430 usec /= 1000UL;
431 ret = sec + usec;
432
Jens Axboe783a3eb2012-02-09 10:27:10 +0100433 return ret;
434}
Jens Axboe39ab7da2012-11-06 22:10:43 +0100435
Jens Axboe0cbbc392014-09-30 16:04:12 -0600436uint64_t mtime_since_now(const struct timeval *s)
Jens Axboe39ab7da2012-11-06 22:10:43 +0100437{
438 struct timeval t;
439 void *p = __builtin_return_address(0);
440
441 fio_gettime(&t, p);
442 return mtime_since(s, &t);
443}
444
Jens Axboe0cbbc392014-09-30 16:04:12 -0600445uint64_t time_since_now(const struct timeval *s)
Jens Axboe39ab7da2012-11-06 22:10:43 +0100446{
447 return mtime_since_now(s) / 1000;
448}
Jens Axboe7d11f872012-12-17 12:03:29 +0100449
Jens Axboe67bf9822013-01-10 11:23:19 +0100450#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
451 defined(CONFIG_SFAA)
Jens Axboe7d11f872012-12-17 12:03:29 +0100452
Jens Axboe10564132014-12-23 10:41:34 -0700453#define CLOCK_ENTRIES_DEBUG 100000
454#define CLOCK_ENTRIES_TEST 10000
Jens Axboe7d11f872012-12-17 12:03:29 +0100455
456struct clock_entry {
Jens Axboe58002f92013-02-25 10:23:58 +0100457 uint32_t seq;
458 uint32_t cpu;
Jens Axboeba458c22013-01-21 05:38:22 -0700459 uint64_t tsc;
Jens Axboe7d11f872012-12-17 12:03:29 +0100460};
461
462struct clock_thread {
463 pthread_t thread;
464 int cpu;
Jens Axboe10564132014-12-23 10:41:34 -0700465 int debug;
Jens Axboe7d11f872012-12-17 12:03:29 +0100466 pthread_mutex_t lock;
467 pthread_mutex_t started;
Jens Axboe10564132014-12-23 10:41:34 -0700468 unsigned long nr_entries;
Jens Axboe58002f92013-02-25 10:23:58 +0100469 uint32_t *seq;
Jens Axboe7d11f872012-12-17 12:03:29 +0100470 struct clock_entry *entries;
471};
472
Jens Axboe58002f92013-02-25 10:23:58 +0100473static inline uint32_t atomic32_inc_return(uint32_t *seq)
Jens Axboe7d11f872012-12-17 12:03:29 +0100474{
475 return 1 + __sync_fetch_and_add(seq, 1);
476}
477
478static void *clock_thread_fn(void *data)
479{
480 struct clock_thread *t = data;
481 struct clock_entry *c;
482 os_cpu_mask_t cpu_mask;
Jens Axboe58002f92013-02-25 10:23:58 +0100483 uint32_t last_seq;
Jens Axboe7d11f872012-12-17 12:03:29 +0100484 int i;
485
Jens Axboe1af4be12015-01-28 14:42:10 -0700486 if (fio_cpuset_init(&cpu_mask)) {
Jens Axboeaa28ce02015-01-29 08:32:28 -0800487 int __err = errno;
Jens Axboe1af4be12015-01-28 14:42:10 -0700488
489 log_err("clock cpuset init failed: %s\n", strerror(__err));
490 goto err_out;
491 }
492
Jens Axboe7d11f872012-12-17 12:03:29 +0100493 fio_cpu_set(&cpu_mask, t->cpu);
494
495 if (fio_setaffinity(gettid(), cpu_mask) == -1) {
Jens Axboe4c64ff72015-01-28 14:34:10 -0700496 int __err = errno;
497
498 log_err("clock setaffinity failed: %s\n", strerror(__err));
Jens Axboe1af4be12015-01-28 14:42:10 -0700499 goto err;
Jens Axboe7d11f872012-12-17 12:03:29 +0100500 }
501
Jens Axboe7d11f872012-12-17 12:03:29 +0100502 pthread_mutex_lock(&t->lock);
Jens Axboeb9b34982012-12-17 14:23:47 +0100503 pthread_mutex_unlock(&t->started);
Jens Axboe7d11f872012-12-17 12:03:29 +0100504
Jens Axboe58002f92013-02-25 10:23:58 +0100505 last_seq = 0;
Jens Axboe7d11f872012-12-17 12:03:29 +0100506 c = &t->entries[0];
Jens Axboe10564132014-12-23 10:41:34 -0700507 for (i = 0; i < t->nr_entries; i++, c++) {
Jens Axboe58002f92013-02-25 10:23:58 +0100508 uint32_t seq;
509 uint64_t tsc;
Jens Axboe7d11f872012-12-17 12:03:29 +0100510
511 c->cpu = t->cpu;
512 do {
Jens Axboe58002f92013-02-25 10:23:58 +0100513 seq = atomic32_inc_return(t->seq);
514 if (seq < last_seq)
515 break;
Jens Axboe7d11f872012-12-17 12:03:29 +0100516 tsc = get_cpu_clock();
517 } while (seq != *t->seq);
518
519 c->seq = seq;
520 c->tsc = tsc;
521 }
522
Jens Axboe10564132014-12-23 10:41:34 -0700523 if (t->debug) {
524 unsigned long long clocks;
525
526 clocks = t->entries[i - 1].tsc - t->entries[0].tsc;
527 log_info("cs: cpu%3d: %llu clocks seen\n", t->cpu, clocks);
528 }
Jens Axboe58002f92013-02-25 10:23:58 +0100529
Jens Axboee2598792013-02-24 21:29:35 +0100530 /*
531 * The most common platform clock breakage is returning zero
532 * indefinitely. Check for that and return failure.
533 */
Jens Axboe58002f92013-02-25 10:23:58 +0100534 if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
Jens Axboe1af4be12015-01-28 14:42:10 -0700535 goto err;
Jens Axboee2598792013-02-24 21:29:35 +0100536
Jens Axboe90881c82015-01-29 09:23:39 -0800537 fio_cpuset_exit(&cpu_mask);
Jens Axboe7d11f872012-12-17 12:03:29 +0100538 return NULL;
Jens Axboe1af4be12015-01-28 14:42:10 -0700539err:
540 fio_cpuset_exit(&cpu_mask);
541err_out:
542 return (void *) 1;
Jens Axboe7d11f872012-12-17 12:03:29 +0100543}
544
545static int clock_cmp(const void *p1, const void *p2)
546{
547 const struct clock_entry *c1 = p1;
548 const struct clock_entry *c2 = p2;
549
Jens Axboeb9b34982012-12-17 14:23:47 +0100550 if (c1->seq == c2->seq)
551 log_err("cs: bug in atomic sequence!\n");
552
Jens Axboe7d11f872012-12-17 12:03:29 +0100553 return c1->seq - c2->seq;
554}
555
Jens Axboe10564132014-12-23 10:41:34 -0700556int fio_monotonic_clocktest(int debug)
Jens Axboe7d11f872012-12-17 12:03:29 +0100557{
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700558 struct clock_thread *cthreads;
Jens Axboe7d11f872012-12-17 12:03:29 +0100559 unsigned int nr_cpus = cpus_online();
560 struct clock_entry *entries;
Jens Axboe10564132014-12-23 10:41:34 -0700561 unsigned long nr_entries, tentries, failed = 0;
Bruce Cran80da8a82013-02-21 14:16:17 +0100562 struct clock_entry *prev, *this;
Jens Axboe58002f92013-02-25 10:23:58 +0100563 uint32_t seq = 0;
Jens Axboecaa3eb12014-04-14 09:05:46 -0600564 unsigned int i;
Jens Axboe7d11f872012-12-17 12:03:29 +0100565
Jens Axboe10564132014-12-23 10:41:34 -0700566 if (debug) {
567 log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
Jens Axboed5e3f5d2013-01-18 20:13:45 +0100568
Jens Axboefc3f3e42014-09-24 09:54:24 -0600569#ifdef FIO_INC_DEBUG
Jens Axboe10564132014-12-23 10:41:34 -0700570 fio_debug |= 1U << FD_TIME;
Jens Axboefc3f3e42014-09-24 09:54:24 -0600571#endif
Jens Axboe10564132014-12-23 10:41:34 -0700572 nr_entries = CLOCK_ENTRIES_DEBUG;
573 } else
574 nr_entries = CLOCK_ENTRIES_TEST;
575
Jens Axboe4f1d43c2012-12-17 14:29:28 +0100576 calibrate_cpu_clock();
Jens Axboe10564132014-12-23 10:41:34 -0700577
578 if (debug) {
Jens Axboefc3f3e42014-09-24 09:54:24 -0600579#ifdef FIO_INC_DEBUG
Jens Axboe10564132014-12-23 10:41:34 -0700580 fio_debug &= ~(1U << FD_TIME);
Jens Axboefc3f3e42014-09-24 09:54:24 -0600581#endif
Jens Axboe10564132014-12-23 10:41:34 -0700582 }
Jens Axboe4f1d43c2012-12-17 14:29:28 +0100583
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700584 cthreads = malloc(nr_cpus * sizeof(struct clock_thread));
Jens Axboe10564132014-12-23 10:41:34 -0700585 tentries = nr_entries * nr_cpus;
Jens Axboe7d11f872012-12-17 12:03:29 +0100586 entries = malloc(tentries * sizeof(struct clock_entry));
587
Jens Axboe10564132014-12-23 10:41:34 -0700588 if (debug)
589 log_info("cs: Testing %u CPUs\n", nr_cpus);
Jens Axboe7d11f872012-12-17 12:03:29 +0100590
591 for (i = 0; i < nr_cpus; i++) {
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700592 struct clock_thread *t = &cthreads[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100593
594 t->cpu = i;
Jens Axboe10564132014-12-23 10:41:34 -0700595 t->debug = debug;
Jens Axboe7d11f872012-12-17 12:03:29 +0100596 t->seq = &seq;
Jens Axboe10564132014-12-23 10:41:34 -0700597 t->nr_entries = nr_entries;
598 t->entries = &entries[i * nr_entries];
Jens Axboe7d11f872012-12-17 12:03:29 +0100599 pthread_mutex_init(&t->lock, NULL);
600 pthread_mutex_init(&t->started, NULL);
601 pthread_mutex_lock(&t->lock);
Jens Axboe6b0110c2014-04-14 12:14:17 -0600602 if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
603 failed++;
604 nr_cpus = i;
605 break;
606 }
Jens Axboe7d11f872012-12-17 12:03:29 +0100607 }
608
609 for (i = 0; i < nr_cpus; i++) {
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700610 struct clock_thread *t = &cthreads[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100611
612 pthread_mutex_lock(&t->started);
613 }
614
615 for (i = 0; i < nr_cpus; i++) {
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700616 struct clock_thread *t = &cthreads[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100617
618 pthread_mutex_unlock(&t->lock);
619 }
620
Jens Axboe814917b2014-04-14 12:20:04 -0600621 for (i = 0; i < nr_cpus; i++) {
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700622 struct clock_thread *t = &cthreads[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100623 void *ret;
624
625 pthread_join(t->thread, &ret);
626 if (ret)
627 failed++;
628 }
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700629 free(cthreads);
Jens Axboe7d11f872012-12-17 12:03:29 +0100630
631 if (failed) {
Jens Axboe10564132014-12-23 10:41:34 -0700632 if (debug)
633 log_err("Clocksource test: %lu threads failed\n", failed);
Jens Axboe7d11f872012-12-17 12:03:29 +0100634 goto err;
635 }
636
637 qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
638
639 for (failed = i = 0; i < tentries; i++) {
Bruce Cran80da8a82013-02-21 14:16:17 +0100640 this = &entries[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100641
642 if (!i) {
643 prev = this;
644 continue;
645 }
646
647 if (prev->tsc > this->tsc) {
648 uint64_t diff = prev->tsc - this->tsc;
649
Jens Axboe10564132014-12-23 10:41:34 -0700650 if (!debug) {
651 failed++;
652 break;
653 }
654
Jens Axboe4e0a8fa2013-04-15 11:40:57 +0200655 log_info("cs: CPU clock mismatch (diff=%llu):\n",
656 (unsigned long long) diff);
657 log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq);
658 log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq);
Jens Axboe7d11f872012-12-17 12:03:29 +0100659 failed++;
660 }
661
662 prev = this;
663 }
664
Jens Axboe10564132014-12-23 10:41:34 -0700665 if (debug) {
666 if (failed)
667 log_info("cs: Failed: %lu\n", failed);
668 else
669 log_info("cs: Pass!\n");
670 }
Jens Axboe7d11f872012-12-17 12:03:29 +0100671err:
672 free(entries);
673 return !!failed;
674}
675
676#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
677
Jens Axboe10564132014-12-23 10:41:34 -0700678int fio_monotonic_clocktest(int debug)
Jens Axboe7d11f872012-12-17 12:03:29 +0100679{
Jens Axboe10564132014-12-23 10:41:34 -0700680 if (debug)
681 log_info("cs: current platform does not support CPU clocks\n");
682 return 1;
Jens Axboe7d11f872012-12-17 12:03:29 +0100683}
684
685#endif