blob: 6863ce3f97cef140e967990b9b8a7dc0bf3b0126 [file] [log] [blame]
Jens Axboe02bcaa82006-11-24 10:42:00 +01001/*
Jens Axboef5cc0242006-11-24 10:47:40 +01002 * Clock functions
Jens Axboe02bcaa82006-11-24 10:42:00 +01003 */
Jens Axboef5cc0242006-11-24 10:47:40 +01004
Jens Axboe02bcaa82006-11-24 10:42:00 +01005#include <unistd.h>
Jens Axboec223da82010-03-24 13:23:53 +01006#include <math.h>
Jens Axboe02bcaa82006-11-24 10:42:00 +01007#include <sys/time.h>
Bruce Cran03e20d62011-01-02 20:14:54 +01008#include <time.h>
Jens Axboe02bcaa82006-11-24 10:42:00 +01009
10#include "fio.h"
Jens Axboebe4ecfd2008-12-08 14:10:52 +010011#include "smalloc.h"
Jens Axboe02bcaa82006-11-24 10:42:00 +010012
13#include "hash.h"
Jens Axboe7d11f872012-12-17 12:03:29 +010014#include "os/os.h"
Jens Axboe02bcaa82006-11-24 10:42:00 +010015
Christian Ehrhardt919e7892014-04-08 17:52:00 +020016#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
Jens Axboec223da82010-03-24 13:23:53 +010017static unsigned long cycles_per_usec;
Jens Axboe71339112012-12-21 22:54:56 +010018static unsigned long inv_cycles_per_usec;
Jens Axboe400531d2014-12-16 22:40:37 -070019static uint64_t max_cycles_for_mult;
Jens Axboe1e62c3f2014-12-17 08:51:23 -070020static unsigned long long cycles_start, cycles_wrap;
Jens Axboe09a32402010-08-15 15:01:51 -040021#endif
Jens Axboe4de98eb2013-01-01 10:59:04 +010022int tsc_reliable = 0;
Jens Axboe5d879392012-12-18 09:12:27 +010023
24struct tv_valid {
Jens Axboeba458c22013-01-21 05:38:22 -070025 uint64_t last_cycles;
Jens Axboe02c7e292014-12-16 15:37:25 -070026 int last_tv_valid;
27 int warned;
Jens Axboe5d879392012-12-18 09:12:27 +010028};
Jens Axboed8fc0cd2014-12-17 09:08:01 -070029#ifdef ARCH_HAVE_CPU_CLOCK
Jens Axboe67bf9822013-01-10 11:23:19 +010030#ifdef CONFIG_TLS_THREAD
Jens Axboeb4ea84d2013-04-11 14:20:33 +020031static __thread struct tv_valid static_tv_valid;
Jens Axboe67bf9822013-01-10 11:23:19 +010032#else
Jens Axboe5d879392012-12-18 09:12:27 +010033static pthread_key_t tv_tls_key;
Jens Axboe67bf9822013-01-10 11:23:19 +010034#endif
Jens Axboed8fc0cd2014-12-17 09:08:01 -070035#endif
Jens Axboe02bcaa82006-11-24 10:42:00 +010036
Bruce Cran16de1bf2012-02-20 17:07:32 +000037enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
Jens Axboefa80fea2012-12-09 20:29:00 +010038int fio_clock_source_set = 0;
Jens Axboe10aa1362014-04-01 21:10:36 -060039static enum fio_cs fio_clock_source_inited = CS_INVAL;
Jens Axboec223da82010-03-24 13:23:53 +010040
Jens Axboe02bcaa82006-11-24 10:42:00 +010041#ifdef FIO_DEBUG_TIME
42
43#define HASH_BITS 8
44#define HASH_SIZE (1 << HASH_BITS)
45
Jens Axboe01743ee2008-06-02 12:19:19 +020046static struct flist_head hash[HASH_SIZE];
Jens Axboe02bcaa82006-11-24 10:42:00 +010047static int gtod_inited;
48
49struct gtod_log {
Jens Axboe01743ee2008-06-02 12:19:19 +020050 struct flist_head list;
Jens Axboe02bcaa82006-11-24 10:42:00 +010051 void *caller;
52 unsigned long calls;
53};
54
55static struct gtod_log *find_hash(void *caller)
56{
57 unsigned long h = hash_ptr(caller, HASH_BITS);
Jens Axboe01743ee2008-06-02 12:19:19 +020058 struct flist_head *entry;
Jens Axboe02bcaa82006-11-24 10:42:00 +010059
Jens Axboe01743ee2008-06-02 12:19:19 +020060 flist_for_each(entry, &hash[h]) {
61 struct gtod_log *log = flist_entry(entry, struct gtod_log,
62 list);
Jens Axboe02bcaa82006-11-24 10:42:00 +010063
64 if (log->caller == caller)
65 return log;
66 }
67
68 return NULL;
69}
70
Jens Axboe5194e342014-12-16 23:03:54 -070071static void inc_caller(void *caller)
Jens Axboe02bcaa82006-11-24 10:42:00 +010072{
73 struct gtod_log *log = find_hash(caller);
74
75 if (!log) {
76 unsigned long h;
77
78 log = malloc(sizeof(*log));
Jens Axboe01743ee2008-06-02 12:19:19 +020079 INIT_FLIST_HEAD(&log->list);
Jens Axboe02bcaa82006-11-24 10:42:00 +010080 log->caller = caller;
81 log->calls = 0;
82
83 h = hash_ptr(caller, HASH_BITS);
Jens Axboe01743ee2008-06-02 12:19:19 +020084 flist_add_tail(&log->list, &hash[h]);
Jens Axboe02bcaa82006-11-24 10:42:00 +010085 }
86
Jens Axboe5194e342014-12-16 23:03:54 -070087 log->calls++;
Jens Axboe02bcaa82006-11-24 10:42:00 +010088}
89
90static void gtod_log_caller(void *caller)
91{
Jens Axboe5194e342014-12-16 23:03:54 -070092 if (gtod_inited)
93 inc_caller(caller);
Jens Axboe02bcaa82006-11-24 10:42:00 +010094}
95
96static void fio_exit fio_dump_gtod(void)
97{
98 unsigned long total_calls = 0;
99 int i;
100
101 for (i = 0; i < HASH_SIZE; i++) {
Jens Axboe01743ee2008-06-02 12:19:19 +0200102 struct flist_head *entry;
Jens Axboe02bcaa82006-11-24 10:42:00 +0100103 struct gtod_log *log;
104
Jens Axboe01743ee2008-06-02 12:19:19 +0200105 flist_for_each(entry, &hash[i]) {
106 log = flist_entry(entry, struct gtod_log, list);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100107
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100108 printf("function %p, calls %lu\n", log->caller,
109 log->calls);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100110 total_calls += log->calls;
111 }
112 }
113
114 printf("Total %lu gettimeofday\n", total_calls);
115}
116
117static void fio_init gtod_init(void)
118{
119 int i;
120
121 for (i = 0; i < HASH_SIZE; i++)
Jens Axboe01743ee2008-06-02 12:19:19 +0200122 INIT_FLIST_HEAD(&hash[i]);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100123
124 gtod_inited = 1;
125}
126
127#endif /* FIO_DEBUG_TIME */
128
Jens Axboe67bf9822013-01-10 11:23:19 +0100129#ifdef CONFIG_CLOCK_GETTIME
Jens Axboe9ff1c072012-12-20 15:17:57 +0100130static int fill_clock_gettime(struct timespec *ts)
131{
Jens Axboe67bf9822013-01-10 11:23:19 +0100132#ifdef CONFIG_CLOCK_MONOTONIC
Jens Axboe9ff1c072012-12-20 15:17:57 +0100133 return clock_gettime(CLOCK_MONOTONIC, ts);
134#else
135 return clock_gettime(CLOCK_REALTIME, ts);
136#endif
137}
Jens Axboe1e97cce2006-12-05 11:44:16 +0100138#endif
Jens Axboe67bf9822013-01-10 11:23:19 +0100139
Jens Axboea7d78f22014-12-16 15:13:45 -0700140static void __fio_gettime(struct timeval *tp)
Jens Axboe02bcaa82006-11-24 10:42:00 +0100141{
Jens Axboec223da82010-03-24 13:23:53 +0100142 switch (fio_clock_source) {
Jens Axboe67bf9822013-01-10 11:23:19 +0100143#ifdef CONFIG_GETTIMEOFDAY
Jens Axboec223da82010-03-24 13:23:53 +0100144 case CS_GTOD:
Jens Axboe02bcaa82006-11-24 10:42:00 +0100145 gettimeofday(tp, NULL);
Jens Axboec223da82010-03-24 13:23:53 +0100146 break;
Jens Axboe67bf9822013-01-10 11:23:19 +0100147#endif
148#ifdef CONFIG_CLOCK_GETTIME
Jens Axboec223da82010-03-24 13:23:53 +0100149 case CS_CGETTIME: {
Jens Axboe02bcaa82006-11-24 10:42:00 +0100150 struct timespec ts;
151
Jens Axboe9ff1c072012-12-20 15:17:57 +0100152 if (fill_clock_gettime(&ts) < 0) {
Jens Axboec223da82010-03-24 13:23:53 +0100153 log_err("fio: clock_gettime fails\n");
154 assert(0);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100155 }
156
157 tp->tv_sec = ts.tv_sec;
158 tp->tv_usec = ts.tv_nsec / 1000;
Jens Axboec223da82010-03-24 13:23:53 +0100159 break;
160 }
Jens Axboe67bf9822013-01-10 11:23:19 +0100161#endif
Jens Axboec223da82010-03-24 13:23:53 +0100162#ifdef ARCH_HAVE_CPU_CLOCK
163 case CS_CPUCLOCK: {
Jens Axboeba458c22013-01-21 05:38:22 -0700164 uint64_t usecs, t;
Jens Axboed8fc0cd2014-12-17 09:08:01 -0700165 struct tv_valid *tv;
166
167#ifdef CONFIG_TLS_THREAD
168 tv = &static_tv_valid;
169#else
170 tv = pthread_getspecific(tv_tls_key);
171#endif
Jens Axboec223da82010-03-24 13:23:53 +0100172
173 t = get_cpu_clock();
Jens Axboe1e62c3f2014-12-17 08:51:23 -0700174 if (t < cycles_start && !cycles_wrap)
175 cycles_wrap = 1;
Jens Axboed8fc0cd2014-12-17 09:08:01 -0700176 else if (cycles_wrap && t >= cycles_start && !tv->warned) {
177 log_err("fio: double CPU clock wrap\n");
178 tv->warned = 1;
Jens Axboe02c7e292014-12-16 15:37:25 -0700179 }
Jens Axboec223da82010-03-24 13:23:53 +0100180
Jens Axboe1e62c3f2014-12-17 08:51:23 -0700181 t -= cycles_start;
Jens Axboea7d78f22014-12-16 15:13:45 -0700182 tv->last_cycles = t;
183 tv->last_tv_valid = 1;
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200184#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
185 usecs = t / ARCH_CPU_CLOCK_CYCLES_PER_USEC;
186#else
Jens Axboe400531d2014-12-16 22:40:37 -0700187 if (t < max_cycles_for_mult)
188 usecs = (t * inv_cycles_per_usec) / 16777216UL;
189 else
190 usecs = t / cycles_per_usec;
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200191#endif
Jens Axboec223da82010-03-24 13:23:53 +0100192 tp->tv_sec = usecs / 1000000;
193 tp->tv_usec = usecs % 1000000;
Jens Axboec223da82010-03-24 13:23:53 +0100194 break;
195 }
196#endif
197 default:
198 log_err("fio: invalid clock source %d\n", fio_clock_source);
199 break;
Jens Axboe02bcaa82006-11-24 10:42:00 +0100200 }
Jens Axboe67bf9822013-01-10 11:23:19 +0100201}
202
203#ifdef FIO_DEBUG_TIME
204void fio_gettime(struct timeval *tp, void *caller)
205#else
206void fio_gettime(struct timeval *tp, void fio_unused *caller)
207#endif
208{
Jens Axboe67bf9822013-01-10 11:23:19 +0100209#ifdef FIO_DEBUG_TIME
210 if (!caller)
211 caller = __builtin_return_address(0);
212
213 gtod_log_caller(caller);
214#endif
Jens Axboe04194192014-12-16 19:43:55 -0700215 if (fio_unlikely(fio_gettime_offload(tp)))
Jens Axboe67bf9822013-01-10 11:23:19 +0100216 return;
Jens Axboe67bf9822013-01-10 11:23:19 +0100217
Jens Axboea7d78f22014-12-16 15:13:45 -0700218 __fio_gettime(tp);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100219}
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100220
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200221#if defined(ARCH_HAVE_CPU_CLOCK) && !defined(ARCH_CPU_CLOCK_CYCLES_PER_USEC)
Jens Axboec223da82010-03-24 13:23:53 +0100222static unsigned long get_cycles_per_usec(void)
223{
224 struct timeval s, e;
Jens Axboeba458c22013-01-21 05:38:22 -0700225 uint64_t c_s, c_e;
Jens Axboe67bf9822013-01-10 11:23:19 +0100226 enum fio_cs old_cs = fio_clock_source;
Jens Axboec223da82010-03-24 13:23:53 +0100227
Jens Axboe67bf9822013-01-10 11:23:19 +0100228#ifdef CONFIG_CLOCK_GETTIME
229 fio_clock_source = CS_CGETTIME;
230#else
231 fio_clock_source = CS_GTOD;
232#endif
233 __fio_gettime(&s);
Jens Axboe9ff1c072012-12-20 15:17:57 +0100234
Jens Axboec223da82010-03-24 13:23:53 +0100235 c_s = get_cpu_clock();
236 do {
Jens Axboeba458c22013-01-21 05:38:22 -0700237 uint64_t elapsed;
Jens Axboec223da82010-03-24 13:23:53 +0100238
Jens Axboe67bf9822013-01-10 11:23:19 +0100239 __fio_gettime(&e);
Jens Axboe9ff1c072012-12-20 15:17:57 +0100240
Jens Axboec223da82010-03-24 13:23:53 +0100241 elapsed = utime_since(&s, &e);
Jens Axboe486332e2012-12-10 08:07:14 +0100242 if (elapsed >= 1280) {
Jens Axboec223da82010-03-24 13:23:53 +0100243 c_e = get_cpu_clock();
244 break;
245 }
246 } while (1);
247
Jens Axboe67bf9822013-01-10 11:23:19 +0100248 fio_clock_source = old_cs;
Jens Axboec0110002012-12-10 08:29:03 +0100249 return (c_e - c_s + 127) >> 7;
Jens Axboec223da82010-03-24 13:23:53 +0100250}
251
Jens Axboefa80fea2012-12-09 20:29:00 +0100252#define NR_TIME_ITERS 50
253
Jens Axboee2598792013-02-24 21:29:35 +0100254static int calibrate_cpu_clock(void)
Jens Axboec223da82010-03-24 13:23:53 +0100255{
256 double delta, mean, S;
Jens Axboefa529152015-01-06 09:36:25 -0700257 uint64_t minc, maxc, avg, cycles[NR_TIME_ITERS];
Jens Axboec223da82010-03-24 13:23:53 +0100258 int i, samples;
259
Jens Axboec223da82010-03-24 13:23:53 +0100260 cycles[0] = get_cycles_per_usec();
261 S = delta = mean = 0.0;
Jens Axboefa80fea2012-12-09 20:29:00 +0100262 for (i = 0; i < NR_TIME_ITERS; i++) {
Jens Axboec223da82010-03-24 13:23:53 +0100263 cycles[i] = get_cycles_per_usec();
264 delta = cycles[i] - mean;
265 if (delta) {
266 mean += delta / (i + 1.0);
267 S += delta * (cycles[i] - mean);
268 }
269 }
270
Jens Axboee2598792013-02-24 21:29:35 +0100271 /*
272 * The most common platform clock breakage is returning zero
273 * indefinitely. Check for that and return failure.
274 */
275 if (!cycles[0] && !cycles[NR_TIME_ITERS - 1])
276 return 1;
277
Jens Axboefa80fea2012-12-09 20:29:00 +0100278 S = sqrt(S / (NR_TIME_ITERS - 1.0));
Jens Axboec223da82010-03-24 13:23:53 +0100279
Jens Axboefa529152015-01-06 09:36:25 -0700280 minc = -1ULL;
281 maxc = samples = avg = 0;
Jens Axboefa80fea2012-12-09 20:29:00 +0100282 for (i = 0; i < NR_TIME_ITERS; i++) {
Jens Axboec223da82010-03-24 13:23:53 +0100283 double this = cycles[i];
284
Jens Axboefa529152015-01-06 09:36:25 -0700285 minc = min(cycles[i], minc);
286 maxc = max(cycles[i], maxc);
287
Bruce Cran03e20d62011-01-02 20:14:54 +0100288 if ((fmax(this, mean) - fmin(this, mean)) > S)
Jens Axboec223da82010-03-24 13:23:53 +0100289 continue;
290 samples++;
291 avg += this;
292 }
293
Jens Axboefa80fea2012-12-09 20:29:00 +0100294 S /= (double) NR_TIME_ITERS;
Jens Axboe89db7272012-12-10 08:29:56 +0100295 mean /= 10.0;
Jens Axboec223da82010-03-24 13:23:53 +0100296
Jens Axboefa80fea2012-12-09 20:29:00 +0100297 for (i = 0; i < NR_TIME_ITERS; i++)
Jens Axboe4b91ee82013-02-25 10:18:33 +0100298 dprint(FD_TIME, "cycles[%d]=%llu\n", i,
299 (unsigned long long) cycles[i] / 10);
Jens Axboec223da82010-03-24 13:23:53 +0100300
Jens Axboed7abad32012-12-10 10:15:59 +0100301 avg /= samples;
Jens Axboeb0ff22d2013-01-01 13:38:18 +0100302 avg = (avg + 5) / 10;
Jens Axboefa529152015-01-06 09:36:25 -0700303 minc /= 10;
304 maxc /= 10;
Jens Axboe4b91ee82013-02-25 10:18:33 +0100305 dprint(FD_TIME, "avg: %llu\n", (unsigned long long) avg);
Jens Axboefa529152015-01-06 09:36:25 -0700306 dprint(FD_TIME, "min=%llu, max=%llu, mean=%f, S=%f\n",
307 (unsigned long long) minc,
308 (unsigned long long) maxc, mean, S);
Jens Axboec223da82010-03-24 13:23:53 +0100309
310 cycles_per_usec = avg;
Jens Axboe71339112012-12-21 22:54:56 +0100311 inv_cycles_per_usec = 16777216UL / cycles_per_usec;
Jens Axboe400531d2014-12-16 22:40:37 -0700312 max_cycles_for_mult = ~0ULL / inv_cycles_per_usec;
Jens Axboe71339112012-12-21 22:54:56 +0100313 dprint(FD_TIME, "inv_cycles_per_usec=%lu\n", inv_cycles_per_usec);
Jens Axboe1e62c3f2014-12-17 08:51:23 -0700314 cycles_start = get_cpu_clock();
315 dprint(FD_TIME, "cycles_start=%llu\n", cycles_start);
Jens Axboee2598792013-02-24 21:29:35 +0100316 return 0;
Jens Axboe09a32402010-08-15 15:01:51 -0400317}
318#else
Jens Axboee2598792013-02-24 21:29:35 +0100319static int calibrate_cpu_clock(void)
Jens Axboe09a32402010-08-15 15:01:51 -0400320{
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200321#ifdef ARCH_CPU_CLOCK_CYCLES_PER_USEC
322 return 0;
323#else
Jens Axboee2598792013-02-24 21:29:35 +0100324 return 1;
Jens Axboe09a32402010-08-15 15:01:51 -0400325#endif
Christian Ehrhardt919e7892014-04-08 17:52:00 +0200326}
327#endif // ARCH_HAVE_CPU_CLOCK
Jens Axboe09a32402010-08-15 15:01:51 -0400328
Jens Axboe67bf9822013-01-10 11:23:19 +0100329#ifndef CONFIG_TLS_THREAD
Jens Axboe5d879392012-12-18 09:12:27 +0100330void fio_local_clock_init(int is_thread)
331{
332 struct tv_valid *t;
333
Jens Axboe572cfb32013-12-06 12:02:10 -0700334 t = calloc(1, sizeof(*t));
Jens Axboea7d78f22014-12-16 15:13:45 -0700335 if (pthread_setspecific(tv_tls_key, t)) {
Jens Axboe5d879392012-12-18 09:12:27 +0100336 log_err("fio: can't set TLS key\n");
Jens Axboea7d78f22014-12-16 15:13:45 -0700337 assert(0);
338 }
Jens Axboe5d879392012-12-18 09:12:27 +0100339}
340
341static void kill_tv_tls_key(void *data)
342{
343 free(data);
344}
Jens Axboe67bf9822013-01-10 11:23:19 +0100345#else
346void fio_local_clock_init(int is_thread)
347{
348}
349#endif
Jens Axboe5d879392012-12-18 09:12:27 +0100350
Jens Axboe09a32402010-08-15 15:01:51 -0400351void fio_clock_init(void)
352{
Jens Axboe01423ea2012-12-14 20:37:06 +0100353 if (fio_clock_source == fio_clock_source_inited)
354 return;
355
Jens Axboe67bf9822013-01-10 11:23:19 +0100356#ifndef CONFIG_TLS_THREAD
Jens Axboe5d879392012-12-18 09:12:27 +0100357 if (pthread_key_create(&tv_tls_key, kill_tv_tls_key))
358 log_err("fio: can't create TLS key\n");
Jens Axboe67bf9822013-01-10 11:23:19 +0100359#endif
Jens Axboe5d879392012-12-18 09:12:27 +0100360
Jens Axboe01423ea2012-12-14 20:37:06 +0100361 fio_clock_source_inited = fio_clock_source;
Jens Axboee2598792013-02-24 21:29:35 +0100362
363 if (calibrate_cpu_clock())
364 tsc_reliable = 0;
Jens Axboefa80fea2012-12-09 20:29:00 +0100365
366 /*
367 * If the arch sets tsc_reliable != 0, then it must be good enough
368 * to use as THE clock source. For x86 CPUs, this means the TSC
369 * runs at a constant rate and is synced across CPU cores.
370 */
371 if (tsc_reliable) {
Jens Axboe10564132014-12-23 10:41:34 -0700372 if (!fio_clock_source_set && !fio_monotonic_clocktest(0))
Jens Axboefa80fea2012-12-09 20:29:00 +0100373 fio_clock_source = CS_CPUCLOCK;
374 } else if (fio_clock_source == CS_CPUCLOCK)
375 log_info("fio: clocksource=cpu may not be reliable\n");
Jens Axboec223da82010-03-24 13:23:53 +0100376}
377
Jens Axboe0cbbc392014-09-30 16:04:12 -0600378uint64_t utime_since(const struct timeval *s, const struct timeval *e)
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100379{
Jens Axboe39ab7da2012-11-06 22:10:43 +0100380 long sec, usec;
Jens Axboeaa60bc52013-01-04 13:24:52 +0100381 uint64_t ret;
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100382
Jens Axboe39ab7da2012-11-06 22:10:43 +0100383 sec = e->tv_sec - s->tv_sec;
384 usec = e->tv_usec - s->tv_usec;
385 if (sec > 0 && usec < 0) {
386 sec--;
387 usec += 1000000;
388 }
Jens Axboe783a3eb2012-02-09 10:27:10 +0100389
390 /*
Jens Axboe39ab7da2012-11-06 22:10:43 +0100391 * time warp bug on some kernels?
Jens Axboe783a3eb2012-02-09 10:27:10 +0100392 */
Jens Axboe39ab7da2012-11-06 22:10:43 +0100393 if (sec < 0 || (sec == 0 && usec < 0))
394 return 0;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100395
Jens Axboe39ab7da2012-11-06 22:10:43 +0100396 ret = sec * 1000000ULL + usec;
397
398 return ret;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100399}
400
Jens Axboe0cbbc392014-09-30 16:04:12 -0600401uint64_t utime_since_now(const struct timeval *s)
Jens Axboe783a3eb2012-02-09 10:27:10 +0100402{
Jens Axboe39ab7da2012-11-06 22:10:43 +0100403 struct timeval t;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100404
Jens Axboe39ab7da2012-11-06 22:10:43 +0100405 fio_gettime(&t, NULL);
406 return utime_since(s, &t);
407}
Jens Axboe783a3eb2012-02-09 10:27:10 +0100408
Jens Axboe0cbbc392014-09-30 16:04:12 -0600409uint64_t mtime_since(const struct timeval *s, const struct timeval *e)
Jens Axboe39ab7da2012-11-06 22:10:43 +0100410{
411 long sec, usec, ret;
412
413 sec = e->tv_sec - s->tv_sec;
414 usec = e->tv_usec - s->tv_usec;
415 if (sec > 0 && usec < 0) {
416 sec--;
417 usec += 1000000;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100418 }
419
Jens Axboe39ab7da2012-11-06 22:10:43 +0100420 if (sec < 0 || (sec == 0 && usec < 0))
421 return 0;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100422
Jens Axboe39ab7da2012-11-06 22:10:43 +0100423 sec *= 1000UL;
424 usec /= 1000UL;
425 ret = sec + usec;
426
Jens Axboe783a3eb2012-02-09 10:27:10 +0100427 return ret;
428}
Jens Axboe39ab7da2012-11-06 22:10:43 +0100429
Jens Axboe0cbbc392014-09-30 16:04:12 -0600430uint64_t mtime_since_now(const struct timeval *s)
Jens Axboe39ab7da2012-11-06 22:10:43 +0100431{
432 struct timeval t;
433 void *p = __builtin_return_address(0);
434
435 fio_gettime(&t, p);
436 return mtime_since(s, &t);
437}
438
Jens Axboe0cbbc392014-09-30 16:04:12 -0600439uint64_t time_since_now(const struct timeval *s)
Jens Axboe39ab7da2012-11-06 22:10:43 +0100440{
441 return mtime_since_now(s) / 1000;
442}
Jens Axboe7d11f872012-12-17 12:03:29 +0100443
Jens Axboe67bf9822013-01-10 11:23:19 +0100444#if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \
445 defined(CONFIG_SFAA)
Jens Axboe7d11f872012-12-17 12:03:29 +0100446
Jens Axboe10564132014-12-23 10:41:34 -0700447#define CLOCK_ENTRIES_DEBUG 100000
448#define CLOCK_ENTRIES_TEST 10000
Jens Axboe7d11f872012-12-17 12:03:29 +0100449
450struct clock_entry {
Jens Axboe58002f92013-02-25 10:23:58 +0100451 uint32_t seq;
452 uint32_t cpu;
Jens Axboeba458c22013-01-21 05:38:22 -0700453 uint64_t tsc;
Jens Axboe7d11f872012-12-17 12:03:29 +0100454};
455
456struct clock_thread {
457 pthread_t thread;
458 int cpu;
Jens Axboe10564132014-12-23 10:41:34 -0700459 int debug;
Jens Axboe7d11f872012-12-17 12:03:29 +0100460 pthread_mutex_t lock;
461 pthread_mutex_t started;
Jens Axboe10564132014-12-23 10:41:34 -0700462 unsigned long nr_entries;
Jens Axboe58002f92013-02-25 10:23:58 +0100463 uint32_t *seq;
Jens Axboe7d11f872012-12-17 12:03:29 +0100464 struct clock_entry *entries;
465};
466
Jens Axboe58002f92013-02-25 10:23:58 +0100467static inline uint32_t atomic32_inc_return(uint32_t *seq)
Jens Axboe7d11f872012-12-17 12:03:29 +0100468{
469 return 1 + __sync_fetch_and_add(seq, 1);
470}
471
472static void *clock_thread_fn(void *data)
473{
474 struct clock_thread *t = data;
475 struct clock_entry *c;
476 os_cpu_mask_t cpu_mask;
Jens Axboe58002f92013-02-25 10:23:58 +0100477 uint32_t last_seq;
Jens Axboe7d11f872012-12-17 12:03:29 +0100478 int i;
479
Jens Axboe1af4be12015-01-28 14:42:10 -0700480 if (fio_cpuset_init(&cpu_mask)) {
Jens Axboeaa28ce02015-01-29 08:32:28 -0800481 int __err = errno;
Jens Axboe1af4be12015-01-28 14:42:10 -0700482
483 log_err("clock cpuset init failed: %s\n", strerror(__err));
484 goto err_out;
485 }
486
Jens Axboe7d11f872012-12-17 12:03:29 +0100487 fio_cpu_set(&cpu_mask, t->cpu);
488
489 if (fio_setaffinity(gettid(), cpu_mask) == -1) {
Jens Axboe4c64ff72015-01-28 14:34:10 -0700490 int __err = errno;
491
492 log_err("clock setaffinity failed: %s\n", strerror(__err));
Jens Axboe1af4be12015-01-28 14:42:10 -0700493 goto err;
Jens Axboe7d11f872012-12-17 12:03:29 +0100494 }
495
Jens Axboe7d11f872012-12-17 12:03:29 +0100496 pthread_mutex_lock(&t->lock);
Jens Axboeb9b34982012-12-17 14:23:47 +0100497 pthread_mutex_unlock(&t->started);
Jens Axboe7d11f872012-12-17 12:03:29 +0100498
Jens Axboe58002f92013-02-25 10:23:58 +0100499 last_seq = 0;
Jens Axboe7d11f872012-12-17 12:03:29 +0100500 c = &t->entries[0];
Jens Axboe10564132014-12-23 10:41:34 -0700501 for (i = 0; i < t->nr_entries; i++, c++) {
Jens Axboe58002f92013-02-25 10:23:58 +0100502 uint32_t seq;
503 uint64_t tsc;
Jens Axboe7d11f872012-12-17 12:03:29 +0100504
505 c->cpu = t->cpu;
506 do {
Jens Axboe58002f92013-02-25 10:23:58 +0100507 seq = atomic32_inc_return(t->seq);
508 if (seq < last_seq)
509 break;
Jens Axboe7d11f872012-12-17 12:03:29 +0100510 tsc = get_cpu_clock();
511 } while (seq != *t->seq);
512
513 c->seq = seq;
514 c->tsc = tsc;
515 }
516
Jens Axboe10564132014-12-23 10:41:34 -0700517 if (t->debug) {
518 unsigned long long clocks;
519
520 clocks = t->entries[i - 1].tsc - t->entries[0].tsc;
521 log_info("cs: cpu%3d: %llu clocks seen\n", t->cpu, clocks);
522 }
Jens Axboe58002f92013-02-25 10:23:58 +0100523
Jens Axboee2598792013-02-24 21:29:35 +0100524 /*
525 * The most common platform clock breakage is returning zero
526 * indefinitely. Check for that and return failure.
527 */
Jens Axboe58002f92013-02-25 10:23:58 +0100528 if (!t->entries[i - 1].tsc && !t->entries[0].tsc)
Jens Axboe1af4be12015-01-28 14:42:10 -0700529 goto err;
Jens Axboee2598792013-02-24 21:29:35 +0100530
Jens Axboe90881c82015-01-29 09:23:39 -0800531 fio_cpuset_exit(&cpu_mask);
Jens Axboe7d11f872012-12-17 12:03:29 +0100532 return NULL;
Jens Axboe1af4be12015-01-28 14:42:10 -0700533err:
534 fio_cpuset_exit(&cpu_mask);
535err_out:
536 return (void *) 1;
Jens Axboe7d11f872012-12-17 12:03:29 +0100537}
538
539static int clock_cmp(const void *p1, const void *p2)
540{
541 const struct clock_entry *c1 = p1;
542 const struct clock_entry *c2 = p2;
543
Jens Axboeb9b34982012-12-17 14:23:47 +0100544 if (c1->seq == c2->seq)
545 log_err("cs: bug in atomic sequence!\n");
546
Jens Axboe7d11f872012-12-17 12:03:29 +0100547 return c1->seq - c2->seq;
548}
549
Jens Axboe10564132014-12-23 10:41:34 -0700550int fio_monotonic_clocktest(int debug)
Jens Axboe7d11f872012-12-17 12:03:29 +0100551{
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700552 struct clock_thread *cthreads;
Jens Axboe7d11f872012-12-17 12:03:29 +0100553 unsigned int nr_cpus = cpus_online();
554 struct clock_entry *entries;
Jens Axboe10564132014-12-23 10:41:34 -0700555 unsigned long nr_entries, tentries, failed = 0;
Bruce Cran80da8a82013-02-21 14:16:17 +0100556 struct clock_entry *prev, *this;
Jens Axboe58002f92013-02-25 10:23:58 +0100557 uint32_t seq = 0;
Jens Axboecaa3eb12014-04-14 09:05:46 -0600558 unsigned int i;
Jens Axboe7d11f872012-12-17 12:03:29 +0100559
Jens Axboe10564132014-12-23 10:41:34 -0700560 if (debug) {
561 log_info("cs: reliable_tsc: %s\n", tsc_reliable ? "yes" : "no");
Jens Axboed5e3f5d2013-01-18 20:13:45 +0100562
Jens Axboefc3f3e42014-09-24 09:54:24 -0600563#ifdef FIO_INC_DEBUG
Jens Axboe10564132014-12-23 10:41:34 -0700564 fio_debug |= 1U << FD_TIME;
Jens Axboefc3f3e42014-09-24 09:54:24 -0600565#endif
Jens Axboe10564132014-12-23 10:41:34 -0700566 nr_entries = CLOCK_ENTRIES_DEBUG;
567 } else
568 nr_entries = CLOCK_ENTRIES_TEST;
569
Jens Axboe4f1d43c2012-12-17 14:29:28 +0100570 calibrate_cpu_clock();
Jens Axboe10564132014-12-23 10:41:34 -0700571
572 if (debug) {
Jens Axboefc3f3e42014-09-24 09:54:24 -0600573#ifdef FIO_INC_DEBUG
Jens Axboe10564132014-12-23 10:41:34 -0700574 fio_debug &= ~(1U << FD_TIME);
Jens Axboefc3f3e42014-09-24 09:54:24 -0600575#endif
Jens Axboe10564132014-12-23 10:41:34 -0700576 }
Jens Axboe4f1d43c2012-12-17 14:29:28 +0100577
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700578 cthreads = malloc(nr_cpus * sizeof(struct clock_thread));
Jens Axboe10564132014-12-23 10:41:34 -0700579 tentries = nr_entries * nr_cpus;
Jens Axboe7d11f872012-12-17 12:03:29 +0100580 entries = malloc(tentries * sizeof(struct clock_entry));
581
Jens Axboe10564132014-12-23 10:41:34 -0700582 if (debug)
583 log_info("cs: Testing %u CPUs\n", nr_cpus);
Jens Axboe7d11f872012-12-17 12:03:29 +0100584
585 for (i = 0; i < nr_cpus; i++) {
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700586 struct clock_thread *t = &cthreads[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100587
588 t->cpu = i;
Jens Axboe10564132014-12-23 10:41:34 -0700589 t->debug = debug;
Jens Axboe7d11f872012-12-17 12:03:29 +0100590 t->seq = &seq;
Jens Axboe10564132014-12-23 10:41:34 -0700591 t->nr_entries = nr_entries;
592 t->entries = &entries[i * nr_entries];
Jens Axboe7d11f872012-12-17 12:03:29 +0100593 pthread_mutex_init(&t->lock, NULL);
594 pthread_mutex_init(&t->started, NULL);
595 pthread_mutex_lock(&t->lock);
Jens Axboe6b0110c2014-04-14 12:14:17 -0600596 if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
597 failed++;
598 nr_cpus = i;
599 break;
600 }
Jens Axboe7d11f872012-12-17 12:03:29 +0100601 }
602
603 for (i = 0; i < nr_cpus; i++) {
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700604 struct clock_thread *t = &cthreads[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100605
606 pthread_mutex_lock(&t->started);
607 }
608
609 for (i = 0; i < nr_cpus; i++) {
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700610 struct clock_thread *t = &cthreads[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100611
612 pthread_mutex_unlock(&t->lock);
613 }
614
Jens Axboe814917b2014-04-14 12:20:04 -0600615 for (i = 0; i < nr_cpus; i++) {
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700616 struct clock_thread *t = &cthreads[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100617 void *ret;
618
619 pthread_join(t->thread, &ret);
620 if (ret)
621 failed++;
622 }
Jens Axboe0f7f9a92014-11-06 09:21:10 -0700623 free(cthreads);
Jens Axboe7d11f872012-12-17 12:03:29 +0100624
625 if (failed) {
Jens Axboe10564132014-12-23 10:41:34 -0700626 if (debug)
627 log_err("Clocksource test: %lu threads failed\n", failed);
Jens Axboe7d11f872012-12-17 12:03:29 +0100628 goto err;
629 }
630
631 qsort(entries, tentries, sizeof(struct clock_entry), clock_cmp);
632
633 for (failed = i = 0; i < tentries; i++) {
Bruce Cran80da8a82013-02-21 14:16:17 +0100634 this = &entries[i];
Jens Axboe7d11f872012-12-17 12:03:29 +0100635
636 if (!i) {
637 prev = this;
638 continue;
639 }
640
641 if (prev->tsc > this->tsc) {
642 uint64_t diff = prev->tsc - this->tsc;
643
Jens Axboe10564132014-12-23 10:41:34 -0700644 if (!debug) {
645 failed++;
646 break;
647 }
648
Jens Axboe4e0a8fa2013-04-15 11:40:57 +0200649 log_info("cs: CPU clock mismatch (diff=%llu):\n",
650 (unsigned long long) diff);
651 log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", prev->cpu, (unsigned long long) prev->tsc, prev->seq);
652 log_info("\t CPU%3u: TSC=%llu, SEQ=%u\n", this->cpu, (unsigned long long) this->tsc, this->seq);
Jens Axboe7d11f872012-12-17 12:03:29 +0100653 failed++;
654 }
655
656 prev = this;
657 }
658
Jens Axboe10564132014-12-23 10:41:34 -0700659 if (debug) {
660 if (failed)
661 log_info("cs: Failed: %lu\n", failed);
662 else
663 log_info("cs: Pass!\n");
664 }
Jens Axboe7d11f872012-12-17 12:03:29 +0100665err:
666 free(entries);
667 return !!failed;
668}
669
670#else /* defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) */
671
Jens Axboe10564132014-12-23 10:41:34 -0700672int fio_monotonic_clocktest(int debug)
Jens Axboe7d11f872012-12-17 12:03:29 +0100673{
Jens Axboe10564132014-12-23 10:41:34 -0700674 if (debug)
675 log_info("cs: current platform does not support CPU clocks\n");
676 return 1;
Jens Axboe7d11f872012-12-17 12:03:29 +0100677}
678
679#endif