blob: 89a3a016e4c871a767365f96318392a65ec30ba6 [file] [log] [blame]
Jens Axboe02bcaa82006-11-24 10:42:00 +01001/*
Jens Axboef5cc0242006-11-24 10:47:40 +01002 * Clock functions
Jens Axboe02bcaa82006-11-24 10:42:00 +01003 */
Jens Axboef5cc0242006-11-24 10:47:40 +01004
Jens Axboe02bcaa82006-11-24 10:42:00 +01005#include <unistd.h>
Jens Axboec223da82010-03-24 13:23:53 +01006#include <math.h>
Jens Axboe02bcaa82006-11-24 10:42:00 +01007#include <sys/time.h>
Bruce Cran03e20d62011-01-02 20:14:54 +01008#include <time.h>
Jens Axboe02bcaa82006-11-24 10:42:00 +01009
10#include "fio.h"
Jens Axboebe4ecfd2008-12-08 14:10:52 +010011#include "smalloc.h"
Jens Axboe02bcaa82006-11-24 10:42:00 +010012
13#include "hash.h"
14
Jens Axboe09a32402010-08-15 15:01:51 -040015#ifdef ARCH_HAVE_CPU_CLOCK
Jens Axboec223da82010-03-24 13:23:53 +010016static unsigned long cycles_per_usec;
Jens Axboec223da82010-03-24 13:23:53 +010017static unsigned long last_cycles;
Jens Axboefa80fea2012-12-09 20:29:00 +010018int tsc_reliable = 0;
Jens Axboe09a32402010-08-15 15:01:51 -040019#endif
20static struct timeval last_tv;
Jens Axboe3e488922008-11-13 12:07:20 +010021static int last_tv_valid;
Jens Axboe02bcaa82006-11-24 10:42:00 +010022
Bruce Cran16de1bf2012-02-20 17:07:32 +000023enum fio_cs fio_clock_source = FIO_PREFERRED_CLOCK_SOURCE;
Jens Axboefa80fea2012-12-09 20:29:00 +010024int fio_clock_source_set = 0;
Jens Axboec223da82010-03-24 13:23:53 +010025
Jens Axboe02bcaa82006-11-24 10:42:00 +010026#ifdef FIO_DEBUG_TIME
27
28#define HASH_BITS 8
29#define HASH_SIZE (1 << HASH_BITS)
30
Jens Axboe01743ee2008-06-02 12:19:19 +020031static struct flist_head hash[HASH_SIZE];
Jens Axboe02bcaa82006-11-24 10:42:00 +010032static int gtod_inited;
33
34struct gtod_log {
Jens Axboe01743ee2008-06-02 12:19:19 +020035 struct flist_head list;
Jens Axboe02bcaa82006-11-24 10:42:00 +010036 void *caller;
37 unsigned long calls;
38};
39
40static struct gtod_log *find_hash(void *caller)
41{
42 unsigned long h = hash_ptr(caller, HASH_BITS);
Jens Axboe01743ee2008-06-02 12:19:19 +020043 struct flist_head *entry;
Jens Axboe02bcaa82006-11-24 10:42:00 +010044
Jens Axboe01743ee2008-06-02 12:19:19 +020045 flist_for_each(entry, &hash[h]) {
46 struct gtod_log *log = flist_entry(entry, struct gtod_log,
47 list);
Jens Axboe02bcaa82006-11-24 10:42:00 +010048
49 if (log->caller == caller)
50 return log;
51 }
52
53 return NULL;
54}
55
56static struct gtod_log *find_log(void *caller)
57{
58 struct gtod_log *log = find_hash(caller);
59
60 if (!log) {
61 unsigned long h;
62
63 log = malloc(sizeof(*log));
Jens Axboe01743ee2008-06-02 12:19:19 +020064 INIT_FLIST_HEAD(&log->list);
Jens Axboe02bcaa82006-11-24 10:42:00 +010065 log->caller = caller;
66 log->calls = 0;
67
68 h = hash_ptr(caller, HASH_BITS);
Jens Axboe01743ee2008-06-02 12:19:19 +020069 flist_add_tail(&log->list, &hash[h]);
Jens Axboe02bcaa82006-11-24 10:42:00 +010070 }
71
72 return log;
73}
74
75static void gtod_log_caller(void *caller)
76{
77 if (gtod_inited) {
78 struct gtod_log *log = find_log(caller);
79
80 log->calls++;
81 }
82}
83
84static void fio_exit fio_dump_gtod(void)
85{
86 unsigned long total_calls = 0;
87 int i;
88
89 for (i = 0; i < HASH_SIZE; i++) {
Jens Axboe01743ee2008-06-02 12:19:19 +020090 struct flist_head *entry;
Jens Axboe02bcaa82006-11-24 10:42:00 +010091 struct gtod_log *log;
92
Jens Axboe01743ee2008-06-02 12:19:19 +020093 flist_for_each(entry, &hash[i]) {
94 log = flist_entry(entry, struct gtod_log, list);
Jens Axboe02bcaa82006-11-24 10:42:00 +010095
Jens Axboe5ec10ea2008-03-06 15:42:00 +010096 printf("function %p, calls %lu\n", log->caller,
97 log->calls);
Jens Axboe02bcaa82006-11-24 10:42:00 +010098 total_calls += log->calls;
99 }
100 }
101
102 printf("Total %lu gettimeofday\n", total_calls);
103}
104
105static void fio_init gtod_init(void)
106{
107 int i;
108
109 for (i = 0; i < HASH_SIZE; i++)
Jens Axboe01743ee2008-06-02 12:19:19 +0200110 INIT_FLIST_HEAD(&hash[i]);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100111
112 gtod_inited = 1;
113}
114
115#endif /* FIO_DEBUG_TIME */
116
Jens Axboe1e97cce2006-12-05 11:44:16 +0100117#ifdef FIO_DEBUG_TIME
Jens Axboe02bcaa82006-11-24 10:42:00 +0100118void fio_gettime(struct timeval *tp, void *caller)
Jens Axboe1e97cce2006-12-05 11:44:16 +0100119#else
120void fio_gettime(struct timeval *tp, void fio_unused *caller)
121#endif
Jens Axboe02bcaa82006-11-24 10:42:00 +0100122{
123#ifdef FIO_DEBUG_TIME
124 if (!caller)
125 caller = __builtin_return_address(0);
126
127 gtod_log_caller(caller);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100128#endif
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100129 if (fio_tv) {
130 memcpy(tp, fio_tv, sizeof(*tp));
131 return;
Jens Axboec223da82010-03-24 13:23:53 +0100132 }
133
134 switch (fio_clock_source) {
135 case CS_GTOD:
Jens Axboe02bcaa82006-11-24 10:42:00 +0100136 gettimeofday(tp, NULL);
Jens Axboec223da82010-03-24 13:23:53 +0100137 break;
138 case CS_CGETTIME: {
Jens Axboe02bcaa82006-11-24 10:42:00 +0100139 struct timespec ts;
140
Jens Axboe5bd9c782012-02-03 12:48:16 +0100141#ifdef FIO_HAVE_CLOCK_MONOTONIC
142 if (clock_gettime(CLOCK_MONOTONIC, &ts) < 0) {
143#else
Jens Axboed481e002009-12-04 09:56:32 +0100144 if (clock_gettime(CLOCK_REALTIME, &ts) < 0) {
Jens Axboe5bd9c782012-02-03 12:48:16 +0100145#endif
Jens Axboec223da82010-03-24 13:23:53 +0100146 log_err("fio: clock_gettime fails\n");
147 assert(0);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100148 }
149
150 tp->tv_sec = ts.tv_sec;
151 tp->tv_usec = ts.tv_nsec / 1000;
Jens Axboec223da82010-03-24 13:23:53 +0100152 break;
153 }
154#ifdef ARCH_HAVE_CPU_CLOCK
155 case CS_CPUCLOCK: {
156 unsigned long long usecs, t;
157
158 t = get_cpu_clock();
159 if (t < last_cycles) {
160 dprint(FD_TIME, "CPU clock going back in time\n");
161 t = last_cycles;
162 }
163
164 usecs = t / cycles_per_usec;
165 tp->tv_sec = usecs / 1000000;
166 tp->tv_usec = usecs % 1000000;
167 last_cycles = t;
168 break;
169 }
170#endif
171 default:
172 log_err("fio: invalid clock source %d\n", fio_clock_source);
173 break;
Jens Axboe02bcaa82006-11-24 10:42:00 +0100174 }
Jens Axboe3e488922008-11-13 12:07:20 +0100175
176 /*
177 * If Linux is using the tsc clock on non-synced processors,
178 * sometimes time can appear to drift backwards. Fix that up.
179 */
180 if (last_tv_valid) {
181 if (tp->tv_sec < last_tv.tv_sec)
182 tp->tv_sec = last_tv.tv_sec;
183 else if (last_tv.tv_sec == tp->tv_sec &&
184 tp->tv_usec < last_tv.tv_usec)
185 tp->tv_usec = last_tv.tv_usec;
186 }
187 last_tv_valid = 1;
188 memcpy(&last_tv, tp, sizeof(*tp));
Jens Axboe02bcaa82006-11-24 10:42:00 +0100189}
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100190
Jens Axboe09a32402010-08-15 15:01:51 -0400191#ifdef ARCH_HAVE_CPU_CLOCK
Jens Axboec223da82010-03-24 13:23:53 +0100192static unsigned long get_cycles_per_usec(void)
193{
194 struct timeval s, e;
195 unsigned long long c_s, c_e;
196
197 gettimeofday(&s, NULL);
198 c_s = get_cpu_clock();
199 do {
200 unsigned long long elapsed;
201
202 gettimeofday(&e, NULL);
203 elapsed = utime_since(&s, &e);
204 if (elapsed >= 10) {
205 c_e = get_cpu_clock();
206 break;
207 }
208 } while (1);
209
210 return c_e - c_s;
211}
212
Jens Axboefa80fea2012-12-09 20:29:00 +0100213#define NR_TIME_ITERS 50
214
Jens Axboe09a32402010-08-15 15:01:51 -0400215static void calibrate_cpu_clock(void)
Jens Axboec223da82010-03-24 13:23:53 +0100216{
217 double delta, mean, S;
Jens Axboefa80fea2012-12-09 20:29:00 +0100218 unsigned long avg, cycles[NR_TIME_ITERS];
Jens Axboec223da82010-03-24 13:23:53 +0100219 int i, samples;
220
Jens Axboec223da82010-03-24 13:23:53 +0100221 cycles[0] = get_cycles_per_usec();
222 S = delta = mean = 0.0;
Jens Axboefa80fea2012-12-09 20:29:00 +0100223 for (i = 0; i < NR_TIME_ITERS; i++) {
Jens Axboec223da82010-03-24 13:23:53 +0100224 cycles[i] = get_cycles_per_usec();
225 delta = cycles[i] - mean;
226 if (delta) {
227 mean += delta / (i + 1.0);
228 S += delta * (cycles[i] - mean);
229 }
230 }
231
Jens Axboefa80fea2012-12-09 20:29:00 +0100232 S = sqrt(S / (NR_TIME_ITERS - 1.0));
Jens Axboec223da82010-03-24 13:23:53 +0100233
234 samples = avg = 0;
Jens Axboefa80fea2012-12-09 20:29:00 +0100235 for (i = 0; i < NR_TIME_ITERS; i++) {
Jens Axboec223da82010-03-24 13:23:53 +0100236 double this = cycles[i];
237
Bruce Cran03e20d62011-01-02 20:14:54 +0100238 if ((fmax(this, mean) - fmin(this, mean)) > S)
Jens Axboec223da82010-03-24 13:23:53 +0100239 continue;
240 samples++;
241 avg += this;
242 }
243
Jens Axboefa80fea2012-12-09 20:29:00 +0100244 S /= (double) NR_TIME_ITERS;
245 mean /= (double) NR_TIME_ITERS;
Jens Axboec223da82010-03-24 13:23:53 +0100246
Jens Axboefa80fea2012-12-09 20:29:00 +0100247 for (i = 0; i < NR_TIME_ITERS; i++)
Jens Axboec223da82010-03-24 13:23:53 +0100248 dprint(FD_TIME, "cycles[%d]=%lu\n", i, cycles[i] / 10);
249
250 avg /= (samples * 10);
251 dprint(FD_TIME, "avg: %lu\n", avg);
252 dprint(FD_TIME, "mean=%f, S=%f\n", mean, S);
253
254 cycles_per_usec = avg;
Jens Axboe09a32402010-08-15 15:01:51 -0400255}
256#else
257static void calibrate_cpu_clock(void)
258{
259}
260#endif
261
262void fio_clock_init(void)
263{
264 last_tv_valid = 0;
265 calibrate_cpu_clock();
Jens Axboefa80fea2012-12-09 20:29:00 +0100266
267 /*
268 * If the arch sets tsc_reliable != 0, then it must be good enough
269 * to use as THE clock source. For x86 CPUs, this means the TSC
270 * runs at a constant rate and is synced across CPU cores.
271 */
272 if (tsc_reliable) {
273 if (!fio_clock_source_set)
274 fio_clock_source = CS_CPUCLOCK;
275 } else if (fio_clock_source == CS_CPUCLOCK)
276 log_info("fio: clocksource=cpu may not be reliable\n");
Jens Axboec223da82010-03-24 13:23:53 +0100277}
278
Jens Axboe39ab7da2012-11-06 22:10:43 +0100279unsigned long long utime_since(struct timeval *s, struct timeval *e)
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100280{
Jens Axboe39ab7da2012-11-06 22:10:43 +0100281 long sec, usec;
282 unsigned long long ret;
Jens Axboebe4ecfd2008-12-08 14:10:52 +0100283
Jens Axboe39ab7da2012-11-06 22:10:43 +0100284 sec = e->tv_sec - s->tv_sec;
285 usec = e->tv_usec - s->tv_usec;
286 if (sec > 0 && usec < 0) {
287 sec--;
288 usec += 1000000;
289 }
Jens Axboe783a3eb2012-02-09 10:27:10 +0100290
291 /*
Jens Axboe39ab7da2012-11-06 22:10:43 +0100292 * time warp bug on some kernels?
Jens Axboe783a3eb2012-02-09 10:27:10 +0100293 */
Jens Axboe39ab7da2012-11-06 22:10:43 +0100294 if (sec < 0 || (sec == 0 && usec < 0))
295 return 0;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100296
Jens Axboe39ab7da2012-11-06 22:10:43 +0100297 ret = sec * 1000000ULL + usec;
298
299 return ret;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100300}
301
Jens Axboe39ab7da2012-11-06 22:10:43 +0100302unsigned long long utime_since_now(struct timeval *s)
Jens Axboe783a3eb2012-02-09 10:27:10 +0100303{
Jens Axboe39ab7da2012-11-06 22:10:43 +0100304 struct timeval t;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100305
Jens Axboe39ab7da2012-11-06 22:10:43 +0100306 fio_gettime(&t, NULL);
307 return utime_since(s, &t);
308}
Jens Axboe783a3eb2012-02-09 10:27:10 +0100309
Jens Axboe39ab7da2012-11-06 22:10:43 +0100310unsigned long mtime_since(struct timeval *s, struct timeval *e)
311{
312 long sec, usec, ret;
313
314 sec = e->tv_sec - s->tv_sec;
315 usec = e->tv_usec - s->tv_usec;
316 if (sec > 0 && usec < 0) {
317 sec--;
318 usec += 1000000;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100319 }
320
Jens Axboe39ab7da2012-11-06 22:10:43 +0100321 if (sec < 0 || (sec == 0 && usec < 0))
322 return 0;
Jens Axboe783a3eb2012-02-09 10:27:10 +0100323
Jens Axboe39ab7da2012-11-06 22:10:43 +0100324 sec *= 1000UL;
325 usec /= 1000UL;
326 ret = sec + usec;
327
Jens Axboe783a3eb2012-02-09 10:27:10 +0100328 return ret;
329}
Jens Axboe39ab7da2012-11-06 22:10:43 +0100330
331unsigned long mtime_since_now(struct timeval *s)
332{
333 struct timeval t;
334 void *p = __builtin_return_address(0);
335
336 fio_gettime(&t, p);
337 return mtime_since(s, &t);
338}
339
340unsigned long time_since_now(struct timeval *s)
341{
342 return mtime_since_now(s) / 1000;
343}