Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 1 | /* |
| 2 | * sched_clock.c: support for extending counters to full 64-bit ns counter |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License version 2 as |
| 6 | * published by the Free Software Foundation. |
| 7 | */ |
| 8 | #include <linux/clocksource.h> |
| 9 | #include <linux/init.h> |
| 10 | #include <linux/jiffies.h> |
Stephen Boyd | a08ca5d | 2013-07-18 16:21:16 -0700 | [diff] [blame] | 11 | #include <linux/ktime.h> |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 12 | #include <linux/kernel.h> |
Russell King | a42c362 | 2012-09-09 18:39:28 +0100 | [diff] [blame] | 13 | #include <linux/moduleparam.h> |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 14 | #include <linux/sched.h> |
Russell King | f153d01 | 2012-02-04 12:31:27 +0000 | [diff] [blame] | 15 | #include <linux/syscore_ops.h> |
Stephen Boyd | a08ca5d | 2013-07-18 16:21:16 -0700 | [diff] [blame] | 16 | #include <linux/hrtimer.h> |
Stephen Boyd | 38ff87f | 2013-06-01 23:39:40 -0700 | [diff] [blame] | 17 | #include <linux/sched_clock.h> |
Stephen Boyd | 85c3d2d | 2013-07-18 16:21:15 -0700 | [diff] [blame] | 18 | #include <linux/seqlock.h> |
Stephen Boyd | e7e3ff1 | 2013-07-18 16:21:17 -0700 | [diff] [blame] | 19 | #include <linux/bitops.h> |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 20 | |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 21 | /** |
| 22 | * struct clock_read_data - data required to read from sched_clock |
| 23 | * |
| 24 | * @epoch_ns: sched_clock value at last update |
| 25 | * @epoch_cyc: Clock cycle value at last update |
| 26 | * @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit |
| 27 | * clocks |
| 28 | * @read_sched_clock: Current clock source (or dummy source when suspended) |
| 29 | * @mult: Multipler for scaled math conversion |
| 30 | * @shift: Shift value for scaled math conversion |
| 31 | * @suspended: Flag to indicate if the clock is suspended (stopped) |
| 32 | * |
| 33 | * Care must be taken when updating this structure; it is read by |
| 34 | * some very hot code paths. It occupies <=48 bytes and, when combined |
| 35 | * with the seqcount used to synchronize access, comfortably fits into |
| 36 | * a 64 byte cache line. |
| 37 | */ |
| 38 | struct clock_read_data { |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 39 | u64 epoch_ns; |
Stephen Boyd | e7e3ff1 | 2013-07-18 16:21:17 -0700 | [diff] [blame] | 40 | u64 epoch_cyc; |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 41 | u64 sched_clock_mask; |
| 42 | u64 (*read_sched_clock)(void); |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 43 | u32 mult; |
| 44 | u32 shift; |
Colin Cross | 237ec6f | 2012-08-07 19:05:10 +0100 | [diff] [blame] | 45 | bool suspended; |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 46 | }; |
| 47 | |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 48 | /** |
| 49 | * struct clock_data - all data needed for sched_clock (including |
| 50 | * registration of a new clock source) |
| 51 | * |
| 52 | * @seq: Sequence counter for protecting updates. |
| 53 | * @read_data: Data required to read from sched_clock. |
| 54 | * @wrap_kt: Duration for which clock can run before wrapping |
| 55 | * @rate: Tick rate of the registered clock |
| 56 | * @actual_read_sched_clock: Registered clock read function |
| 57 | * |
| 58 | * The ordering of this structure has been chosen to optimize cache |
| 59 | * performance. In particular seq and read_data (combined) should fit |
| 60 | * into a single 64 byte cache line. |
| 61 | */ |
| 62 | struct clock_data { |
| 63 | seqcount_t seq; |
| 64 | struct clock_read_data read_data; |
| 65 | ktime_t wrap_kt; |
| 66 | unsigned long rate; |
| 67 | }; |
| 68 | |
Stephen Boyd | a08ca5d | 2013-07-18 16:21:16 -0700 | [diff] [blame] | 69 | static struct hrtimer sched_clock_timer; |
Russell King | a42c362 | 2012-09-09 18:39:28 +0100 | [diff] [blame] | 70 | static int irqtime = -1; |
| 71 | |
| 72 | core_param(irqtime, irqtime, int, 0400); |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 73 | |
Stephen Boyd | e7e3ff1 | 2013-07-18 16:21:17 -0700 | [diff] [blame] | 74 | static u64 notrace jiffy_sched_clock_read(void) |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 75 | { |
Stephen Boyd | e7e3ff1 | 2013-07-18 16:21:17 -0700 | [diff] [blame] | 76 | /* |
| 77 | * We don't need to use get_jiffies_64 on 32-bit arches here |
| 78 | * because we register with BITS_PER_LONG |
| 79 | */ |
| 80 | return (u64)(jiffies - INITIAL_JIFFIES); |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 81 | } |
| 82 | |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 83 | static struct clock_data cd ____cacheline_aligned = { |
| 84 | .read_data = { .mult = NSEC_PER_SEC / HZ, |
| 85 | .read_sched_clock = jiffy_sched_clock_read, }, |
| 86 | }; |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 87 | |
Stephen Boyd | cea1509 | 2013-04-18 17:33:40 +0100 | [diff] [blame] | 88 | static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 89 | { |
| 90 | return (cyc * mult) >> shift; |
| 91 | } |
| 92 | |
Stephen Boyd | b4042ce | 2013-07-18 16:21:19 -0700 | [diff] [blame] | 93 | unsigned long long notrace sched_clock(void) |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 94 | { |
Daniel Thompson | 8710e91 | 2015-03-26 12:23:22 -0700 | [diff] [blame] | 95 | u64 cyc, res; |
Stephen Boyd | 85c3d2d | 2013-07-18 16:21:15 -0700 | [diff] [blame] | 96 | unsigned long seq; |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 97 | struct clock_read_data *rd = &cd.read_data; |
Stephen Boyd | 336ae11 | 2013-06-17 15:40:58 -0700 | [diff] [blame] | 98 | |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 99 | do { |
John Stultz | 7a06c41 | 2014-01-02 15:11:14 -0800 | [diff] [blame] | 100 | seq = raw_read_seqcount_begin(&cd.seq); |
Daniel Thompson | 8710e91 | 2015-03-26 12:23:22 -0700 | [diff] [blame] | 101 | |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 102 | res = rd->epoch_ns; |
| 103 | if (!rd->suspended) { |
| 104 | cyc = rd->read_sched_clock(); |
| 105 | cyc = (cyc - rd->epoch_cyc) & rd->sched_clock_mask; |
| 106 | res += cyc_to_ns(cyc, rd->mult, rd->shift); |
Daniel Thompson | 8710e91 | 2015-03-26 12:23:22 -0700 | [diff] [blame] | 107 | } |
Stephen Boyd | 85c3d2d | 2013-07-18 16:21:15 -0700 | [diff] [blame] | 108 | } while (read_seqcount_retry(&cd.seq, seq)); |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 109 | |
Daniel Thompson | 8710e91 | 2015-03-26 12:23:22 -0700 | [diff] [blame] | 110 | return res; |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 111 | } |
| 112 | |
| 113 | /* |
| 114 | * Atomically update the sched_clock epoch. |
| 115 | */ |
| 116 | static void notrace update_sched_clock(void) |
| 117 | { |
| 118 | unsigned long flags; |
Stephen Boyd | e7e3ff1 | 2013-07-18 16:21:17 -0700 | [diff] [blame] | 119 | u64 cyc; |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 120 | u64 ns; |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 121 | struct clock_read_data *rd = &cd.read_data; |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 122 | |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 123 | cyc = rd->read_sched_clock(); |
| 124 | ns = rd->epoch_ns + |
| 125 | cyc_to_ns((cyc - rd->epoch_cyc) & rd->sched_clock_mask, |
| 126 | rd->mult, rd->shift); |
Stephen Boyd | 85c3d2d | 2013-07-18 16:21:15 -0700 | [diff] [blame] | 127 | |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 128 | raw_local_irq_save(flags); |
John Stultz | 7a06c41 | 2014-01-02 15:11:14 -0800 | [diff] [blame] | 129 | raw_write_seqcount_begin(&cd.seq); |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 130 | rd->epoch_ns = ns; |
| 131 | rd->epoch_cyc = cyc; |
John Stultz | 7a06c41 | 2014-01-02 15:11:14 -0800 | [diff] [blame] | 132 | raw_write_seqcount_end(&cd.seq); |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 133 | raw_local_irq_restore(flags); |
| 134 | } |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 135 | |
Stephen Boyd | a08ca5d | 2013-07-18 16:21:16 -0700 | [diff] [blame] | 136 | static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 137 | { |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 138 | update_sched_clock(); |
Stephen Boyd | a08ca5d | 2013-07-18 16:21:16 -0700 | [diff] [blame] | 139 | hrtimer_forward_now(hrt, cd.wrap_kt); |
| 140 | return HRTIMER_RESTART; |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 141 | } |
| 142 | |
Stephen Boyd | e7e3ff1 | 2013-07-18 16:21:17 -0700 | [diff] [blame] | 143 | void __init sched_clock_register(u64 (*read)(void), int bits, |
| 144 | unsigned long rate) |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 145 | { |
Stephen Boyd | 5ae8aab | 2014-02-17 10:45:36 -0800 | [diff] [blame] | 146 | u64 res, wrap, new_mask, new_epoch, cyc, ns; |
| 147 | u32 new_mult, new_shift; |
Stephen Boyd | a08ca5d | 2013-07-18 16:21:16 -0700 | [diff] [blame] | 148 | unsigned long r; |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 149 | char r_unit; |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 150 | struct clock_read_data *rd = &cd.read_data; |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 151 | |
Rob Herring | c115739 | 2013-02-08 16:14:59 -0600 | [diff] [blame] | 152 | if (cd.rate > rate) |
| 153 | return; |
| 154 | |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 155 | WARN_ON(!irqs_disabled()); |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 156 | |
| 157 | /* calculate the mult/shift to convert counter ticks to ns. */ |
Stephen Boyd | 5ae8aab | 2014-02-17 10:45:36 -0800 | [diff] [blame] | 158 | clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); |
| 159 | |
| 160 | new_mask = CLOCKSOURCE_MASK(bits); |
Daniel Thompson | 8710e91 | 2015-03-26 12:23:22 -0700 | [diff] [blame] | 161 | cd.rate = rate; |
Stephen Boyd | 5ae8aab | 2014-02-17 10:45:36 -0800 | [diff] [blame] | 162 | |
John Stultz | 362fde0 | 2015-03-11 21:16:30 -0700 | [diff] [blame] | 163 | /* calculate how many nanosecs until we risk wrapping */ |
John Stultz | fb82fe2 | 2015-03-11 21:16:31 -0700 | [diff] [blame] | 164 | wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask, NULL); |
Daniel Thompson | 8710e91 | 2015-03-26 12:23:22 -0700 | [diff] [blame] | 165 | cd.wrap_kt = ns_to_ktime(wrap); |
Stephen Boyd | 5ae8aab | 2014-02-17 10:45:36 -0800 | [diff] [blame] | 166 | |
| 167 | /* update epoch for new counter and update epoch_ns from old counter*/ |
| 168 | new_epoch = read(); |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 169 | cyc = rd->read_sched_clock(); |
| 170 | ns = rd->epoch_ns + |
| 171 | cyc_to_ns((cyc - rd->epoch_cyc) & rd->sched_clock_mask, |
| 172 | rd->mult, rd->shift); |
Stephen Boyd | 5ae8aab | 2014-02-17 10:45:36 -0800 | [diff] [blame] | 173 | |
| 174 | raw_write_seqcount_begin(&cd.seq); |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 175 | rd->read_sched_clock = read; |
| 176 | rd->sched_clock_mask = new_mask; |
| 177 | rd->mult = new_mult; |
| 178 | rd->shift = new_shift; |
| 179 | rd->epoch_cyc = new_epoch; |
| 180 | rd->epoch_ns = ns; |
Stephen Boyd | 5ae8aab | 2014-02-17 10:45:36 -0800 | [diff] [blame] | 181 | raw_write_seqcount_end(&cd.seq); |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 182 | |
| 183 | r = rate; |
| 184 | if (r >= 4000000) { |
| 185 | r /= 1000000; |
| 186 | r_unit = 'M'; |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 187 | } else if (r >= 1000) { |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 188 | r /= 1000; |
| 189 | r_unit = 'k'; |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 190 | } else |
| 191 | r_unit = ' '; |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 192 | |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 193 | /* calculate the ns resolution of this counter */ |
Stephen Boyd | 5ae8aab | 2014-02-17 10:45:36 -0800 | [diff] [blame] | 194 | res = cyc_to_ns(1ULL, new_mult, new_shift); |
| 195 | |
Stephen Boyd | a08ca5d | 2013-07-18 16:21:16 -0700 | [diff] [blame] | 196 | pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", |
| 197 | bits, r, r_unit, res, wrap); |
Russell King | 112f38a4 | 2010-12-15 19:23:07 +0000 | [diff] [blame] | 198 | |
Russell King | a42c362 | 2012-09-09 18:39:28 +0100 | [diff] [blame] | 199 | /* Enable IRQ time accounting if we have a fast enough sched_clock */ |
| 200 | if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) |
| 201 | enable_sched_clock_irqtime(); |
| 202 | |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 203 | pr_debug("Registered %pF as sched_clock source\n", read); |
| 204 | } |
| 205 | |
Russell King | 211baa70 | 2011-01-11 16:23:04 +0000 | [diff] [blame] | 206 | void __init sched_clock_postinit(void) |
| 207 | { |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 208 | /* |
| 209 | * If no sched_clock function has been provided at that point, |
| 210 | * make it the final one one. |
| 211 | */ |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 212 | if (cd.read_data.read_sched_clock == jiffy_sched_clock_read) |
Stephen Boyd | e7e3ff1 | 2013-07-18 16:21:17 -0700 | [diff] [blame] | 213 | sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); |
Marc Zyngier | 2f0778af | 2011-12-15 12:19:23 +0100 | [diff] [blame] | 214 | |
Stephen Boyd | a08ca5d | 2013-07-18 16:21:16 -0700 | [diff] [blame] | 215 | update_sched_clock(); |
| 216 | |
| 217 | /* |
| 218 | * Start the timer to keep sched_clock() properly updated and |
| 219 | * sets the initial epoch. |
| 220 | */ |
| 221 | hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 222 | sched_clock_timer.function = sched_clock_poll; |
| 223 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); |
Russell King | 211baa70 | 2011-01-11 16:23:04 +0000 | [diff] [blame] | 224 | } |
Russell King | f153d01 | 2012-02-04 12:31:27 +0000 | [diff] [blame] | 225 | |
| 226 | static int sched_clock_suspend(void) |
| 227 | { |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 228 | struct clock_read_data *rd = &cd.read_data; |
| 229 | |
Stephen Boyd | f723aa1 | 2014-07-23 21:03:50 -0700 | [diff] [blame] | 230 | update_sched_clock(); |
| 231 | hrtimer_cancel(&sched_clock_timer); |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 232 | rd->suspended = true; |
Russell King | f153d01 | 2012-02-04 12:31:27 +0000 | [diff] [blame] | 233 | return 0; |
| 234 | } |
| 235 | |
Colin Cross | 237ec6f | 2012-08-07 19:05:10 +0100 | [diff] [blame] | 236 | static void sched_clock_resume(void) |
| 237 | { |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 238 | struct clock_read_data *rd = &cd.read_data; |
| 239 | |
| 240 | rd->epoch_cyc = rd->read_sched_clock(); |
Stephen Boyd | f723aa1 | 2014-07-23 21:03:50 -0700 | [diff] [blame] | 241 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); |
Daniel Thompson | cf7c9c1 | 2015-03-26 12:23:23 -0700 | [diff] [blame^] | 242 | rd->suspended = false; |
Colin Cross | 237ec6f | 2012-08-07 19:05:10 +0100 | [diff] [blame] | 243 | } |
| 244 | |
Russell King | f153d01 | 2012-02-04 12:31:27 +0000 | [diff] [blame] | 245 | static struct syscore_ops sched_clock_ops = { |
| 246 | .suspend = sched_clock_suspend, |
Colin Cross | 237ec6f | 2012-08-07 19:05:10 +0100 | [diff] [blame] | 247 | .resume = sched_clock_resume, |
Russell King | f153d01 | 2012-02-04 12:31:27 +0000 | [diff] [blame] | 248 | }; |
| 249 | |
| 250 | static int __init sched_clock_syscore_init(void) |
| 251 | { |
| 252 | register_syscore_ops(&sched_clock_ops); |
| 253 | return 0; |
| 254 | } |
| 255 | device_initcall(sched_clock_syscore_init); |