blob: 14f24062349752a268a7584efcc538431c371152 [file] [log] [blame]
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -07001/*
2 * Xen time implementation.
3 *
4 * This is implemented in terms of a clocksource driver which uses
5 * the hypervisor clock as a nanosecond timebase, and a clockevent
6 * driver which uses the hypervisor's timer mechanism.
7 *
8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
9 */
10#include <linux/kernel.h>
11#include <linux/interrupt.h>
12#include <linux/clocksource.h>
13#include <linux/clockchips.h>
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -070014#include <linux/kernel_stat.h>
Jeremy Fitzhardingef595ec92008-06-12 10:47:56 +020015#include <linux/math64.h>
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -070016
Gerd Hoffmann1c7b67f2008-06-03 16:17:30 +020017#include <asm/pvclock.h>
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -070018#include <asm/xen/hypervisor.h>
19#include <asm/xen/hypercall.h>
20
21#include <xen/events.h>
22#include <xen/interface/xen.h>
23#include <xen/interface/vcpu.h>
24
25#include "xen-ops.h"
26
27#define XEN_SHIFT 22
28
29/* Xen may fire a timer up to this many ns early */
30#define TIMER_SLOP 100000
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -070031#define NS_PER_TICK (1000000000LL / HZ)
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -070032
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -070033/* runstate info updated by Xen */
34static DEFINE_PER_CPU(struct vcpu_runstate_info, runstate);
35
36/* snapshots of runstate info */
37static DEFINE_PER_CPU(struct vcpu_runstate_info, runstate_snapshot);
38
39/* unused ns of stolen and blocked time */
40static DEFINE_PER_CPU(u64, residual_stolen);
41static DEFINE_PER_CPU(u64, residual_blocked);
42
43/* return an consistent snapshot of 64-bit time/counter value */
44static u64 get64(const u64 *p)
45{
46 u64 ret;
47
48 if (BITS_PER_LONG < 64) {
49 u32 *p32 = (u32 *)p;
50 u32 h, l;
51
52 /*
53 * Read high then low, and then make sure high is
54 * still the same; this will only loop if low wraps
55 * and carries into high.
56 * XXX some clean way to make this endian-proof?
57 */
58 do {
59 h = p32[1];
60 barrier();
61 l = p32[0];
62 barrier();
63 } while (p32[1] != h);
64
65 ret = (((u64)h) << 32) | l;
66 } else
67 ret = *p;
68
69 return ret;
70}
71
72/*
73 * Runstate accounting
74 */
75static void get_runstate_snapshot(struct vcpu_runstate_info *res)
76{
77 u64 state_time;
78 struct vcpu_runstate_info *state;
79
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -070080 BUG_ON(preemptible());
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -070081
82 state = &__get_cpu_var(runstate);
83
84 /*
85 * The runstate info is always updated by the hypervisor on
86 * the current CPU, so there's no need to use anything
87 * stronger than a compiler barrier when fetching it.
88 */
89 do {
90 state_time = get64(&state->state_entry_time);
91 barrier();
92 *res = *state;
93 barrier();
94 } while (get64(&state->state_entry_time) != state_time);
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -070095}
96
Jeremy Fitzhardingef0d73392007-10-16 11:51:30 -070097/* return true when a vcpu could run but has no real cpu to run on */
98bool xen_vcpu_stolen(int vcpu)
99{
100 return per_cpu(runstate, vcpu).state == RUNSTATE_runnable;
101}
102
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700103static void setup_runstate_info(int cpu)
104{
105 struct vcpu_register_runstate_memory_area area;
106
107 area.addr.v = &per_cpu(runstate, cpu);
108
109 if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
110 cpu, &area))
111 BUG();
112}
113
114static void do_stolen_accounting(void)
115{
116 struct vcpu_runstate_info state;
117 struct vcpu_runstate_info *snap;
118 s64 blocked, runnable, offline, stolen;
119 cputime_t ticks;
120
121 get_runstate_snapshot(&state);
122
123 WARN_ON(state.state != RUNSTATE_running);
124
125 snap = &__get_cpu_var(runstate_snapshot);
126
127 /* work out how much time the VCPU has not been runn*ing* */
128 blocked = state.time[RUNSTATE_blocked] - snap->time[RUNSTATE_blocked];
129 runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
130 offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
131
132 *snap = state;
133
134 /* Add the appropriate number of ticks of stolen time,
Martin Schwidefsky79741dd2008-12-31 15:11:38 +0100135 including any left-overs from last time. */
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700136 stolen = runnable + offline + __get_cpu_var(residual_stolen);
137
138 if (stolen < 0)
139 stolen = 0;
140
Jeremy Fitzhardingef595ec92008-06-12 10:47:56 +0200141 ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700142 __get_cpu_var(residual_stolen) = stolen;
Martin Schwidefsky79741dd2008-12-31 15:11:38 +0100143 account_steal_ticks(ticks);
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700144
145 /* Add the appropriate number of ticks of blocked time,
Martin Schwidefsky79741dd2008-12-31 15:11:38 +0100146 including any left-overs from last time. */
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700147 blocked += __get_cpu_var(residual_blocked);
148
149 if (blocked < 0)
150 blocked = 0;
151
Jeremy Fitzhardingef595ec92008-06-12 10:47:56 +0200152 ticks = iter_div_u64_rem(blocked, NS_PER_TICK, &blocked);
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700153 __get_cpu_var(residual_blocked) = blocked;
Martin Schwidefsky79741dd2008-12-31 15:11:38 +0100154 account_idle_ticks(ticks);
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700155}
156
Jeremy Fitzhardingeab550282007-07-17 18:37:05 -0700157/*
158 * Xen sched_clock implementation. Returns the number of unstolen
159 * nanoseconds, which is nanoseconds the VCPU spent in RUNNING+BLOCKED
160 * states.
161 */
162unsigned long long xen_sched_clock(void)
163{
164 struct vcpu_runstate_info state;
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700165 cycle_t now;
166 u64 ret;
Jeremy Fitzhardingeab550282007-07-17 18:37:05 -0700167 s64 offset;
168
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700169 /*
170 * Ideally sched_clock should be called on a per-cpu basis
171 * anyway, so preempt should already be disabled, but that's
172 * not current practice at the moment.
173 */
174 preempt_disable();
175
176 now = xen_clocksource_read();
177
Jeremy Fitzhardingeab550282007-07-17 18:37:05 -0700178 get_runstate_snapshot(&state);
179
180 WARN_ON(state.state != RUNSTATE_running);
181
182 offset = now - state.state_entry_time;
183 if (offset < 0)
184 offset = 0;
185
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700186 ret = state.time[RUNSTATE_blocked] +
Jeremy Fitzhardingeab550282007-07-17 18:37:05 -0700187 state.time[RUNSTATE_running] +
188 offset;
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700189
190 preempt_enable();
191
192 return ret;
Jeremy Fitzhardingeab550282007-07-17 18:37:05 -0700193}
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700194
195
Alok Katariae93ef942008-07-01 11:43:36 -0700196/* Get the TSC speed from Xen */
197unsigned long xen_tsc_khz(void)
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700198{
Glauber Costa3807f342008-07-28 11:47:52 -0300199 struct pvclock_vcpu_time_info *info =
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700200 &HYPERVISOR_shared_info->vcpu_info[0].time;
201
Glauber Costa3807f342008-07-28 11:47:52 -0300202 return pvclock_tsc_khz(info);
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700203}
204
Jeremy Fitzhardingeee7686b2008-08-21 13:17:56 -0700205cycle_t xen_clocksource_read(void)
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700206{
Gerd Hoffmann1c7b67f2008-06-03 16:17:30 +0200207 struct pvclock_vcpu_time_info *src;
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700208 cycle_t ret;
209
Gerd Hoffmann1c7b67f2008-06-03 16:17:30 +0200210 src = &get_cpu_var(xen_vcpu)->time;
211 ret = pvclock_clocksource_read(src);
212 put_cpu_var(xen_vcpu);
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700213 return ret;
214}
215
216static void xen_read_wallclock(struct timespec *ts)
217{
Gerd Hoffmann1c7b67f2008-06-03 16:17:30 +0200218 struct shared_info *s = HYPERVISOR_shared_info;
219 struct pvclock_wall_clock *wall_clock = &(s->wc);
220 struct pvclock_vcpu_time_info *vcpu_time;
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700221
Gerd Hoffmann1c7b67f2008-06-03 16:17:30 +0200222 vcpu_time = &get_cpu_var(xen_vcpu)->time;
223 pvclock_read_wallclock(wall_clock, vcpu_time, ts);
224 put_cpu_var(xen_vcpu);
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700225}
226
227unsigned long xen_get_wallclock(void)
228{
229 struct timespec ts;
230
231 xen_read_wallclock(&ts);
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700232 return ts.tv_sec;
233}
234
235int xen_set_wallclock(unsigned long now)
236{
237 /* do nothing for domU */
238 return -1;
239}
240
241static struct clocksource xen_clocksource __read_mostly = {
242 .name = "xen",
243 .rating = 400,
244 .read = xen_clocksource_read,
245 .mask = ~0,
246 .mult = 1<<XEN_SHIFT, /* time directly in nanoseconds */
247 .shift = XEN_SHIFT,
248 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
249};
250
251/*
252 Xen clockevent implementation
253
254 Xen has two clockevent implementations:
255
256 The old timer_op one works with all released versions of Xen prior
257 to version 3.0.4. This version of the hypervisor provides a
258 single-shot timer with nanosecond resolution. However, sharing the
259 same event channel is a 100Hz tick which is delivered while the
260 vcpu is running. We don't care about or use this tick, but it will
261 cause the core time code to think the timer fired too soon, and
262 will end up resetting it each time. It could be filtered, but
263 doing so has complications when the ktime clocksource is not yet
264 the xen clocksource (ie, at boot time).
265
266 The new vcpu_op-based timer interface allows the tick timer period
267 to be changed or turned off. The tick timer is not useful as a
268 periodic timer because events are only delivered to running vcpus.
269 The one-shot timer can report when a timeout is in the past, so
270 set_next_event is capable of returning -ETIME when appropriate.
271 This interface is used when available.
272*/
273
274
275/*
276 Get a hypervisor absolute time. In theory we could maintain an
277 offset between the kernel's time and the hypervisor's time, and
278 apply that to a kernel's absolute timeout. Unfortunately the
279 hypervisor and kernel times can drift even if the kernel is using
280 the Xen clocksource, because ntp can warp the kernel's clocksource.
281*/
282static s64 get_abs_timeout(unsigned long delta)
283{
284 return xen_clocksource_read() + delta;
285}
286
287static void xen_timerop_set_mode(enum clock_event_mode mode,
288 struct clock_event_device *evt)
289{
290 switch (mode) {
291 case CLOCK_EVT_MODE_PERIODIC:
292 /* unsupported */
293 WARN_ON(1);
294 break;
295
296 case CLOCK_EVT_MODE_ONESHOT:
Thomas Gleixner18de5bc2007-07-21 04:37:34 -0700297 case CLOCK_EVT_MODE_RESUME:
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700298 break;
299
300 case CLOCK_EVT_MODE_UNUSED:
301 case CLOCK_EVT_MODE_SHUTDOWN:
302 HYPERVISOR_set_timer_op(0); /* cancel timeout */
303 break;
304 }
305}
306
307static int xen_timerop_set_next_event(unsigned long delta,
308 struct clock_event_device *evt)
309{
310 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
311
312 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
313 BUG();
314
315 /* We may have missed the deadline, but there's no real way of
316 knowing for sure. If the event was in the past, then we'll
317 get an immediate interrupt. */
318
319 return 0;
320}
321
322static const struct clock_event_device xen_timerop_clockevent = {
323 .name = "xen",
324 .features = CLOCK_EVT_FEAT_ONESHOT,
325
326 .max_delta_ns = 0xffffffff,
327 .min_delta_ns = TIMER_SLOP,
328
329 .mult = 1,
330 .shift = 0,
331 .rating = 500,
332
333 .set_mode = xen_timerop_set_mode,
334 .set_next_event = xen_timerop_set_next_event,
335};
336
337
338
339static void xen_vcpuop_set_mode(enum clock_event_mode mode,
340 struct clock_event_device *evt)
341{
342 int cpu = smp_processor_id();
343
344 switch (mode) {
345 case CLOCK_EVT_MODE_PERIODIC:
346 WARN_ON(1); /* unsupported */
347 break;
348
349 case CLOCK_EVT_MODE_ONESHOT:
350 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
351 BUG();
352 break;
353
354 case CLOCK_EVT_MODE_UNUSED:
355 case CLOCK_EVT_MODE_SHUTDOWN:
356 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
357 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
358 BUG();
359 break;
Thomas Gleixner18de5bc2007-07-21 04:37:34 -0700360 case CLOCK_EVT_MODE_RESUME:
361 break;
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700362 }
363}
364
365static int xen_vcpuop_set_next_event(unsigned long delta,
366 struct clock_event_device *evt)
367{
368 int cpu = smp_processor_id();
369 struct vcpu_set_singleshot_timer single;
370 int ret;
371
372 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
373
374 single.timeout_abs_ns = get_abs_timeout(delta);
375 single.flags = VCPU_SSHOTTMR_future;
376
377 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
378
379 BUG_ON(ret != 0 && ret != -ETIME);
380
381 return ret;
382}
383
384static const struct clock_event_device xen_vcpuop_clockevent = {
385 .name = "xen",
386 .features = CLOCK_EVT_FEAT_ONESHOT,
387
388 .max_delta_ns = 0xffffffff,
389 .min_delta_ns = TIMER_SLOP,
390
391 .mult = 1,
392 .shift = 0,
393 .rating = 500,
394
395 .set_mode = xen_vcpuop_set_mode,
396 .set_next_event = xen_vcpuop_set_next_event,
397};
398
399static const struct clock_event_device *xen_clockevent =
400 &xen_timerop_clockevent;
401static DEFINE_PER_CPU(struct clock_event_device, xen_clock_events);
402
403static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
404{
405 struct clock_event_device *evt = &__get_cpu_var(xen_clock_events);
406 irqreturn_t ret;
407
408 ret = IRQ_NONE;
409 if (evt->event_handler) {
410 evt->event_handler(evt);
411 ret = IRQ_HANDLED;
412 }
413
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700414 do_stolen_accounting();
415
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700416 return ret;
417}
418
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700419void xen_setup_timer(int cpu)
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700420{
421 const char *name;
422 struct clock_event_device *evt;
423 int irq;
424
425 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
426
427 name = kasprintf(GFP_KERNEL, "timer%d", cpu);
428 if (!name)
429 name = "<timer kasprintf failed>";
430
431 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
432 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
433 name, NULL);
434
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700435 evt = &per_cpu(xen_clock_events, cpu);
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700436 memcpy(evt, xen_clockevent, sizeof(*evt));
437
Rusty Russell320ab2b2008-12-13 21:20:26 +1030438 evt->cpumask = cpumask_of(cpu);
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700439 evt->irq = irq;
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700440
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700441 setup_runstate_info(cpu);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700442}
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700443
Alex Nixond68d82a2008-08-22 11:52:15 +0100444void xen_teardown_timer(int cpu)
445{
446 struct clock_event_device *evt;
447 BUG_ON(cpu == 0);
448 evt = &per_cpu(xen_clock_events, cpu);
449 unbind_from_irqhandler(evt->irq, NULL);
450}
451
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700452void xen_setup_cpu_clockevents(void)
453{
454 BUG_ON(preemptible());
455
456 clockevents_register_device(&__get_cpu_var(xen_clock_events));
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700457}
458
Jeremy Fitzhardinged07af1f2008-05-31 01:33:03 +0100459void xen_timer_resume(void)
460{
461 int cpu;
462
463 if (xen_clockevent != &xen_vcpuop_clockevent)
464 return;
465
466 for_each_online_cpu(cpu) {
467 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
468 BUG();
469 }
470}
471
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700472__init void xen_time_init(void)
473{
474 int cpu = smp_processor_id();
475
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700476 clocksource_register(&xen_clocksource);
477
478 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
Jeremy Fitzhardingef91a8b42007-07-17 18:37:05 -0700479 /* Successfully turned off 100Hz tick, so we have the
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700480 vcpuop-based timer interface */
481 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
482 xen_clockevent = &xen_vcpuop_clockevent;
483 }
484
485 /* Set initial system time with full resolution */
486 xen_read_wallclock(&xtime);
487 set_normalized_timespec(&wall_to_monotonic,
488 -xtime.tv_sec, -xtime.tv_nsec);
489
Andi Kleen404ee5b2008-01-30 13:33:20 +0100490 setup_force_cpu_cap(X86_FEATURE_TSC);
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700491
492 xen_setup_timer(cpu);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700493 xen_setup_cpu_clockevents();
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700494}