blob: d81e8d70910225d8225e53c0ee4a26ed577d32d9 [file] [log] [blame]
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001/*
2 * Core of Xen paravirt_ops implementation.
3 *
4 * This file contains the xen_paravirt_ops structure itself, and the
5 * implementations for:
6 * - privileged instructions
7 * - interrupt flags
8 * - segment operations
9 * - booting and setup
10 *
11 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
12 */
13
14#include <linux/kernel.h>
15#include <linux/init.h>
16#include <linux/smp.h>
17#include <linux/preempt.h>
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -070018#include <linux/hardirq.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070019#include <linux/percpu.h>
20#include <linux/delay.h>
21#include <linux/start_kernel.h>
22#include <linux/sched.h>
23#include <linux/bootmem.h>
24#include <linux/module.h>
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -070025#include <linux/mm.h>
26#include <linux/page-flags.h>
27#include <linux/highmem.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070028
29#include <xen/interface/xen.h>
30#include <xen/interface/physdev.h>
31#include <xen/interface/vcpu.h>
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -070032#include <xen/interface/sched.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070033#include <xen/features.h>
34#include <xen/page.h>
35
36#include <asm/paravirt.h>
37#include <asm/page.h>
38#include <asm/xen/hypercall.h>
39#include <asm/xen/hypervisor.h>
40#include <asm/fixmap.h>
41#include <asm/processor.h>
42#include <asm/setup.h>
43#include <asm/desc.h>
44#include <asm/pgtable.h>
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070045#include <asm/tlbflush.h>
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -070046#include <asm/reboot.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070047
48#include "xen-ops.h"
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070049#include "mmu.h"
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070050#include "multicalls.h"
51
52EXPORT_SYMBOL_GPL(hypercall_page);
53
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070054DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
55DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -070056
57/*
58 * Note about cr3 (pagetable base) values:
59 *
60 * xen_cr3 contains the current logical cr3 value; it contains the
61 * last set cr3. This may not be the current effective cr3, because
62 * its update may be being lazily deferred. However, a vcpu looking
63 * at its own cr3 can use this value knowing that it everything will
64 * be self-consistent.
65 *
66 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
67 * hypercall to set the vcpu cr3 is complete (so it may be a little
68 * out of date, but it will never be set early). If one vcpu is
69 * looking at another vcpu's cr3 value, it should use this variable.
70 */
71DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */
72DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070073
74struct start_info *xen_start_info;
75EXPORT_SYMBOL_GPL(xen_start_info);
76
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -070077static /* __initdata */ struct shared_info dummy_shared_info;
78
79/*
80 * Point at some empty memory to start with. We map the real shared_info
81 * page as soon as fixmap is up and running.
82 */
83struct shared_info *HYPERVISOR_shared_info = (void *)&dummy_shared_info;
84
85/*
86 * Flag to determine whether vcpu info placement is available on all
87 * VCPUs. We assume it is to start with, and then set it to zero on
88 * the first failure. This is because it can succeed on some VCPUs
89 * and not others, since it can involve hypervisor memory allocation,
90 * or because the guest failed to guarantee all the appropriate
91 * constraints on all VCPUs (ie buffer can't cross a page boundary).
92 *
93 * Note that any particular CPU may be using a placed vcpu structure,
94 * but we can only optimise if the all are.
95 *
96 * 0: not available, 1: available
97 */
Jeremy Fitzhardingef9c4cfe2008-01-23 16:07:17 -080098static int have_vcpu_info_placement = 0;
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -070099
100static void __init xen_vcpu_setup(int cpu)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700101{
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700102 struct vcpu_register_vcpu_info info;
103 int err;
104 struct vcpu_info *vcpup;
105
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700106 per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700107
108 if (!have_vcpu_info_placement)
109 return; /* already tested, not available */
110
111 vcpup = &per_cpu(xen_vcpu_info, cpu);
112
113 info.mfn = virt_to_mfn(vcpup);
114 info.offset = offset_in_page(vcpup);
115
Jeremy Fitzhardingee3d26972007-10-16 11:51:31 -0700116 printk(KERN_DEBUG "trying to map vcpu_info %d at %p, mfn %llx, offset %d\n",
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700117 cpu, vcpup, info.mfn, info.offset);
118
119 /* Check to see if the hypervisor will put the vcpu_info
120 structure where we want it, which allows direct access via
121 a percpu-variable. */
122 err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);
123
124 if (err) {
125 printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
126 have_vcpu_info_placement = 0;
127 } else {
128 /* This cpu is using the registered vcpu info, even if
129 later ones fail to. */
130 per_cpu(xen_vcpu, cpu) = vcpup;
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700131
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700132 printk(KERN_DEBUG "cpu %d using vcpu_info at %p\n",
133 cpu, vcpup);
134 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700135}
136
137static void __init xen_banner(void)
138{
139 printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700140 pv_info.name);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700141 printk(KERN_INFO "Hypervisor signature: %s\n", xen_start_info->magic);
142}
143
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100144static void xen_cpuid(unsigned int *ax, unsigned int *bx,
145 unsigned int *cx, unsigned int *dx)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700146{
147 unsigned maskedx = ~0;
148
149 /*
150 * Mask out inconvenient features, to try and disable as many
151 * unsupported kernel subsystems as possible.
152 */
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100153 if (*ax == 1)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700154 maskedx = ~((1 << X86_FEATURE_APIC) | /* disable APIC */
155 (1 << X86_FEATURE_ACPI) | /* disable ACPI */
156 (1 << X86_FEATURE_ACC)); /* thermal monitoring */
157
158 asm(XEN_EMULATE_PREFIX "cpuid"
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100159 : "=a" (*ax),
160 "=b" (*bx),
161 "=c" (*cx),
162 "=d" (*dx)
163 : "0" (*ax), "2" (*cx));
164 *dx &= maskedx;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700165}
166
167static void xen_set_debugreg(int reg, unsigned long val)
168{
169 HYPERVISOR_set_debugreg(reg, val);
170}
171
172static unsigned long xen_get_debugreg(int reg)
173{
174 return HYPERVISOR_get_debugreg(reg);
175}
176
177static unsigned long xen_save_fl(void)
178{
179 struct vcpu_info *vcpu;
180 unsigned long flags;
181
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700182 vcpu = x86_read_percpu(xen_vcpu);
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700183
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700184 /* flag has opposite sense of mask */
185 flags = !vcpu->evtchn_upcall_mask;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700186
187 /* convert to IF type flag
188 -0 -> 0x00000000
189 -1 -> 0xffffffff
190 */
191 return (-flags) & X86_EFLAGS_IF;
192}
193
194static void xen_restore_fl(unsigned long flags)
195{
196 struct vcpu_info *vcpu;
197
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700198 /* convert from IF type flag */
199 flags = !(flags & X86_EFLAGS_IF);
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700200
201 /* There's a one instruction preempt window here. We need to
202 make sure we're don't switch CPUs between getting the vcpu
203 pointer and updating the mask. */
204 preempt_disable();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700205 vcpu = x86_read_percpu(xen_vcpu);
206 vcpu->evtchn_upcall_mask = flags;
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700207 preempt_enable_no_resched();
208
209 /* Doesn't matter if we get preempted here, because any
210 pending event will get dealt with anyway. */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700211
212 if (flags == 0) {
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700213 preempt_check_resched();
214 barrier(); /* unmask then check (avoid races) */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700215 if (unlikely(vcpu->evtchn_upcall_pending))
216 force_evtchn_callback();
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700217 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700218}
219
220static void xen_irq_disable(void)
221{
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700222 /* There's a one instruction preempt window here. We need to
223 make sure we're don't switch CPUs between getting the vcpu
224 pointer and updating the mask. */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700225 preempt_disable();
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700226 x86_read_percpu(xen_vcpu)->evtchn_upcall_mask = 1;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700227 preempt_enable_no_resched();
228}
229
230static void xen_irq_enable(void)
231{
232 struct vcpu_info *vcpu;
233
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700234 /* There's a one instruction preempt window here. We need to
235 make sure we're don't switch CPUs between getting the vcpu
236 pointer and updating the mask. */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700237 preempt_disable();
238 vcpu = x86_read_percpu(xen_vcpu);
239 vcpu->evtchn_upcall_mask = 0;
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700240 preempt_enable_no_resched();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700241
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700242 /* Doesn't matter if we get preempted here, because any
243 pending event will get dealt with anyway. */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700244
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700245 barrier(); /* unmask then check (avoid races) */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700246 if (unlikely(vcpu->evtchn_upcall_pending))
247 force_evtchn_callback();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700248}
249
250static void xen_safe_halt(void)
251{
252 /* Blocking includes an implicit local_irq_enable(). */
253 if (HYPERVISOR_sched_op(SCHEDOP_block, 0) != 0)
254 BUG();
255}
256
257static void xen_halt(void)
258{
259 if (irqs_disabled())
260 HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
261 else
262 xen_safe_halt();
263}
264
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700265static void xen_leave_lazy(void)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700266{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700267 paravirt_leave_lazy(paravirt_get_lazy_mode());
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700268 xen_mc_flush();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700269}
270
271static unsigned long xen_store_tr(void)
272{
273 return 0;
274}
275
276static void xen_set_ldt(const void *addr, unsigned entries)
277{
278 unsigned long linear_addr = (unsigned long)addr;
279 struct mmuext_op *op;
280 struct multicall_space mcs = xen_mc_entry(sizeof(*op));
281
282 op = mcs.args;
283 op->cmd = MMUEXT_SET_LDT;
284 if (linear_addr) {
285 /* ldt my be vmalloced, use arbitrary_virt_to_machine */
286 xmaddr_t maddr;
287 maddr = arbitrary_virt_to_machine((unsigned long)addr);
288 linear_addr = (unsigned long)maddr.maddr;
289 }
290 op->arg1.linear_addr = linear_addr;
291 op->arg2.nr_ents = entries;
292
293 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
294
295 xen_mc_issue(PARAVIRT_LAZY_CPU);
296}
297
298static void xen_load_gdt(const struct Xgt_desc_struct *dtr)
299{
300 unsigned long *frames;
301 unsigned long va = dtr->address;
302 unsigned int size = dtr->size + 1;
303 unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
304 int f;
305 struct multicall_space mcs;
306
307 /* A GDT can be up to 64k in size, which corresponds to 8192
308 8-byte entries, or 16 4k pages.. */
309
310 BUG_ON(size > 65536);
311 BUG_ON(va & ~PAGE_MASK);
312
313 mcs = xen_mc_entry(sizeof(*frames) * pages);
314 frames = mcs.args;
315
316 for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
317 frames[f] = virt_to_mfn(va);
318 make_lowmem_page_readonly((void *)va);
319 }
320
321 MULTI_set_gdt(mcs.mc, frames, size / sizeof(struct desc_struct));
322
323 xen_mc_issue(PARAVIRT_LAZY_CPU);
324}
325
326static void load_TLS_descriptor(struct thread_struct *t,
327 unsigned int cpu, unsigned int i)
328{
329 struct desc_struct *gdt = get_cpu_gdt_table(cpu);
330 xmaddr_t maddr = virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
331 struct multicall_space mc = __xen_mc_entry(0);
332
333 MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
334}
335
336static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
337{
338 xen_mc_batch();
339
340 load_TLS_descriptor(t, cpu, 0);
341 load_TLS_descriptor(t, cpu, 1);
342 load_TLS_descriptor(t, cpu, 2);
343
344 xen_mc_issue(PARAVIRT_LAZY_CPU);
Jeremy Fitzhardinge8b84ad92007-07-17 18:37:06 -0700345
346 /*
347 * XXX sleazy hack: If we're being called in a lazy-cpu zone,
348 * it means we're in a context switch, and %gs has just been
349 * saved. This means we can zero it out to prevent faults on
350 * exit from the hypervisor if the next process has no %gs.
351 * Either way, it has been saved, and the new value will get
352 * loaded properly. This will go away as soon as Xen has been
353 * modified to not save/restore %gs for normal hypercalls.
354 */
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700355 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU)
Jeremy Fitzhardinge8b84ad92007-07-17 18:37:06 -0700356 loadsegment(gs, 0);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700357}
358
359static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
360 u32 low, u32 high)
361{
362 unsigned long lp = (unsigned long)&dt[entrynum];
363 xmaddr_t mach_lp = virt_to_machine(lp);
364 u64 entry = (u64)high << 32 | low;
365
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700366 preempt_disable();
367
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700368 xen_mc_flush();
369 if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
370 BUG();
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700371
372 preempt_enable();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700373}
374
375static int cvt_gate_to_trap(int vector, u32 low, u32 high,
376 struct trap_info *info)
377{
378 u8 type, dpl;
379
380 type = (high >> 8) & 0x1f;
381 dpl = (high >> 13) & 3;
382
383 if (type != 0xf && type != 0xe)
384 return 0;
385
386 info->vector = vector;
387 info->address = (high & 0xffff0000) | (low & 0x0000ffff);
388 info->cs = low >> 16;
389 info->flags = dpl;
390 /* interrupt gates clear IF */
391 if (type == 0xe)
392 info->flags |= 4;
393
394 return 1;
395}
396
397/* Locations of each CPU's IDT */
398static DEFINE_PER_CPU(struct Xgt_desc_struct, idt_desc);
399
400/* Set an IDT entry. If the entry is part of the current IDT, then
401 also update Xen. */
402static void xen_write_idt_entry(struct desc_struct *dt, int entrynum,
403 u32 low, u32 high)
404{
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700405 unsigned long p = (unsigned long)&dt[entrynum];
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700406 unsigned long start, end;
407
408 preempt_disable();
409
410 start = __get_cpu_var(idt_desc).address;
411 end = start + __get_cpu_var(idt_desc).size + 1;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700412
413 xen_mc_flush();
414
415 write_dt_entry(dt, entrynum, low, high);
416
417 if (p >= start && (p + 8) <= end) {
418 struct trap_info info[2];
419
420 info[1].address = 0;
421
422 if (cvt_gate_to_trap(entrynum, low, high, &info[0]))
423 if (HYPERVISOR_set_trap_table(info))
424 BUG();
425 }
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700426
427 preempt_enable();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700428}
429
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700430static void xen_convert_trap_info(const struct Xgt_desc_struct *desc,
431 struct trap_info *traps)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700432{
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700433 unsigned in, out, count;
434
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700435 count = (desc->size+1) / 8;
436 BUG_ON(count > 256);
437
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700438 for (in = out = 0; in < count; in++) {
439 const u32 *entry = (u32 *)(desc->address + in * 8);
440
441 if (cvt_gate_to_trap(in, entry[0], entry[1], &traps[out]))
442 out++;
443 }
444 traps[out].address = 0;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700445}
446
447void xen_copy_trap_info(struct trap_info *traps)
448{
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700449 const struct Xgt_desc_struct *desc = &__get_cpu_var(idt_desc);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700450
451 xen_convert_trap_info(desc, traps);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700452}
453
454/* Load a new IDT into Xen. In principle this can be per-CPU, so we
455 hold a spinlock to protect the static traps[] array (static because
456 it avoids allocation, and saves stack space). */
457static void xen_load_idt(const struct Xgt_desc_struct *desc)
458{
459 static DEFINE_SPINLOCK(lock);
460 static struct trap_info traps[257];
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700461
462 spin_lock(&lock);
463
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700464 __get_cpu_var(idt_desc) = *desc;
465
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700466 xen_convert_trap_info(desc, traps);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700467
468 xen_mc_flush();
469 if (HYPERVISOR_set_trap_table(traps))
470 BUG();
471
472 spin_unlock(&lock);
473}
474
475/* Write a GDT descriptor entry. Ignore LDT descriptors, since
476 they're handled differently. */
477static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
478 u32 low, u32 high)
479{
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700480 preempt_disable();
481
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700482 switch ((high >> 8) & 0xff) {
483 case DESCTYPE_LDT:
484 case DESCTYPE_TSS:
485 /* ignore */
486 break;
487
488 default: {
489 xmaddr_t maddr = virt_to_machine(&dt[entry]);
490 u64 desc = (u64)high << 32 | low;
491
492 xen_mc_flush();
493 if (HYPERVISOR_update_descriptor(maddr.maddr, desc))
494 BUG();
495 }
496
497 }
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700498
499 preempt_enable();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700500}
501
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100502static void xen_load_sp0(struct tss_struct *tss,
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700503 struct thread_struct *thread)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700504{
505 struct multicall_space mcs = xen_mc_entry(0);
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100506 MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700507 xen_mc_issue(PARAVIRT_LAZY_CPU);
508}
509
510static void xen_set_iopl_mask(unsigned mask)
511{
512 struct physdev_set_iopl set_iopl;
513
514 /* Force the change at ring 0. */
515 set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
516 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
517}
518
519static void xen_io_delay(void)
520{
521}
522
523#ifdef CONFIG_X86_LOCAL_APIC
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100524static u32 xen_apic_read(unsigned long reg)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700525{
526 return 0;
527}
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700528
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100529static void xen_apic_write(unsigned long reg, u32 val)
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700530{
531 /* Warn to see if there's any stray references */
532 WARN_ON(1);
533}
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700534#endif
535
536static void xen_flush_tlb(void)
537{
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700538 struct mmuext_op *op;
539 struct multicall_space mcs = xen_mc_entry(sizeof(*op));
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700540
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700541 op = mcs.args;
542 op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
543 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
544
545 xen_mc_issue(PARAVIRT_LAZY_MMU);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700546}
547
548static void xen_flush_tlb_single(unsigned long addr)
549{
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700550 struct mmuext_op *op;
551 struct multicall_space mcs = xen_mc_entry(sizeof(*op));
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700552
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700553 op = mcs.args;
554 op->cmd = MMUEXT_INVLPG_LOCAL;
555 op->arg1.linear_addr = addr & PAGE_MASK;
556 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
557
558 xen_mc_issue(PARAVIRT_LAZY_MMU);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700559}
560
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700561static void xen_flush_tlb_others(const cpumask_t *cpus, struct mm_struct *mm,
562 unsigned long va)
563{
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700564 struct {
565 struct mmuext_op op;
566 cpumask_t mask;
567 } *args;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700568 cpumask_t cpumask = *cpus;
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700569 struct multicall_space mcs;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700570
571 /*
572 * A couple of (to be removed) sanity checks:
573 *
574 * - current CPU must not be in mask
575 * - mask must exist :)
576 */
577 BUG_ON(cpus_empty(cpumask));
578 BUG_ON(cpu_isset(smp_processor_id(), cpumask));
579 BUG_ON(!mm);
580
581 /* If a CPU which we ran on has gone down, OK. */
582 cpus_and(cpumask, cpumask, cpu_online_map);
583 if (cpus_empty(cpumask))
584 return;
585
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700586 mcs = xen_mc_entry(sizeof(*args));
587 args = mcs.args;
588 args->mask = cpumask;
589 args->op.arg2.vcpumask = &args->mask;
590
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700591 if (va == TLB_FLUSH_ALL) {
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700592 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700593 } else {
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700594 args->op.cmd = MMUEXT_INVLPG_MULTI;
595 args->op.arg1.linear_addr = va;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700596 }
597
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700598 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
599
600 xen_mc_issue(PARAVIRT_LAZY_MMU);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700601}
602
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700603static void xen_write_cr2(unsigned long cr2)
604{
605 x86_read_percpu(xen_vcpu)->arch.cr2 = cr2;
606}
607
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700608static unsigned long xen_read_cr2(void)
609{
610 return x86_read_percpu(xen_vcpu)->arch.cr2;
611}
612
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700613static unsigned long xen_read_cr2_direct(void)
614{
615 return x86_read_percpu(xen_vcpu_info.arch.cr2);
616}
617
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700618static void xen_write_cr4(unsigned long cr4)
619{
Jeremy Fitzhardinge389a3c02007-09-18 22:46:33 -0700620 /* Just ignore cr4 changes; Xen doesn't allow us to do
621 anything anyway. */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700622}
623
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700624static unsigned long xen_read_cr3(void)
625{
626 return x86_read_percpu(xen_cr3);
627}
628
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -0700629static void set_current_cr3(void *v)
630{
631 x86_write_percpu(xen_current_cr3, (unsigned long)v);
632}
633
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700634static void xen_write_cr3(unsigned long cr3)
635{
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -0700636 struct mmuext_op *op;
637 struct multicall_space mcs;
638 unsigned long mfn = pfn_to_mfn(PFN_DOWN(cr3));
639
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700640 BUG_ON(preemptible());
641
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -0700642 mcs = xen_mc_entry(sizeof(*op)); /* disables interrupts */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700643
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -0700644 /* Update while interrupts are disabled, so its atomic with
645 respect to ipis */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700646 x86_write_percpu(xen_cr3, cr3);
647
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -0700648 op = mcs.args;
649 op->cmd = MMUEXT_NEW_BASEPTR;
650 op->arg1.mfn = mfn;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700651
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -0700652 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700653
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -0700654 /* Update xen_update_cr3 once the batch has actually
655 been submitted. */
656 xen_mc_callback(set_current_cr3, (void *)cr3);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700657
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -0700658 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700659}
660
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700661/* Early in boot, while setting up the initial pagetable, assume
662 everything is pinned. */
Jeremy Fitzhardinge9a4029f2007-07-17 18:37:05 -0700663static __init void xen_alloc_pt_init(struct mm_struct *mm, u32 pfn)
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700664{
665 BUG_ON(mem_map); /* should only be used early */
666 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
667}
668
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700669static void pin_pagetable_pfn(unsigned level, unsigned long pfn)
670{
671 struct mmuext_op op;
672 op.cmd = level;
673 op.arg1.mfn = pfn_to_mfn(pfn);
674 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
675 BUG();
676}
677
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700678/* This needs to make sure the new pte page is pinned iff its being
679 attached to a pinned pagetable. */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700680static void xen_alloc_pt(struct mm_struct *mm, u32 pfn)
681{
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700682 struct page *page = pfn_to_page(pfn);
683
684 if (PagePinned(virt_to_page(mm->pgd))) {
685 SetPagePinned(page);
686
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700687 if (!PageHighMem(page)) {
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700688 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700689 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
690 } else
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700691 /* make sure there are no stray mappings of
692 this page */
693 kmap_flush_unused();
694 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700695}
696
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700697/* This should never happen until we're OK to use struct page */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700698static void xen_release_pt(u32 pfn)
699{
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700700 struct page *page = pfn_to_page(pfn);
701
702 if (PagePinned(page)) {
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700703 if (!PageHighMem(page)) {
704 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700705 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700706 }
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700707 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700708}
709
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700710#ifdef CONFIG_HIGHPTE
711static void *xen_kmap_atomic_pte(struct page *page, enum km_type type)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700712{
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700713 pgprot_t prot = PAGE_KERNEL;
714
715 if (PagePinned(page))
716 prot = PAGE_KERNEL_RO;
717
718 if (0 && PageHighMem(page))
719 printk("mapping highpte %lx type %d prot %s\n",
720 page_to_pfn(page), type,
721 (unsigned long)pgprot_val(prot) & _PAGE_RW ? "WRITE" : "READ");
722
723 return kmap_atomic_prot(page, type, prot);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700724}
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700725#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700726
Jeremy Fitzhardinge9a4029f2007-07-17 18:37:05 -0700727static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte)
728{
729 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
730 if (pte_val_ma(*ptep) & _PAGE_PRESENT)
731 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
732 pte_val_ma(pte));
733
734 return pte;
735}
736
737/* Init-time set_pte while constructing initial pagetables, which
738 doesn't allow RO pagetable pages to be remapped RW */
739static __init void xen_set_pte_init(pte_t *ptep, pte_t pte)
740{
741 pte = mask_rw_pte(ptep, pte);
742
743 xen_set_pte(ptep, pte);
744}
745
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700746static __init void xen_pagetable_setup_start(pgd_t *base)
747{
748 pgd_t *xen_pgd = (pgd_t *)xen_start_info->pt_base;
749
Jeremy Fitzhardinge9a4029f2007-07-17 18:37:05 -0700750 /* special set_pte for pagetable initialization */
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700751 pv_mmu_ops.set_pte = xen_set_pte_init;
Jeremy Fitzhardinge9a4029f2007-07-17 18:37:05 -0700752
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700753 init_mm.pgd = base;
754 /*
755 * copy top-level of Xen-supplied pagetable into place. For
756 * !PAE we can use this as-is, but for PAE it is a stand-in
757 * while we copy the pmd pages.
758 */
759 memcpy(base, xen_pgd, PTRS_PER_PGD * sizeof(pgd_t));
760
761 if (PTRS_PER_PMD > 1) {
762 int i;
763 /*
764 * For PAE, need to allocate new pmds, rather than
765 * share Xen's, since Xen doesn't like pmd's being
766 * shared between address spaces.
767 */
768 for (i = 0; i < PTRS_PER_PGD; i++) {
769 if (pgd_val_ma(xen_pgd[i]) & _PAGE_PRESENT) {
770 pmd_t *pmd = (pmd_t *)alloc_bootmem_low_pages(PAGE_SIZE);
771
772 memcpy(pmd, (void *)pgd_page_vaddr(xen_pgd[i]),
773 PAGE_SIZE);
774
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700775 make_lowmem_page_readonly(pmd);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700776
777 set_pgd(&base[i], __pgd(1 + __pa(pmd)));
778 } else
779 pgd_clear(&base[i]);
780 }
781 }
782
783 /* make sure zero_page is mapped RO so we can use it in pagetables */
784 make_lowmem_page_readonly(empty_zero_page);
785 make_lowmem_page_readonly(base);
786 /*
787 * Switch to new pagetable. This is done before
788 * pagetable_init has done anything so that the new pages
789 * added to the table can be prepared properly for Xen.
790 */
791 xen_write_cr3(__pa(base));
792}
793
794static __init void xen_pagetable_setup_done(pgd_t *base)
795{
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700796 /* This will work as long as patching hasn't happened yet
797 (which it hasn't) */
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700798 pv_mmu_ops.alloc_pt = xen_alloc_pt;
799 pv_mmu_ops.set_pte = xen_set_pte;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700800
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700801 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
802 /*
803 * Create a mapping for the shared info page.
804 * Should be set_fixmap(), but shared_info is a machine
805 * address with no corresponding pseudo-phys address.
806 */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700807 set_pte_mfn(fix_to_virt(FIX_PARAVIRT_BOOTMAP),
808 PFN_DOWN(xen_start_info->shared_info),
809 PAGE_KERNEL);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700810
811 HYPERVISOR_shared_info =
812 (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
813
814 } else
815 HYPERVISOR_shared_info =
816 (struct shared_info *)__va(xen_start_info->shared_info);
817
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700818 /* Actually pin the pagetable down, but we can't set PG_pinned
819 yet because the page structures don't exist yet. */
820 {
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700821 unsigned level;
822
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700823#ifdef CONFIG_X86_PAE
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700824 level = MMUEXT_PIN_L3_TABLE;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700825#else
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700826 level = MMUEXT_PIN_L2_TABLE;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700827#endif
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700828
829 pin_pagetable_pfn(level, PFN_DOWN(__pa(base)));
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700830 }
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700831}
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700832
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700833/* This is called once we have the cpu_possible_map */
834void __init xen_setup_vcpu_info_placement(void)
835{
836 int cpu;
837
838 for_each_possible_cpu(cpu)
839 xen_vcpu_setup(cpu);
840
841 /* xen_vcpu_setup managed to place the vcpu_info within the
842 percpu area for all cpus, so make use of it */
843 if (have_vcpu_info_placement) {
844 printk(KERN_INFO "Xen: using vcpu_info placement\n");
845
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700846 pv_irq_ops.save_fl = xen_save_fl_direct;
847 pv_irq_ops.restore_fl = xen_restore_fl_direct;
848 pv_irq_ops.irq_disable = xen_irq_disable_direct;
849 pv_irq_ops.irq_enable = xen_irq_enable_direct;
850 pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
851 pv_cpu_ops.iret = xen_iret_direct;
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700852 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700853}
854
Andi Kleenab144f52007-08-10 22:31:03 +0200855static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
856 unsigned long addr, unsigned len)
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700857{
858 char *start, *end, *reloc;
859 unsigned ret;
860
861 start = end = reloc = NULL;
862
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700863#define SITE(op, x) \
864 case PARAVIRT_PATCH(op.x): \
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700865 if (have_vcpu_info_placement) { \
866 start = (char *)xen_##x##_direct; \
867 end = xen_##x##_direct_end; \
868 reloc = xen_##x##_direct_reloc; \
869 } \
870 goto patch_site
871
872 switch (type) {
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700873 SITE(pv_irq_ops, irq_enable);
874 SITE(pv_irq_ops, irq_disable);
875 SITE(pv_irq_ops, save_fl);
876 SITE(pv_irq_ops, restore_fl);
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700877#undef SITE
878
879 patch_site:
880 if (start == NULL || (end-start) > len)
881 goto default_patch;
882
Andi Kleenab144f52007-08-10 22:31:03 +0200883 ret = paravirt_patch_insns(insnbuf, len, start, end);
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700884
885 /* Note: because reloc is assigned from something that
886 appears to be an array, gcc assumes it's non-null,
887 but doesn't know its relationship with start and
888 end. */
889 if (reloc > start && reloc < end) {
890 int reloc_off = reloc - start;
Andi Kleenab144f52007-08-10 22:31:03 +0200891 long *relocp = (long *)(insnbuf + reloc_off);
892 long delta = start - (char *)addr;
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700893
894 *relocp += delta;
895 }
896 break;
897
898 default_patch:
899 default:
Andi Kleenab144f52007-08-10 22:31:03 +0200900 ret = paravirt_patch_default(type, clobbers, insnbuf,
901 addr, len);
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700902 break;
903 }
904
905 return ret;
906}
907
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700908static const struct pv_info xen_info __initdata = {
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700909 .paravirt_enabled = 1,
910 .shared_kernel_pmd = 0,
911
912 .name = "Xen",
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700913};
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700914
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700915static const struct pv_init_ops xen_init_ops __initdata = {
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700916 .patch = xen_patch,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700917
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700918 .banner = xen_banner,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700919 .memory_setup = xen_memory_setup,
920 .arch_setup = xen_arch_setup,
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700921 .post_allocator_init = xen_mark_init_mm_pinned,
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700922};
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700923
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700924static const struct pv_time_ops xen_time_ops __initdata = {
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700925 .time_init = xen_time_init,
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700926
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700927 .set_wallclock = xen_set_wallclock,
928 .get_wallclock = xen_get_wallclock,
929 .get_cpu_khz = xen_cpu_khz,
Jeremy Fitzhardingeab550282007-07-17 18:37:05 -0700930 .sched_clock = xen_sched_clock,
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700931};
Jeremy Fitzhardinge15c84732007-07-17 18:37:05 -0700932
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700933static const struct pv_cpu_ops xen_cpu_ops __initdata = {
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700934 .cpuid = xen_cpuid,
935
936 .set_debugreg = xen_set_debugreg,
937 .get_debugreg = xen_get_debugreg,
938
939 .clts = native_clts,
940
941 .read_cr0 = native_read_cr0,
942 .write_cr0 = native_write_cr0,
943
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700944 .read_cr4 = native_read_cr4,
945 .read_cr4_safe = native_read_cr4_safe,
946 .write_cr4 = xen_write_cr4,
947
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700948 .wbinvd = native_wbinvd,
949
950 .read_msr = native_read_msr_safe,
951 .write_msr = native_write_msr_safe,
952 .read_tsc = native_read_tsc,
953 .read_pmc = native_read_pmc,
954
955 .iret = (void *)&hypercall_page[__HYPERVISOR_iret],
Glauber de Oliveira Costa6abcd982008-01-30 13:30:33 +0100956 .irq_enable_syscall_ret = NULL, /* never called */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700957
958 .load_tr_desc = paravirt_nop,
959 .set_ldt = xen_set_ldt,
960 .load_gdt = xen_load_gdt,
961 .load_idt = xen_load_idt,
962 .load_tls = xen_load_tls,
963
964 .store_gdt = native_store_gdt,
965 .store_idt = native_store_idt,
966 .store_tr = xen_store_tr,
967
968 .write_ldt_entry = xen_write_ldt_entry,
969 .write_gdt_entry = xen_write_gdt_entry,
970 .write_idt_entry = xen_write_idt_entry,
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100971 .load_sp0 = xen_load_sp0,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700972
973 .set_iopl_mask = xen_set_iopl_mask,
974 .io_delay = xen_io_delay,
975
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700976 .lazy_mode = {
977 .enter = paravirt_enter_lazy_cpu,
978 .leave = xen_leave_lazy,
979 },
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700980};
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700981
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700982static const struct pv_irq_ops xen_irq_ops __initdata = {
983 .init_IRQ = xen_init_IRQ,
984 .save_fl = xen_save_fl,
985 .restore_fl = xen_restore_fl,
986 .irq_disable = xen_irq_disable,
987 .irq_enable = xen_irq_enable,
988 .safe_halt = xen_safe_halt,
989 .halt = xen_halt,
990};
991
992static const struct pv_apic_ops xen_apic_ops __initdata = {
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700993#ifdef CONFIG_X86_LOCAL_APIC
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700994 .apic_write = xen_apic_write,
995 .apic_write_atomic = xen_apic_write,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700996 .apic_read = xen_apic_read,
997 .setup_boot_clock = paravirt_nop,
998 .setup_secondary_clock = paravirt_nop,
999 .startup_ipi_hook = paravirt_nop,
1000#endif
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001001};
1002
1003static const struct pv_mmu_ops xen_mmu_ops __initdata = {
1004 .pagetable_setup_start = xen_pagetable_setup_start,
1005 .pagetable_setup_done = xen_pagetable_setup_done,
1006
1007 .read_cr2 = xen_read_cr2,
1008 .write_cr2 = xen_write_cr2,
1009
1010 .read_cr3 = xen_read_cr3,
1011 .write_cr3 = xen_write_cr3,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001012
1013 .flush_tlb_user = xen_flush_tlb,
1014 .flush_tlb_kernel = xen_flush_tlb,
1015 .flush_tlb_single = xen_flush_tlb_single,
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001016 .flush_tlb_others = xen_flush_tlb_others,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001017
1018 .pte_update = paravirt_nop,
1019 .pte_update_defer = paravirt_nop,
1020
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001021 .alloc_pt = xen_alloc_pt_init,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001022 .release_pt = xen_release_pt,
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001023 .alloc_pd = paravirt_nop,
1024 .alloc_pd_clone = paravirt_nop,
1025 .release_pd = paravirt_nop,
1026
1027#ifdef CONFIG_HIGHPTE
1028 .kmap_atomic_pte = xen_kmap_atomic_pte,
1029#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001030
Jeremy Fitzhardinge9a4029f2007-07-17 18:37:05 -07001031 .set_pte = NULL, /* see xen_pagetable_setup_* */
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001032 .set_pte_at = xen_set_pte_at,
1033 .set_pmd = xen_set_pmd,
1034
1035 .pte_val = xen_pte_val,
1036 .pgd_val = xen_pgd_val,
1037
1038 .make_pte = xen_make_pte,
1039 .make_pgd = xen_make_pgd,
1040
1041#ifdef CONFIG_X86_PAE
1042 .set_pte_atomic = xen_set_pte_atomic,
1043 .set_pte_present = xen_set_pte_at,
1044 .set_pud = xen_set_pud,
1045 .pte_clear = xen_pte_clear,
1046 .pmd_clear = xen_pmd_clear,
1047
1048 .make_pmd = xen_make_pmd,
1049 .pmd_val = xen_pmd_val,
1050#endif /* PAE */
1051
1052 .activate_mm = xen_activate_mm,
1053 .dup_mmap = xen_dup_mmap,
1054 .exit_mmap = xen_exit_mmap,
1055
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001056 .lazy_mode = {
1057 .enter = paravirt_enter_lazy_mmu,
1058 .leave = xen_leave_lazy,
1059 },
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001060};
1061
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001062#ifdef CONFIG_SMP
1063static const struct smp_ops xen_smp_ops __initdata = {
1064 .smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
1065 .smp_prepare_cpus = xen_smp_prepare_cpus,
1066 .cpu_up = xen_cpu_up,
1067 .smp_cpus_done = xen_smp_cpus_done,
1068
1069 .smp_send_stop = xen_smp_send_stop,
1070 .smp_send_reschedule = xen_smp_send_reschedule,
1071 .smp_call_function_mask = xen_smp_call_function_mask,
1072};
1073#endif /* CONFIG_SMP */
1074
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -07001075static void xen_reboot(int reason)
1076{
1077#ifdef CONFIG_SMP
1078 smp_send_stop();
1079#endif
1080
1081 if (HYPERVISOR_sched_op(SCHEDOP_shutdown, reason))
1082 BUG();
1083}
1084
1085static void xen_restart(char *msg)
1086{
1087 xen_reboot(SHUTDOWN_reboot);
1088}
1089
1090static void xen_emergency_restart(void)
1091{
1092 xen_reboot(SHUTDOWN_reboot);
1093}
1094
1095static void xen_machine_halt(void)
1096{
1097 xen_reboot(SHUTDOWN_poweroff);
1098}
1099
1100static void xen_crash_shutdown(struct pt_regs *regs)
1101{
1102 xen_reboot(SHUTDOWN_crash);
1103}
1104
1105static const struct machine_ops __initdata xen_machine_ops = {
1106 .restart = xen_restart,
1107 .halt = xen_machine_halt,
1108 .power_off = xen_machine_halt,
1109 .shutdown = xen_machine_halt,
1110 .crash_shutdown = xen_crash_shutdown,
1111 .emergency_restart = xen_emergency_restart,
1112};
1113
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -07001114
Jeremy Fitzhardingefb1d8402007-10-16 11:51:31 -07001115static void __init xen_reserve_top(void)
1116{
1117 unsigned long top = HYPERVISOR_VIRT_START;
1118 struct xen_platform_parameters pp;
1119
1120 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1121 top = pp.virt_start;
1122
1123 reserve_top_address(-top + 2 * PAGE_SIZE);
1124}
1125
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001126/* First C function to be called on Xen boot */
1127asmlinkage void __init xen_start_kernel(void)
1128{
1129 pgd_t *pgd;
1130
1131 if (!xen_start_info)
1132 return;
1133
Jeremy Fitzhardinge7999f4b2007-12-10 13:00:41 -08001134 BUG_ON(memcmp(xen_start_info->magic, "xen-3", 5) != 0);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001135
1136 /* Install Xen paravirt ops */
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001137 pv_info = xen_info;
1138 pv_init_ops = xen_init_ops;
1139 pv_time_ops = xen_time_ops;
1140 pv_cpu_ops = xen_cpu_ops;
1141 pv_irq_ops = xen_irq_ops;
1142 pv_apic_ops = xen_apic_ops;
1143 pv_mmu_ops = xen_mmu_ops;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001144
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -07001145 machine_ops = xen_machine_ops;
1146
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001147#ifdef CONFIG_SMP
1148 smp_ops = xen_smp_ops;
1149#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001150
1151 xen_setup_features();
1152
1153 /* Get mfn list */
1154 if (!xen_feature(XENFEAT_auto_translated_physmap))
1155 phys_to_machine_mapping = (unsigned long *)xen_start_info->mfn_list;
1156
1157 pgd = (pgd_t *)xen_start_info->pt_base;
1158
1159 init_pg_tables_end = __pa(pgd) + xen_start_info->nr_pt_frames*PAGE_SIZE;
1160
1161 init_mm.pgd = pgd; /* use the Xen pagetables to start */
1162
1163 /* keep using Xen gdt for now; no urgent need to change it */
1164
1165 x86_write_percpu(xen_cr3, __pa(pgd));
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -07001166 x86_write_percpu(xen_current_cr3, __pa(pgd));
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -07001167
1168#ifdef CONFIG_SMP
1169 /* Don't do the full vcpu_info placement stuff until we have a
1170 possible map. */
1171 per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1172#else
1173 /* May as well do it now, since there's no good time to call
1174 it later on UP. */
1175 xen_setup_vcpu_info_placement();
1176#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001177
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001178 pv_info.kernel_rpl = 1;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001179 if (xen_feature(XENFEAT_supervisor_mode_kernel))
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001180 pv_info.kernel_rpl = 0;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001181
1182 /* set the limit of our address space */
Jeremy Fitzhardingefb1d8402007-10-16 11:51:31 -07001183 xen_reserve_top();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001184
1185 /* set up basic CPUID stuff */
1186 cpu_detect(&new_cpu_data);
1187 new_cpu_data.hard_math = 1;
1188 new_cpu_data.x86_capability[0] = cpuid_edx(1);
1189
1190 /* Poke various useful things into boot_params */
H. Peter Anvin30c82642007-10-15 17:13:22 -07001191 boot_params.hdr.type_of_loader = (9 << 4) | 0;
1192 boot_params.hdr.ramdisk_image = xen_start_info->mod_start
1193 ? __pa(xen_start_info->mod_start) : 0;
1194 boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001195
1196 /* Start the world */
1197 start_kernel();
1198}