blob: 02c710bebf7a3d911b639c07e17bb1187fd719eb [file] [log] [blame]
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001/*
2 * Core of Xen paravirt_ops implementation.
3 *
4 * This file contains the xen_paravirt_ops structure itself, and the
5 * implementations for:
6 * - privileged instructions
7 * - interrupt flags
8 * - segment operations
9 * - booting and setup
10 *
11 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
12 */
13
Sheng Yang38e20b02010-05-14 12:40:51 +010014#include <linux/cpu.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070015#include <linux/kernel.h>
16#include <linux/init.h>
17#include <linux/smp.h>
18#include <linux/preempt.h>
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -070019#include <linux/hardirq.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070020#include <linux/percpu.h>
21#include <linux/delay.h>
22#include <linux/start_kernel.h>
23#include <linux/sched.h>
Jeremy Fitzhardinge6cac5a92009-03-29 19:56:29 -070024#include <linux/kprobes.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070025#include <linux/bootmem.h>
26#include <linux/module.h>
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -070027#include <linux/mm.h>
28#include <linux/page-flags.h>
29#include <linux/highmem.h>
Markus Armbrusterb8c2d3d2008-02-27 14:56:35 +010030#include <linux/console.h>
Chris Wright5d990b62009-12-04 12:15:21 -080031#include <linux/pci.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090032#include <linux/gfp.h>
Jeremy Fitzhardinge236260b2010-10-06 15:52:29 -070033#include <linux/memblock.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070034
Jeremy Fitzhardinge1ccbf532009-10-06 15:11:14 -070035#include <xen/xen.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070036#include <xen/interface/xen.h>
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080037#include <xen/interface/version.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070038#include <xen/interface/physdev.h>
39#include <xen/interface/vcpu.h>
Sheng Yangbee6ab52010-05-14 12:39:33 +010040#include <xen/interface/memory.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070041#include <xen/features.h>
42#include <xen/page.h>
Sheng Yang38e20b02010-05-14 12:40:51 +010043#include <xen/hvm.h>
Jeremy Fitzhardinge084a2a42008-07-08 15:06:50 -070044#include <xen/hvc-console.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070045
46#include <asm/paravirt.h>
Ingo Molnar7b6aa332009-02-17 13:58:15 +010047#include <asm/apic.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070048#include <asm/page.h>
Alex Nixonb5401a92010-03-18 16:31:34 -040049#include <asm/xen/pci.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070050#include <asm/xen/hypercall.h>
51#include <asm/xen/hypervisor.h>
52#include <asm/fixmap.h>
53#include <asm/processor.h>
Jeremy Fitzhardinge707ebbc2009-03-27 11:29:02 -070054#include <asm/proto.h>
Jeremy Fitzhardinge11539682008-07-08 15:07:16 -070055#include <asm/msr-index.h>
Jeremy Fitzhardinge6cac5a92009-03-29 19:56:29 -070056#include <asm/traps.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070057#include <asm/setup.h>
58#include <asm/desc.h>
Ian Campbell817a8242010-02-26 17:16:00 +000059#include <asm/pgalloc.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070060#include <asm/pgtable.h>
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070061#include <asm/tlbflush.h>
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -070062#include <asm/reboot.h>
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -070063#include <asm/stackprotector.h>
Sheng Yangbee6ab52010-05-14 12:39:33 +010064#include <asm/hypervisor.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070065
66#include "xen-ops.h"
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070067#include "mmu.h"
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070068#include "multicalls.h"
69
70EXPORT_SYMBOL_GPL(hypercall_page);
71
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070072DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
73DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -070074
Jeremy Fitzhardinge6e833582008-08-19 13:16:17 -070075enum xen_domain_type xen_domain_type = XEN_NATIVE;
76EXPORT_SYMBOL_GPL(xen_domain_type);
77
Ian Campbell7e775062010-09-30 12:37:26 +010078unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
79EXPORT_SYMBOL(machine_to_phys_mapping);
80unsigned int machine_to_phys_order;
81EXPORT_SYMBOL(machine_to_phys_order);
82
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070083struct start_info *xen_start_info;
84EXPORT_SYMBOL_GPL(xen_start_info);
85
Jeremy Fitzhardingea0d695c2008-05-26 23:31:21 +010086struct shared_info xen_dummy_shared_info;
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -070087
Jeremy Fitzhardinge38341432009-02-02 13:55:31 -080088void *xen_initial_gdt;
89
Sheng Yangbee6ab52010-05-14 12:39:33 +010090RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
Sheng Yang38e20b02010-05-14 12:40:51 +010091__read_mostly int xen_have_vector_callback;
92EXPORT_SYMBOL_GPL(xen_have_vector_callback);
Sheng Yangbee6ab52010-05-14 12:39:33 +010093
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -070094/*
95 * Point at some empty memory to start with. We map the real shared_info
96 * page as soon as fixmap is up and running.
97 */
Jeremy Fitzhardingea0d695c2008-05-26 23:31:21 +010098struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -070099
100/*
101 * Flag to determine whether vcpu info placement is available on all
102 * VCPUs. We assume it is to start with, and then set it to zero on
103 * the first failure. This is because it can succeed on some VCPUs
104 * and not others, since it can involve hypervisor memory allocation,
105 * or because the guest failed to guarantee all the appropriate
106 * constraints on all VCPUs (ie buffer can't cross a page boundary).
107 *
108 * Note that any particular CPU may be using a placed vcpu structure,
109 * but we can only optimise if the all are.
110 *
111 * 0: not available, 1: available
112 */
Jeremy Fitzhardingee4d04072009-02-02 13:55:54 -0800113static int have_vcpu_info_placement = 1;
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700114
Mukesh Rathorc06ee782010-07-19 10:25:08 -0700115static void clamp_max_cpus(void)
116{
117#ifdef CONFIG_SMP
118 if (setup_max_cpus > MAX_VIRT_CPUS)
119 setup_max_cpus = MAX_VIRT_CPUS;
120#endif
121}
122
Jeremy Fitzhardinge9c7a7942008-05-31 01:33:02 +0100123static void xen_vcpu_setup(int cpu)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700124{
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700125 struct vcpu_register_vcpu_info info;
126 int err;
127 struct vcpu_info *vcpup;
128
Jeremy Fitzhardingea0d695c2008-05-26 23:31:21 +0100129 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700130
Mukesh Rathorc06ee782010-07-19 10:25:08 -0700131 if (cpu < MAX_VIRT_CPUS)
132 per_cpu(xen_vcpu,cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
133
134 if (!have_vcpu_info_placement) {
135 if (cpu >= MAX_VIRT_CPUS)
136 clamp_max_cpus();
137 return;
138 }
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700139
140 vcpup = &per_cpu(xen_vcpu_info, cpu);
Jeremy Fitzhardinge9976b392009-02-27 09:19:26 -0800141 info.mfn = arbitrary_virt_to_mfn(vcpup);
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700142 info.offset = offset_in_page(vcpup);
143
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700144 /* Check to see if the hypervisor will put the vcpu_info
145 structure where we want it, which allows direct access via
146 a percpu-variable. */
147 err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);
148
149 if (err) {
150 printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
151 have_vcpu_info_placement = 0;
Mukesh Rathorc06ee782010-07-19 10:25:08 -0700152 clamp_max_cpus();
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700153 } else {
154 /* This cpu is using the registered vcpu info, even if
155 later ones fail to. */
156 per_cpu(xen_vcpu, cpu) = vcpup;
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700157 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700158}
159
Jeremy Fitzhardinge9c7a7942008-05-31 01:33:02 +0100160/*
161 * On restore, set the vcpu placement up again.
162 * If it fails, then we're in a bad state, since
163 * we can't back out from using it...
164 */
165void xen_vcpu_restore(void)
166{
Jeremy Fitzhardinge3905bb22009-11-21 08:46:29 +0800167 int cpu;
Jeremy Fitzhardinge9c7a7942008-05-31 01:33:02 +0100168
Jeremy Fitzhardinge3905bb22009-11-21 08:46:29 +0800169 for_each_online_cpu(cpu) {
170 bool other_cpu = (cpu != smp_processor_id());
Jeremy Fitzhardinge9c7a7942008-05-31 01:33:02 +0100171
Jeremy Fitzhardinge3905bb22009-11-21 08:46:29 +0800172 if (other_cpu &&
173 HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL))
174 BUG();
Jeremy Fitzhardinge9c7a7942008-05-31 01:33:02 +0100175
Jeremy Fitzhardinge3905bb22009-11-21 08:46:29 +0800176 xen_setup_runstate_info(cpu);
Ian Campbellbe012922009-11-21 08:35:55 +0800177
Jeremy Fitzhardinge3905bb22009-11-21 08:46:29 +0800178 if (have_vcpu_info_placement)
Jeremy Fitzhardinge9c7a7942008-05-31 01:33:02 +0100179 xen_vcpu_setup(cpu);
180
Jeremy Fitzhardinge3905bb22009-11-21 08:46:29 +0800181 if (other_cpu &&
182 HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
183 BUG();
Jeremy Fitzhardinge9c7a7942008-05-31 01:33:02 +0100184 }
185}
186
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700187static void __init xen_banner(void)
188{
Jeremy Fitzhardinge95c7c232008-07-15 13:42:34 -0700189 unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
190 struct xen_extraversion extra;
191 HYPERVISOR_xen_version(XENVER_extraversion, &extra);
192
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700193 printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700194 pv_info.name);
Jeremy Fitzhardinge95c7c232008-07-15 13:42:34 -0700195 printk(KERN_INFO "Xen version: %d.%d%s%s\n",
196 version >> 16, version & 0xffff, extra.extraversion,
Jeremy Fitzhardingee57778a2008-06-16 04:30:02 -0700197 xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700198}
199
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800200static __read_mostly unsigned int cpuid_leaf1_edx_mask = ~0;
201static __read_mostly unsigned int cpuid_leaf1_ecx_mask = ~0;
202
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100203static void xen_cpuid(unsigned int *ax, unsigned int *bx,
204 unsigned int *cx, unsigned int *dx)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700205{
Jeremy Fitzhardinge82d64692009-10-22 16:41:15 -0700206 unsigned maskebx = ~0;
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800207 unsigned maskecx = ~0;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700208 unsigned maskedx = ~0;
209
210 /*
211 * Mask out inconvenient features, to try and disable as many
212 * unsupported kernel subsystems as possible.
213 */
Jeremy Fitzhardinge82d64692009-10-22 16:41:15 -0700214 switch (*ax) {
215 case 1:
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800216 maskecx = cpuid_leaf1_ecx_mask;
217 maskedx = cpuid_leaf1_edx_mask;
Jeremy Fitzhardinge82d64692009-10-22 16:41:15 -0700218 break;
219
220 case 0xb:
221 /* Suppress extended topology stuff */
222 maskebx = 0;
223 break;
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800224 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700225
226 asm(XEN_EMULATE_PREFIX "cpuid"
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100227 : "=a" (*ax),
228 "=b" (*bx),
229 "=c" (*cx),
230 "=d" (*dx)
231 : "0" (*ax), "2" (*cx));
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800232
Jeremy Fitzhardinge82d64692009-10-22 16:41:15 -0700233 *bx &= maskebx;
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800234 *cx &= maskecx;
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100235 *dx &= maskedx;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700236}
237
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800238static __init void xen_init_cpuid_mask(void)
239{
240 unsigned int ax, bx, cx, dx;
241
242 cpuid_leaf1_edx_mask =
243 ~((1 << X86_FEATURE_MCE) | /* disable MCE */
244 (1 << X86_FEATURE_MCA) | /* disable MCA */
Stefano Stabelliniff128492010-09-28 16:45:51 +0100245 (1 << X86_FEATURE_MTRR) | /* disable MTRR */
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800246 (1 << X86_FEATURE_ACC)); /* thermal monitoring */
247
248 if (!xen_initial_domain())
249 cpuid_leaf1_edx_mask &=
250 ~((1 << X86_FEATURE_APIC) | /* disable local APIC */
251 (1 << X86_FEATURE_ACPI)); /* disable ACPI */
252
253 ax = 1;
H. Peter Anvin7adb4df2009-08-25 21:06:03 -0700254 cx = 0;
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -0800255 xen_cpuid(&ax, &bx, &cx, &dx);
256
257 /* cpuid claims we support xsave; try enabling it to see what happens */
258 if (cx & (1 << (X86_FEATURE_XSAVE % 32))) {
259 unsigned long cr4;
260
261 set_in_cr4(X86_CR4_OSXSAVE);
262
263 cr4 = read_cr4();
264
265 if ((cr4 & X86_CR4_OSXSAVE) == 0)
266 cpuid_leaf1_ecx_mask &= ~(1 << (X86_FEATURE_XSAVE % 32));
267
268 clear_in_cr4(X86_CR4_OSXSAVE);
269 }
270}
271
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700272static void xen_set_debugreg(int reg, unsigned long val)
273{
274 HYPERVISOR_set_debugreg(reg, val);
275}
276
277static unsigned long xen_get_debugreg(int reg)
278{
279 return HYPERVISOR_get_debugreg(reg);
280}
281
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800282static void xen_end_context_switch(struct task_struct *next)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700283{
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700284 xen_mc_flush();
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -0800285 paravirt_end_context_switch(next);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700286}
287
288static unsigned long xen_store_tr(void)
289{
290 return 0;
291}
292
Jeremy Fitzhardingea05d2eb2008-07-27 08:45:02 -0700293/*
Jeremy Fitzhardingecef43bf2008-07-28 13:33:44 -0700294 * Set the page permissions for a particular virtual address. If the
295 * address is a vmalloc mapping (or other non-linear mapping), then
296 * find the linear mapping of the page and also set its protections to
297 * match.
Jeremy Fitzhardingea05d2eb2008-07-27 08:45:02 -0700298 */
299static void set_aliased_prot(void *v, pgprot_t prot)
300{
301 int level;
302 pte_t *ptep;
303 pte_t pte;
304 unsigned long pfn;
305 struct page *page;
306
307 ptep = lookup_address((unsigned long)v, &level);
308 BUG_ON(ptep == NULL);
309
310 pfn = pte_pfn(*ptep);
311 page = pfn_to_page(pfn);
312
313 pte = pfn_pte(pfn, prot);
314
315 if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0))
316 BUG();
317
318 if (!PageHighMem(page)) {
319 void *av = __va(PFN_PHYS(pfn));
320
321 if (av != v)
322 if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0))
323 BUG();
324 } else
325 kmap_flush_unused();
326}
327
Jeremy Fitzhardinge38ffbe62008-07-23 14:21:18 -0700328static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries)
329{
Jeremy Fitzhardingea05d2eb2008-07-27 08:45:02 -0700330 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
Jeremy Fitzhardinge38ffbe62008-07-23 14:21:18 -0700331 int i;
332
Jeremy Fitzhardingea05d2eb2008-07-27 08:45:02 -0700333 for(i = 0; i < entries; i += entries_per_page)
334 set_aliased_prot(ldt + i, PAGE_KERNEL_RO);
Jeremy Fitzhardinge38ffbe62008-07-23 14:21:18 -0700335}
336
337static void xen_free_ldt(struct desc_struct *ldt, unsigned entries)
338{
Jeremy Fitzhardingea05d2eb2008-07-27 08:45:02 -0700339 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
Jeremy Fitzhardinge38ffbe62008-07-23 14:21:18 -0700340 int i;
341
Jeremy Fitzhardingea05d2eb2008-07-27 08:45:02 -0700342 for(i = 0; i < entries; i += entries_per_page)
343 set_aliased_prot(ldt + i, PAGE_KERNEL);
Jeremy Fitzhardinge38ffbe62008-07-23 14:21:18 -0700344}
345
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700346static void xen_set_ldt(const void *addr, unsigned entries)
347{
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700348 struct mmuext_op *op;
349 struct multicall_space mcs = xen_mc_entry(sizeof(*op));
350
351 op = mcs.args;
352 op->cmd = MMUEXT_SET_LDT;
Jan Beulich4dbf7af2008-01-30 13:33:14 +0100353 op->arg1.linear_addr = (unsigned long)addr;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700354 op->arg2.nr_ents = entries;
355
356 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
357
358 xen_mc_issue(PARAVIRT_LAZY_CPU);
359}
360
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100361static void xen_load_gdt(const struct desc_ptr *dtr)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700362{
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700363 unsigned long va = dtr->address;
364 unsigned int size = dtr->size + 1;
365 unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
Jeremy Fitzhardinge3ce5fa72009-03-04 15:26:00 -0800366 unsigned long frames[pages];
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700367 int f;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700368
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -0700369 /*
370 * A GDT can be up to 64k in size, which corresponds to 8192
371 * 8-byte entries, or 16 4k pages..
372 */
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700373
374 BUG_ON(size > 65536);
375 BUG_ON(va & ~PAGE_MASK);
376
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700377 for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
Jeremy Fitzhardinge6ed6bf42009-03-04 13:02:18 -0800378 int level;
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -0700379 pte_t *ptep;
Jeremy Fitzhardinge6ed6bf42009-03-04 13:02:18 -0800380 unsigned long pfn, mfn;
381 void *virt;
382
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -0700383 /*
384 * The GDT is per-cpu and is in the percpu data area.
385 * That can be virtually mapped, so we need to do a
386 * page-walk to get the underlying MFN for the
387 * hypercall. The page can also be in the kernel's
388 * linear range, so we need to RO that mapping too.
389 */
390 ptep = lookup_address(va, &level);
Jeremy Fitzhardinge6ed6bf42009-03-04 13:02:18 -0800391 BUG_ON(ptep == NULL);
392
393 pfn = pte_pfn(*ptep);
394 mfn = pfn_to_mfn(pfn);
395 virt = __va(PFN_PHYS(pfn));
396
397 frames[f] = mfn;
Jeremy Fitzhardinge9976b392009-02-27 09:19:26 -0800398
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700399 make_lowmem_page_readonly((void *)va);
Jeremy Fitzhardinge6ed6bf42009-03-04 13:02:18 -0800400 make_lowmem_page_readonly(virt);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700401 }
402
Jeremy Fitzhardinge3ce5fa72009-03-04 15:26:00 -0800403 if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
404 BUG();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700405}
406
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -0700407/*
408 * load_gdt for early boot, when the gdt is only mapped once
409 */
410static __init void xen_load_gdt_boot(const struct desc_ptr *dtr)
411{
412 unsigned long va = dtr->address;
413 unsigned int size = dtr->size + 1;
414 unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
415 unsigned long frames[pages];
416 int f;
417
418 /*
419 * A GDT can be up to 64k in size, which corresponds to 8192
420 * 8-byte entries, or 16 4k pages..
421 */
422
423 BUG_ON(size > 65536);
424 BUG_ON(va & ~PAGE_MASK);
425
426 for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
427 pte_t pte;
428 unsigned long pfn, mfn;
429
430 pfn = virt_to_pfn(va);
431 mfn = pfn_to_mfn(pfn);
432
433 pte = pfn_pte(pfn, PAGE_KERNEL_RO);
434
435 if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
436 BUG();
437
438 frames[f] = mfn;
439 }
440
441 if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
442 BUG();
443}
444
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700445static void load_TLS_descriptor(struct thread_struct *t,
446 unsigned int cpu, unsigned int i)
447{
448 struct desc_struct *gdt = get_cpu_gdt_table(cpu);
Jeremy Fitzhardinge9976b392009-02-27 09:19:26 -0800449 xmaddr_t maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700450 struct multicall_space mc = __xen_mc_entry(0);
451
452 MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
453}
454
455static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
456{
Jeremy Fitzhardinge8b84ad92007-07-17 18:37:06 -0700457 /*
Tejun Heoccbeed32009-02-09 22:17:40 +0900458 * XXX sleazy hack: If we're being called in a lazy-cpu zone
459 * and lazy gs handling is enabled, it means we're in a
460 * context switch, and %gs has just been saved. This means we
461 * can zero it out to prevent faults on exit from the
462 * hypervisor if the next process has no %gs. Either way, it
463 * has been saved, and the new value will get loaded properly.
464 * This will go away as soon as Xen has been modified to not
465 * save/restore %gs for normal hypercalls.
Eduardo Habkost8a954082008-07-08 15:07:10 -0700466 *
467 * On x86_64, this hack is not used for %gs, because gs points
468 * to KERNEL_GS_BASE (and uses it for PDA references), so we
469 * must not zero %gs on x86_64
470 *
471 * For x86_64, we need to zero %fs, otherwise we may get an
472 * exception between the new %fs descriptor being loaded and
473 * %fs being effectively cleared at __switch_to().
Jeremy Fitzhardinge8b84ad92007-07-17 18:37:06 -0700474 */
Eduardo Habkost8a954082008-07-08 15:07:10 -0700475 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
476#ifdef CONFIG_X86_32
Tejun Heoccbeed32009-02-09 22:17:40 +0900477 lazy_load_gs(0);
Eduardo Habkost8a954082008-07-08 15:07:10 -0700478#else
479 loadsegment(fs, 0);
480#endif
481 }
482
483 xen_mc_batch();
484
485 load_TLS_descriptor(t, cpu, 0);
486 load_TLS_descriptor(t, cpu, 1);
487 load_TLS_descriptor(t, cpu, 2);
488
489 xen_mc_issue(PARAVIRT_LAZY_CPU);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700490}
491
Eduardo Habkosta8fc1082008-07-08 15:07:05 -0700492#ifdef CONFIG_X86_64
493static void xen_load_gs_index(unsigned int idx)
494{
495 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
496 BUG();
497}
498#endif
499
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700500static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100501 const void *ptr)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700502{
Jeremy Fitzhardingecef43bf2008-07-28 13:33:44 -0700503 xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]);
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100504 u64 entry = *(u64 *)ptr;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700505
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700506 preempt_disable();
507
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700508 xen_mc_flush();
509 if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
510 BUG();
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700511
512 preempt_enable();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700513}
514
Eduardo Habkoste176d362008-07-08 15:06:59 -0700515static int cvt_gate_to_trap(int vector, const gate_desc *val,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700516 struct trap_info *info)
517{
Jeremy Fitzhardinge6cac5a92009-03-29 19:56:29 -0700518 unsigned long addr;
519
Jeremy Fitzhardinge6d02c422009-03-29 22:57:15 -0700520 if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700521 return 0;
522
523 info->vector = vector;
Jeremy Fitzhardinge6cac5a92009-03-29 19:56:29 -0700524
525 addr = gate_offset(*val);
526#ifdef CONFIG_X86_64
Jeremy Fitzhardingeb80119b2009-04-24 00:22:08 -0700527 /*
528 * Look for known traps using IST, and substitute them
529 * appropriately. The debugger ones are the only ones we care
530 * about. Xen will handle faults like double_fault and
531 * machine_check, so we should never see them. Warn if
532 * there's an unexpected IST-using fault handler.
533 */
Jeremy Fitzhardinge6cac5a92009-03-29 19:56:29 -0700534 if (addr == (unsigned long)debug)
535 addr = (unsigned long)xen_debug;
536 else if (addr == (unsigned long)int3)
537 addr = (unsigned long)xen_int3;
538 else if (addr == (unsigned long)stack_segment)
539 addr = (unsigned long)xen_stack_segment;
Jeremy Fitzhardingeb80119b2009-04-24 00:22:08 -0700540 else if (addr == (unsigned long)double_fault ||
541 addr == (unsigned long)nmi) {
542 /* Don't need to handle these */
543 return 0;
544#ifdef CONFIG_X86_MCE
545 } else if (addr == (unsigned long)machine_check) {
546 return 0;
547#endif
548 } else {
549 /* Some other trap using IST? */
550 if (WARN_ON(val->ist != 0))
551 return 0;
552 }
Jeremy Fitzhardinge6cac5a92009-03-29 19:56:29 -0700553#endif /* CONFIG_X86_64 */
554 info->address = addr;
555
Eduardo Habkoste176d362008-07-08 15:06:59 -0700556 info->cs = gate_segment(*val);
557 info->flags = val->dpl;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700558 /* interrupt gates clear IF */
Jeremy Fitzhardinge6d02c422009-03-29 22:57:15 -0700559 if (val->type == GATE_INTERRUPT)
560 info->flags |= 1 << 2;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700561
562 return 1;
563}
564
565/* Locations of each CPU's IDT */
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100566static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700567
568/* Set an IDT entry. If the entry is part of the current IDT, then
569 also update Xen. */
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100570static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700571{
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700572 unsigned long p = (unsigned long)&dt[entrynum];
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700573 unsigned long start, end;
574
575 preempt_disable();
576
577 start = __get_cpu_var(idt_desc).address;
578 end = start + __get_cpu_var(idt_desc).size + 1;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700579
580 xen_mc_flush();
581
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100582 native_write_idt_entry(dt, entrynum, g);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700583
584 if (p >= start && (p + 8) <= end) {
585 struct trap_info info[2];
586
587 info[1].address = 0;
588
Eduardo Habkoste176d362008-07-08 15:06:59 -0700589 if (cvt_gate_to_trap(entrynum, g, &info[0]))
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700590 if (HYPERVISOR_set_trap_table(info))
591 BUG();
592 }
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700593
594 preempt_enable();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700595}
596
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100597static void xen_convert_trap_info(const struct desc_ptr *desc,
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700598 struct trap_info *traps)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700599{
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700600 unsigned in, out, count;
601
Eduardo Habkoste176d362008-07-08 15:06:59 -0700602 count = (desc->size+1) / sizeof(gate_desc);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700603 BUG_ON(count > 256);
604
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700605 for (in = out = 0; in < count; in++) {
Eduardo Habkoste176d362008-07-08 15:06:59 -0700606 gate_desc *entry = (gate_desc*)(desc->address) + in;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700607
Eduardo Habkoste176d362008-07-08 15:06:59 -0700608 if (cvt_gate_to_trap(in, entry, &traps[out]))
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700609 out++;
610 }
611 traps[out].address = 0;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700612}
613
614void xen_copy_trap_info(struct trap_info *traps)
615{
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100616 const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700617
618 xen_convert_trap_info(desc, traps);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700619}
620
621/* Load a new IDT into Xen. In principle this can be per-CPU, so we
622 hold a spinlock to protect the static traps[] array (static because
623 it avoids allocation, and saves stack space). */
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100624static void xen_load_idt(const struct desc_ptr *desc)
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700625{
626 static DEFINE_SPINLOCK(lock);
627 static struct trap_info traps[257];
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700628
629 spin_lock(&lock);
630
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700631 __get_cpu_var(idt_desc) = *desc;
632
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700633 xen_convert_trap_info(desc, traps);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700634
635 xen_mc_flush();
636 if (HYPERVISOR_set_trap_table(traps))
637 BUG();
638
639 spin_unlock(&lock);
640}
641
642/* Write a GDT descriptor entry. Ignore LDT descriptors, since
643 they're handled differently. */
644static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100645 const void *desc, int type)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700646{
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700647 preempt_disable();
648
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100649 switch (type) {
650 case DESC_LDT:
651 case DESC_TSS:
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700652 /* ignore */
653 break;
654
655 default: {
Jeremy Fitzhardinge9976b392009-02-27 09:19:26 -0800656 xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700657
658 xen_mc_flush();
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100659 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700660 BUG();
661 }
662
663 }
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700664
665 preempt_enable();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700666}
667
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -0700668/*
669 * Version of write_gdt_entry for use at early boot-time needed to
670 * update an entry as simply as possible.
671 */
672static __init void xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
673 const void *desc, int type)
674{
675 switch (type) {
676 case DESC_LDT:
677 case DESC_TSS:
678 /* ignore */
679 break;
680
681 default: {
682 xmaddr_t maddr = virt_to_machine(&dt[entry]);
683
684 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
685 dt[entry] = *(struct desc_struct *)desc;
686 }
687
688 }
689}
690
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100691static void xen_load_sp0(struct tss_struct *tss,
Jeremy Fitzhardingea05d2eb2008-07-27 08:45:02 -0700692 struct thread_struct *thread)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700693{
694 struct multicall_space mcs = xen_mc_entry(0);
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100695 MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700696 xen_mc_issue(PARAVIRT_LAZY_CPU);
697}
698
699static void xen_set_iopl_mask(unsigned mask)
700{
701 struct physdev_set_iopl set_iopl;
702
703 /* Force the change at ring 0. */
704 set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
705 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
706}
707
708static void xen_io_delay(void)
709{
710}
711
712#ifdef CONFIG_X86_LOCAL_APIC
Suresh Siddhaad66dd32008-07-11 13:11:56 -0700713static u32 xen_apic_read(u32 reg)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700714{
715 return 0;
716}
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700717
Suresh Siddhaad66dd32008-07-11 13:11:56 -0700718static void xen_apic_write(u32 reg, u32 val)
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700719{
720 /* Warn to see if there's any stray references */
721 WARN_ON(1);
722}
Suresh Siddhaad66dd32008-07-11 13:11:56 -0700723
Suresh Siddhaad66dd32008-07-11 13:11:56 -0700724static u64 xen_apic_icr_read(void)
725{
726 return 0;
727}
728
729static void xen_apic_icr_write(u32 low, u32 id)
730{
731 /* Warn to see if there's any stray references */
732 WARN_ON(1);
733}
734
735static void xen_apic_wait_icr_idle(void)
736{
737 return;
738}
739
Yinghai Lu94a8c3c2008-07-13 22:19:35 -0700740static u32 xen_safe_apic_wait_icr_idle(void)
741{
742 return 0;
743}
744
Yinghai Luc1eeb2d2009-02-16 23:02:14 -0800745static void set_xen_basic_apic_ops(void)
746{
747 apic->read = xen_apic_read;
748 apic->write = xen_apic_write;
749 apic->icr_read = xen_apic_icr_read;
750 apic->icr_write = xen_apic_icr_write;
751 apic->wait_icr_idle = xen_apic_wait_icr_idle;
752 apic->safe_wait_icr_idle = xen_safe_apic_wait_icr_idle;
753}
Suresh Siddhaad66dd32008-07-11 13:11:56 -0700754
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700755#endif
756
Jeremy Fitzhardinge7b1333a2008-05-26 23:31:01 +0100757static void xen_clts(void)
758{
759 struct multicall_space mcs;
760
761 mcs = xen_mc_entry(0);
762
763 MULTI_fpu_taskswitch(mcs.mc, 0);
764
765 xen_mc_issue(PARAVIRT_LAZY_CPU);
766}
767
Jeremy Fitzhardingea789ed52009-04-24 00:26:50 -0700768static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
769
770static unsigned long xen_read_cr0(void)
771{
772 unsigned long cr0 = percpu_read(xen_cr0_value);
773
774 if (unlikely(cr0 == 0)) {
775 cr0 = native_read_cr0();
776 percpu_write(xen_cr0_value, cr0);
777 }
778
779 return cr0;
780}
781
Jeremy Fitzhardinge7b1333a2008-05-26 23:31:01 +0100782static void xen_write_cr0(unsigned long cr0)
783{
784 struct multicall_space mcs;
785
Jeremy Fitzhardingea789ed52009-04-24 00:26:50 -0700786 percpu_write(xen_cr0_value, cr0);
787
Jeremy Fitzhardinge7b1333a2008-05-26 23:31:01 +0100788 /* Only pay attention to cr0.TS; everything else is
789 ignored. */
790 mcs = xen_mc_entry(0);
791
792 MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);
793
794 xen_mc_issue(PARAVIRT_LAZY_CPU);
795}
796
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700797static void xen_write_cr4(unsigned long cr4)
798{
Jeremy Fitzhardinge2956a352008-05-26 23:31:04 +0100799 cr4 &= ~X86_CR4_PGE;
800 cr4 &= ~X86_CR4_PSE;
801
802 native_write_cr4(cr4);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700803}
804
Jeremy Fitzhardinge11539682008-07-08 15:07:16 -0700805static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
806{
807 int ret;
808
809 ret = 0;
810
Tejf63c2f22008-12-16 11:56:06 -0800811 switch (msr) {
Jeremy Fitzhardinge11539682008-07-08 15:07:16 -0700812#ifdef CONFIG_X86_64
813 unsigned which;
814 u64 base;
815
816 case MSR_FS_BASE: which = SEGBASE_FS; goto set;
817 case MSR_KERNEL_GS_BASE: which = SEGBASE_GS_USER; goto set;
818 case MSR_GS_BASE: which = SEGBASE_GS_KERNEL; goto set;
819
820 set:
821 base = ((u64)high << 32) | low;
822 if (HYPERVISOR_set_segment_base(which, base) != 0)
H. Peter Anvin0cc02132009-08-31 14:23:29 -0700823 ret = -EIO;
Jeremy Fitzhardinge11539682008-07-08 15:07:16 -0700824 break;
825#endif
Jeremy Fitzhardinged89961e2008-07-24 13:48:58 -0700826
827 case MSR_STAR:
828 case MSR_CSTAR:
829 case MSR_LSTAR:
830 case MSR_SYSCALL_MASK:
831 case MSR_IA32_SYSENTER_CS:
832 case MSR_IA32_SYSENTER_ESP:
833 case MSR_IA32_SYSENTER_EIP:
834 /* Fast syscall setup is all done in hypercalls, so
835 these are all ignored. Stub them out here to stop
836 Xen console noise. */
837 break;
838
Jeremy Fitzhardinge41f2e472010-03-30 11:47:40 -0700839 case MSR_IA32_CR_PAT:
840 if (smp_processor_id() == 0)
841 xen_set_pat(((u64)high << 32) | low);
842 break;
843
Jeremy Fitzhardinge11539682008-07-08 15:07:16 -0700844 default:
845 ret = native_write_msr_safe(msr, low, high);
846 }
847
848 return ret;
849}
850
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100851void xen_setup_shared_info(void)
Jeremy Fitzhardinge2e8fe712008-03-17 16:36:53 -0700852{
853 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
Jeremy Fitzhardinge15664f92008-07-08 15:06:47 -0700854 set_fixmap(FIX_PARAVIRT_BOOTMAP,
855 xen_start_info->shared_info);
Jeremy Fitzhardinge2e8fe712008-03-17 16:36:53 -0700856
Jeremy Fitzhardinge15664f92008-07-08 15:06:47 -0700857 HYPERVISOR_shared_info =
858 (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
Jeremy Fitzhardinge2e8fe712008-03-17 16:36:53 -0700859 } else
860 HYPERVISOR_shared_info =
861 (struct shared_info *)__va(xen_start_info->shared_info);
862
863#ifndef CONFIG_SMP
864 /* In UP this is as good a place as any to set up shared info */
865 xen_setup_vcpu_info_placement();
866#endif
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100867
868 xen_setup_mfn_list_list();
Jeremy Fitzhardinge2e8fe712008-03-17 16:36:53 -0700869}
870
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700871/* This is called once we have the cpu_possible_map */
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100872void xen_setup_vcpu_info_placement(void)
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700873{
874 int cpu;
875
876 for_each_possible_cpu(cpu)
877 xen_vcpu_setup(cpu);
878
879 /* xen_vcpu_setup managed to place the vcpu_info within the
880 percpu area for all cpus, so make use of it */
881 if (have_vcpu_info_placement) {
Jeremy Fitzhardingeecb93d12009-01-28 14:35:05 -0800882 pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
883 pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
884 pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
885 pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700886 pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
Jeremy Fitzhardinge60223a32007-07-17 18:37:07 -0700887 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700888}
889
Andi Kleenab144f52007-08-10 22:31:03 +0200890static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
891 unsigned long addr, unsigned len)
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700892{
893 char *start, *end, *reloc;
894 unsigned ret;
895
896 start = end = reloc = NULL;
897
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700898#define SITE(op, x) \
899 case PARAVIRT_PATCH(op.x): \
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700900 if (have_vcpu_info_placement) { \
901 start = (char *)xen_##x##_direct; \
902 end = xen_##x##_direct_end; \
903 reloc = xen_##x##_direct_reloc; \
904 } \
905 goto patch_site
906
907 switch (type) {
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700908 SITE(pv_irq_ops, irq_enable);
909 SITE(pv_irq_ops, irq_disable);
910 SITE(pv_irq_ops, save_fl);
911 SITE(pv_irq_ops, restore_fl);
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700912#undef SITE
913
914 patch_site:
915 if (start == NULL || (end-start) > len)
916 goto default_patch;
917
Andi Kleenab144f52007-08-10 22:31:03 +0200918 ret = paravirt_patch_insns(insnbuf, len, start, end);
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700919
920 /* Note: because reloc is assigned from something that
921 appears to be an array, gcc assumes it's non-null,
922 but doesn't know its relationship with start and
923 end. */
924 if (reloc > start && reloc < end) {
925 int reloc_off = reloc - start;
Andi Kleenab144f52007-08-10 22:31:03 +0200926 long *relocp = (long *)(insnbuf + reloc_off);
927 long delta = start - (char *)addr;
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700928
929 *relocp += delta;
930 }
931 break;
932
933 default_patch:
934 default:
Andi Kleenab144f52007-08-10 22:31:03 +0200935 ret = paravirt_patch_default(type, clobbers, insnbuf,
936 addr, len);
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700937 break;
938 }
939
940 return ret;
941}
942
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700943static const struct pv_info xen_info __initdata = {
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700944 .paravirt_enabled = 1,
945 .shared_kernel_pmd = 0,
946
947 .name = "Xen",
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700948};
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700949
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700950static const struct pv_init_ops xen_init_ops __initdata = {
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700951 .patch = xen_patch,
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700952};
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700953
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700954static const struct pv_cpu_ops xen_cpu_ops __initdata = {
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700955 .cpuid = xen_cpuid,
956
957 .set_debugreg = xen_set_debugreg,
958 .get_debugreg = xen_get_debugreg,
959
Jeremy Fitzhardinge7b1333a2008-05-26 23:31:01 +0100960 .clts = xen_clts,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700961
Jeremy Fitzhardingea789ed52009-04-24 00:26:50 -0700962 .read_cr0 = xen_read_cr0,
Jeremy Fitzhardinge7b1333a2008-05-26 23:31:01 +0100963 .write_cr0 = xen_write_cr0,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700964
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700965 .read_cr4 = native_read_cr4,
966 .read_cr4_safe = native_read_cr4_safe,
967 .write_cr4 = xen_write_cr4,
968
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700969 .wbinvd = native_wbinvd,
970
971 .read_msr = native_read_msr_safe,
Jeremy Fitzhardinge11539682008-07-08 15:07:16 -0700972 .write_msr = xen_write_msr_safe,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700973 .read_tsc = native_read_tsc,
974 .read_pmc = native_read_pmc,
975
Jeremy Fitzhardinge81e103f2008-04-17 17:40:51 +0200976 .iret = xen_iret,
Jeremy Fitzhardinged75cd222008-06-25 00:19:26 -0400977 .irq_enable_sysexit = xen_sysexit,
Jeremy Fitzhardinge6fcac6d2008-07-08 15:07:14 -0700978#ifdef CONFIG_X86_64
979 .usergs_sysret32 = xen_sysret32,
980 .usergs_sysret64 = xen_sysret64,
981#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700982
983 .load_tr_desc = paravirt_nop,
984 .set_ldt = xen_set_ldt,
985 .load_gdt = xen_load_gdt,
986 .load_idt = xen_load_idt,
987 .load_tls = xen_load_tls,
Eduardo Habkosta8fc1082008-07-08 15:07:05 -0700988#ifdef CONFIG_X86_64
989 .load_gs_index = xen_load_gs_index,
990#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700991
Jeremy Fitzhardinge38ffbe62008-07-23 14:21:18 -0700992 .alloc_ldt = xen_alloc_ldt,
993 .free_ldt = xen_free_ldt,
994
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700995 .store_gdt = native_store_gdt,
996 .store_idt = native_store_idt,
997 .store_tr = xen_store_tr,
998
999 .write_ldt_entry = xen_write_ldt_entry,
1000 .write_gdt_entry = xen_write_gdt_entry,
1001 .write_idt_entry = xen_write_idt_entry,
H. Peter Anvinfaca6222008-01-30 13:31:02 +01001002 .load_sp0 = xen_load_sp0,
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001003
1004 .set_iopl_mask = xen_set_iopl_mask,
1005 .io_delay = xen_io_delay,
1006
Jeremy Fitzhardinge952d1d72008-07-08 15:07:01 -07001007 /* Xen takes care of %gs when switching to usermode for us */
1008 .swapgs = paravirt_nop,
1009
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -08001010 .start_context_switch = paravirt_start_context_switch,
1011 .end_context_switch = xen_end_context_switch,
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001012};
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001013
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001014static const struct pv_apic_ops xen_apic_ops __initdata = {
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001015#ifdef CONFIG_X86_LOCAL_APIC
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001016 .startup_ipi_hook = paravirt_nop,
1017#endif
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001018};
1019
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -07001020static void xen_reboot(int reason)
1021{
Jeremy Fitzhardinge349c7092008-05-26 23:31:02 +01001022 struct sched_shutdown r = { .reason = reason };
1023
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -07001024#ifdef CONFIG_SMP
Alok Kataria76fac072010-10-11 14:37:08 -07001025 stop_other_cpus();
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -07001026#endif
1027
Jeremy Fitzhardinge349c7092008-05-26 23:31:02 +01001028 if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -07001029 BUG();
1030}
1031
1032static void xen_restart(char *msg)
1033{
1034 xen_reboot(SHUTDOWN_reboot);
1035}
1036
1037static void xen_emergency_restart(void)
1038{
1039 xen_reboot(SHUTDOWN_reboot);
1040}
1041
1042static void xen_machine_halt(void)
1043{
1044 xen_reboot(SHUTDOWN_poweroff);
1045}
1046
1047static void xen_crash_shutdown(struct pt_regs *regs)
1048{
1049 xen_reboot(SHUTDOWN_crash);
1050}
1051
Donald Dutilef09f6d12010-07-15 14:56:49 -04001052static int
1053xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr)
1054{
Ian Campbell086748e2010-08-03 14:55:14 -07001055 xen_reboot(SHUTDOWN_crash);
Donald Dutilef09f6d12010-07-15 14:56:49 -04001056 return NOTIFY_DONE;
1057}
1058
1059static struct notifier_block xen_panic_block = {
1060 .notifier_call= xen_panic_event,
1061};
1062
1063int xen_panic_handler_init(void)
1064{
1065 atomic_notifier_chain_register(&panic_notifier_list, &xen_panic_block);
1066 return 0;
1067}
1068
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -07001069static const struct machine_ops __initdata xen_machine_ops = {
1070 .restart = xen_restart,
1071 .halt = xen_machine_halt,
1072 .power_off = xen_machine_halt,
1073 .shutdown = xen_machine_halt,
1074 .crash_shutdown = xen_crash_shutdown,
1075 .emergency_restart = xen_emergency_restart,
1076};
1077
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -07001078/*
1079 * Set up the GDT and segment registers for -fstack-protector. Until
1080 * we do this, we have to be careful not to call any stack-protected
1081 * function, which is most of the kernel.
1082 */
1083static void __init xen_setup_stackprotector(void)
1084{
1085 pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
1086 pv_cpu_ops.load_gdt = xen_load_gdt_boot;
1087
1088 setup_stack_canary_segment(0);
1089 switch_to_new_gdt(0);
1090
1091 pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
1092 pv_cpu_ops.load_gdt = xen_load_gdt;
1093}
1094
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001095/* First C function to be called on Xen boot */
1096asmlinkage void __init xen_start_kernel(void)
1097{
Konrad Rzeszutek Wilkec35a692010-11-16 12:09:59 -05001098 struct physdev_set_iopl set_iopl;
1099 int rc;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001100 pgd_t *pgd;
1101
1102 if (!xen_start_info)
1103 return;
1104
Jeremy Fitzhardinge6e833582008-08-19 13:16:17 -07001105 xen_domain_type = XEN_PV_DOMAIN;
1106
Ian Campbell7e775062010-09-30 12:37:26 +01001107 xen_setup_machphys_mapping();
1108
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001109 /* Install Xen paravirt ops */
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001110 pv_info = xen_info;
1111 pv_init_ops = xen_init_ops;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001112 pv_cpu_ops = xen_cpu_ops;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001113 pv_apic_ops = xen_apic_ops;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001114
Thomas Gleixner6b18ae32009-08-20 10:19:54 +02001115 x86_init.resources.memory_setup = xen_memory_setup;
Thomas Gleixner42bbdb42009-08-20 13:04:10 +02001116 x86_init.oem.arch_setup = xen_arch_setup;
Thomas Gleixner6f30c1a2009-08-20 13:19:57 +02001117 x86_init.oem.banner = xen_banner;
Thomas Gleixner845b3942009-08-19 15:37:03 +02001118
Stefano Stabellini409771d2010-05-14 12:48:19 +01001119 xen_init_time_ops();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001120
Jeremy Fitzhardingece2eef32009-08-17 12:26:53 -07001121 /*
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -07001122 * Set up some pagetable state before starting to set any ptes.
Jeremy Fitzhardingece2eef32009-08-17 12:26:53 -07001123 */
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -07001124
Jeremy Fitzhardinge973df352009-10-27 16:54:19 -07001125 xen_init_mmu_ops();
1126
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -07001127 /* Prevent unwanted bits from being set in PTEs. */
1128 __supported_pte_mask &= ~_PAGE_GLOBAL;
1129 if (!xen_initial_domain())
1130 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
1131
1132 __supported_pte_mask |= _PAGE_IOMAP;
1133
Ian Campbell817a8242010-02-26 17:16:00 +00001134 /*
1135 * Prevent page tables from being allocated in highmem, even
1136 * if CONFIG_HIGHPTE is enabled.
1137 */
1138 __userpte_alloc_gfp &= ~__GFP_HIGHMEM;
1139
Jeremy Fitzhardingeb75fe4e2009-09-21 13:34:06 -07001140 /* Work out if we support NX */
H. Peter Anvin4763ed42009-11-13 15:28:16 -08001141 x86_configure_nx();
Jeremy Fitzhardingeb75fe4e2009-09-21 13:34:06 -07001142
Jeremy Fitzhardinge577eebe2009-08-27 12:46:35 -07001143 xen_setup_features();
1144
1145 /* Get mfn list */
1146 if (!xen_feature(XENFEAT_auto_translated_physmap))
1147 xen_build_dynamic_phys_to_machine();
1148
1149 /*
1150 * Set up kernel GDT and segment registers, mainly so that
1151 * -fstack-protector code can be executed.
1152 */
1153 xen_setup_stackprotector();
Jeremy Fitzhardinge0d1edf42008-07-28 11:53:57 -07001154
Jeremy Fitzhardingece2eef32009-08-17 12:26:53 -07001155 xen_init_irq_ops();
Jeremy Fitzhardingee826fe12009-03-07 17:09:27 -08001156 xen_init_cpuid_mask();
1157
Yinghai Lu94a8c3c2008-07-13 22:19:35 -07001158#ifdef CONFIG_X86_LOCAL_APIC
Suresh Siddhaad66dd32008-07-11 13:11:56 -07001159 /*
Yinghai Lu94a8c3c2008-07-13 22:19:35 -07001160 * set up the basic apic ops.
Suresh Siddhaad66dd32008-07-11 13:11:56 -07001161 */
Yinghai Luc1eeb2d2009-02-16 23:02:14 -08001162 set_xen_basic_apic_ops();
Suresh Siddhaad66dd32008-07-11 13:11:56 -07001163#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001164
Jeremy Fitzhardingee57778a2008-06-16 04:30:02 -07001165 if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
1166 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
1167 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
1168 }
1169
Jeremy Fitzhardingefefa6292007-07-17 18:37:07 -07001170 machine_ops = xen_machine_ops;
1171
Jeremy Fitzhardinge38341432009-02-02 13:55:31 -08001172 /*
1173 * The only reliable way to retain the initial address of the
1174 * percpu gdt_page is to remember it here, so we can go and
1175 * mark it RW later, when the initial percpu area is freed.
1176 */
1177 xen_initial_gdt = &per_cpu(gdt_page, 0);
Jeremy Fitzhardinge795f99b2009-01-30 17:47:54 +09001178
Jeremy Fitzhardingea9e70622008-07-08 15:06:41 -07001179 xen_smp_init();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001180
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001181 pgd = (pgd_t *)xen_start_info->pt_base;
1182
Alex Nixon7347b402010-02-19 13:31:06 -05001183 if (!xen_initial_domain())
1184 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
1185
1186 __supported_pte_mask |= _PAGE_IOMAP;
Jeremy Fitzhardinge084a2a42008-07-08 15:06:50 -07001187 /* Don't do the full vcpu_info placement stuff until we have a
1188 possible map and a non-dummy shared_info. */
1189 per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1190
Jeremy Fitzhardinge55d80852009-02-25 09:42:25 -08001191 local_irq_disable();
1192 early_boot_irqs_off();
1193
Jeremy Fitzhardinge236260b2010-10-06 15:52:29 -07001194 memblock_init();
1195
Jeremy Fitzhardinge084a2a42008-07-08 15:06:50 -07001196 xen_raw_console_write("mapping kernel into physical memory\n");
Jeremy Fitzhardinged114e1982008-07-08 15:06:52 -07001197 pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages);
Juan Quintela4ec53872010-09-02 15:45:43 +01001198 xen_ident_map_ISA();
Jeremy Fitzhardinge084a2a42008-07-08 15:06:50 -07001199
Jeremy Fitzhardinge33a84752010-08-27 15:18:19 -07001200 /* Allocate and initialize top and mid mfn levels for p2m structure */
1201 xen_build_mfn_list_list();
1202
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001203 /* keep using Xen gdt for now; no urgent need to change it */
1204
Ian Campbelle68266b2010-01-13 10:16:08 +00001205#ifdef CONFIG_X86_32
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001206 pv_info.kernel_rpl = 1;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001207 if (xen_feature(XENFEAT_supervisor_mode_kernel))
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001208 pv_info.kernel_rpl = 0;
Ian Campbelle68266b2010-01-13 10:16:08 +00001209#else
1210 pv_info.kernel_rpl = 0;
1211#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001212 /* set the limit of our address space */
Jeremy Fitzhardingefb1d8402007-10-16 11:51:31 -07001213 xen_reserve_top();
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001214
Konrad Rzeszutek Wilkec35a692010-11-16 12:09:59 -05001215 /* We used to do this in xen_arch_setup, but that is too late on AMD
1216 * were early_cpu_init (run before ->arch_setup()) calls early_amd_init
1217 * which pokes 0xcf8 port.
1218 */
1219 set_iopl.iopl = 1;
1220 rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
1221 if (rc != 0)
1222 xen_raw_printk("physdev_op failed %d\n", rc);
1223
Jeremy Fitzhardinge7d087b62008-07-08 15:06:48 -07001224#ifdef CONFIG_X86_32
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001225 /* set up basic CPUID stuff */
1226 cpu_detect(&new_cpu_data);
1227 new_cpu_data.hard_math = 1;
Jeremy Fitzhardinged560bc62009-08-25 12:53:02 -07001228 new_cpu_data.wp_works_ok = 1;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001229 new_cpu_data.x86_capability[0] = cpuid_edx(1);
Jeremy Fitzhardinge7d087b62008-07-08 15:06:48 -07001230#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001231
1232 /* Poke various useful things into boot_params */
H. Peter Anvin30c82642007-10-15 17:13:22 -07001233 boot_params.hdr.type_of_loader = (9 << 4) | 0;
1234 boot_params.hdr.ramdisk_image = xen_start_info->mod_start
1235 ? __pa(xen_start_info->mod_start) : 0;
1236 boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
Jeremy Fitzhardingeb7c3c5c2008-07-08 15:07:07 -07001237 boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001238
Jeremy Fitzhardinge6e833582008-08-19 13:16:17 -07001239 if (!xen_initial_domain()) {
Jeremy Fitzhardinge83abc702008-05-26 23:31:12 +01001240 add_preferred_console("xenboot", 0, NULL);
Markus Armbruster9e124fe2008-05-26 23:31:07 +01001241 add_preferred_console("tty", 0, NULL);
Markus Armbrusterb8c2d3d2008-02-27 14:56:35 +01001242 add_preferred_console("hvc", 0, NULL);
Alex Nixonb5401a92010-03-18 16:31:34 -04001243 if (pci_xen)
1244 x86_init.pci.arch_init = pci_xen_init;
Chris Wright5d990b62009-12-04 12:15:21 -08001245 } else {
1246 /* Make sure ACS will be enabled */
1247 pci_request_acs();
Markus Armbruster9e124fe2008-05-26 23:31:07 +01001248 }
Chris Wright5d990b62009-12-04 12:15:21 -08001249
Markus Armbrusterb8c2d3d2008-02-27 14:56:35 +01001250
Jeremy Fitzhardinge084a2a42008-07-08 15:06:50 -07001251 xen_raw_console_write("about to get started...\n");
1252
Jeremy Fitzhardinge499d19b2009-11-24 09:38:25 -08001253 xen_setup_runstate_info(0);
1254
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001255 /* Start the world */
Jeremy Fitzhardingef5d36de2008-07-08 15:06:39 -07001256#ifdef CONFIG_X86_32
Yinghai Luf0d43102008-05-29 12:56:36 -07001257 i386_start_kernel();
Jeremy Fitzhardingef5d36de2008-07-08 15:06:39 -07001258#else
Jeremy Fitzhardinge084a2a42008-07-08 15:06:50 -07001259 x86_64_start_reservations((char *)__pa_symbol(&boot_params));
Jeremy Fitzhardingef5d36de2008-07-08 15:06:39 -07001260#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001261}
Sheng Yangbee6ab52010-05-14 12:39:33 +01001262
1263static uint32_t xen_cpuid_base(void)
1264{
1265 uint32_t base, eax, ebx, ecx, edx;
1266 char signature[13];
1267
1268 for (base = 0x40000000; base < 0x40010000; base += 0x100) {
1269 cpuid(base, &eax, &ebx, &ecx, &edx);
1270 *(uint32_t *)(signature + 0) = ebx;
1271 *(uint32_t *)(signature + 4) = ecx;
1272 *(uint32_t *)(signature + 8) = edx;
1273 signature[12] = 0;
1274
1275 if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
1276 return base;
1277 }
1278
1279 return 0;
1280}
1281
1282static int init_hvm_pv_info(int *major, int *minor)
1283{
1284 uint32_t eax, ebx, ecx, edx, pages, msr, base;
1285 u64 pfn;
1286
1287 base = xen_cpuid_base();
1288 cpuid(base + 1, &eax, &ebx, &ecx, &edx);
1289
1290 *major = eax >> 16;
1291 *minor = eax & 0xffff;
1292 printk(KERN_INFO "Xen version %d.%d.\n", *major, *minor);
1293
1294 cpuid(base + 2, &pages, &msr, &ecx, &edx);
1295
1296 pfn = __pa(hypercall_page);
1297 wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));
1298
1299 xen_setup_features();
1300
1301 pv_info = xen_info;
1302 pv_info.kernel_rpl = 0;
1303
1304 xen_domain_type = XEN_HVM_DOMAIN;
1305
1306 return 0;
1307}
1308
Stefano Stabellini016b6f52010-05-14 12:45:07 +01001309void xen_hvm_init_shared_info(void)
Sheng Yangbee6ab52010-05-14 12:39:33 +01001310{
Stefano Stabellini016b6f52010-05-14 12:45:07 +01001311 int cpu;
Sheng Yangbee6ab52010-05-14 12:39:33 +01001312 struct xen_add_to_physmap xatp;
Stefano Stabellini016b6f52010-05-14 12:45:07 +01001313 static struct shared_info *shared_info_page = 0;
Sheng Yangbee6ab52010-05-14 12:39:33 +01001314
Stefano Stabellini016b6f52010-05-14 12:45:07 +01001315 if (!shared_info_page)
1316 shared_info_page = (struct shared_info *)
1317 extend_brk(PAGE_SIZE, PAGE_SIZE);
Sheng Yangbee6ab52010-05-14 12:39:33 +01001318 xatp.domid = DOMID_SELF;
1319 xatp.idx = 0;
1320 xatp.space = XENMAPSPACE_shared_info;
1321 xatp.gpfn = __pa(shared_info_page) >> PAGE_SHIFT;
1322 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
1323 BUG();
1324
1325 HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;
1326
Stefano Stabellini016b6f52010-05-14 12:45:07 +01001327 /* xen_vcpu is a pointer to the vcpu_info struct in the shared_info
1328 * page, we use it in the event channel upcall and in some pvclock
1329 * related functions. We don't need the vcpu_info placement
1330 * optimizations because we don't use any pv_mmu or pv_irq op on
1331 * HVM.
1332 * When xen_hvm_init_shared_info is run at boot time only vcpu 0 is
1333 * online but xen_hvm_init_shared_info is run at resume time too and
1334 * in that case multiple vcpus might be online. */
1335 for_each_online_cpu(cpu) {
1336 per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
1337 }
Sheng Yangbee6ab52010-05-14 12:39:33 +01001338}
1339
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001340#ifdef CONFIG_XEN_PVHVM
Sheng Yang38e20b02010-05-14 12:40:51 +01001341static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self,
1342 unsigned long action, void *hcpu)
1343{
1344 int cpu = (long)hcpu;
1345 switch (action) {
1346 case CPU_UP_PREPARE:
1347 per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
1348 break;
1349 default:
1350 break;
1351 }
1352 return NOTIFY_OK;
1353}
1354
1355static struct notifier_block __cpuinitdata xen_hvm_cpu_notifier = {
1356 .notifier_call = xen_hvm_cpu_notify,
1357};
1358
Sheng Yangbee6ab52010-05-14 12:39:33 +01001359static void __init xen_hvm_guest_init(void)
1360{
1361 int r;
1362 int major, minor;
1363
1364 r = init_hvm_pv_info(&major, &minor);
1365 if (r < 0)
1366 return;
1367
Stefano Stabellini016b6f52010-05-14 12:45:07 +01001368 xen_hvm_init_shared_info();
Sheng Yang38e20b02010-05-14 12:40:51 +01001369
1370 if (xen_feature(XENFEAT_hvm_callback_vector))
1371 xen_have_vector_callback = 1;
1372 register_cpu_notifier(&xen_hvm_cpu_notifier);
Stefano Stabellinic1c54132010-05-14 12:44:30 +01001373 xen_unplug_emulated_devices();
Sheng Yang38e20b02010-05-14 12:40:51 +01001374 have_vcpu_info_placement = 0;
1375 x86_init.irqs.intr_init = xen_init_IRQ;
Stefano Stabellini409771d2010-05-14 12:48:19 +01001376 xen_hvm_init_time_ops();
Stefano Stabellini59151002010-06-17 14:22:52 +01001377 xen_hvm_init_mmu_ops();
Sheng Yangbee6ab52010-05-14 12:39:33 +01001378}
1379
1380static bool __init xen_hvm_platform(void)
1381{
1382 if (xen_pv_domain())
1383 return false;
1384
1385 if (!xen_cpuid_base())
1386 return false;
1387
1388 return true;
1389}
1390
1391const __refconst struct hypervisor_x86 x86_hyper_xen_hvm = {
1392 .name = "Xen HVM",
1393 .detect = xen_hvm_platform,
1394 .init_platform = xen_hvm_guest_init,
1395};
1396EXPORT_SYMBOL(x86_hyper_xen_hvm);
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001397#endif