blob: 82cdcde4b222f5f41d802f75629ee8b300f9328b [file] [log] [blame]
Rusty Russelld3561b72006-12-07 02:14:07 +01001#ifndef __ASM_PARAVIRT_H
2#define __ASM_PARAVIRT_H
3/* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +02005
6#ifdef CONFIG_PARAVIRT
Rusty Russellda181a82006-12-07 02:14:08 +01007#include <asm/page.h>
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01008#include <asm/asm.h>
Rusty Russelld3561b72006-12-07 02:14:07 +01009
Rusty Russell139ec7c2006-12-07 02:14:08 +010010/* Bitmask of what can be clobbered: usually at least eax. */
Glauber de Oliveira Costa21438f72008-01-30 13:32:09 +010011#define CLBR_NONE 0
12#define CLBR_EAX (1 << 0)
13#define CLBR_ECX (1 << 1)
14#define CLBR_EDX (1 << 2)
15
16#ifdef CONFIG_X86_64
17#define CLBR_RSI (1 << 3)
18#define CLBR_RDI (1 << 4)
19#define CLBR_R8 (1 << 5)
20#define CLBR_R9 (1 << 6)
21#define CLBR_R10 (1 << 7)
22#define CLBR_R11 (1 << 8)
23#define CLBR_ANY ((1 << 9) - 1)
24#include <asm/desc_defs.h>
25#else
26/* CLBR_ANY should match all regs platform has. For i386, that's just it */
27#define CLBR_ANY ((1 << 3) - 1)
28#endif /* X86_64 */
Rusty Russell139ec7c2006-12-07 02:14:08 +010029
Rusty Russelld3561b72006-12-07 02:14:07 +010030#ifndef __ASSEMBLY__
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +020031#include <linux/types.h>
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +020032#include <linux/cpumask.h>
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +020033#include <asm/kmap_types.h>
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +010034#include <asm/desc_defs.h>
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +020035
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +020036struct page;
Rusty Russelld3561b72006-12-07 02:14:07 +010037struct thread_struct;
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +010038struct desc_ptr;
Rusty Russelld3561b72006-12-07 02:14:07 +010039struct tss_struct;
Rusty Russellda181a82006-12-07 02:14:08 +010040struct mm_struct;
Rusty Russell90a0a062007-05-02 19:27:10 +020041struct desc_struct;
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020042
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070043/* general info */
44struct pv_info {
Rusty Russelld3561b72006-12-07 02:14:07 +010045 unsigned int kernel_rpl;
Jeremy Fitzhardinge5311ab62007-05-02 19:27:13 +020046 int shared_kernel_pmd;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070047 int paravirt_enabled;
Rusty Russelld3561b72006-12-07 02:14:07 +010048 const char *name;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070049};
Rusty Russelld3561b72006-12-07 02:14:07 +010050
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070051struct pv_init_ops {
Rusty Russell139ec7c2006-12-07 02:14:08 +010052 /*
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070053 * Patch may replace one of the defined code sequences with
54 * arbitrary code, subject to the same register constraints.
55 * This generally means the code is not free to clobber any
56 * registers other than EAX. The patch function should return
57 * the number of bytes of code generated, as we nop pad the
58 * rest in generic code.
Rusty Russell139ec7c2006-12-07 02:14:08 +010059 */
Andi Kleenab144f52007-08-10 22:31:03 +020060 unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
61 unsigned long addr, unsigned len);
Rusty Russell139ec7c2006-12-07 02:14:08 +010062
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020063 /* Basic arch-specific setup */
Rusty Russelld3561b72006-12-07 02:14:07 +010064 void (*arch_setup)(void);
65 char *(*memory_setup)(void);
Jeremy Fitzhardinge6996d3b2007-07-17 18:37:03 -070066 void (*post_allocator_init)(void);
67
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020068 /* Print a banner to identify the environment */
Rusty Russelld3561b72006-12-07 02:14:07 +010069 void (*banner)(void);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070070};
71
72
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -070073struct pv_lazy_ops {
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070074 /* Set deferred update mode, used for batching operations. */
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -070075 void (*enter)(void);
76 void (*leave)(void);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070077};
78
79struct pv_time_ops {
80 void (*time_init)(void);
Rusty Russelld3561b72006-12-07 02:14:07 +010081
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020082 /* Set and set time of day */
Rusty Russelld3561b72006-12-07 02:14:07 +010083 unsigned long (*get_wallclock)(void);
84 int (*set_wallclock)(unsigned long);
Rusty Russelld3561b72006-12-07 02:14:07 +010085
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070086 unsigned long long (*sched_clock)(void);
87 unsigned long (*get_cpu_khz)(void);
88};
Rusty Russelld3561b72006-12-07 02:14:07 +010089
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070090struct pv_cpu_ops {
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020091 /* hooks for various privileged instructions */
Andi Kleen1a1eecd2007-02-13 13:26:25 +010092 unsigned long (*get_debugreg)(int regno);
93 void (*set_debugreg)(int regno, unsigned long value);
Rusty Russelld3561b72006-12-07 02:14:07 +010094
Andi Kleen1a1eecd2007-02-13 13:26:25 +010095 void (*clts)(void);
Rusty Russelld3561b72006-12-07 02:14:07 +010096
Andi Kleen1a1eecd2007-02-13 13:26:25 +010097 unsigned long (*read_cr0)(void);
98 void (*write_cr0)(unsigned long);
Rusty Russelld3561b72006-12-07 02:14:07 +010099
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100100 unsigned long (*read_cr4_safe)(void);
101 unsigned long (*read_cr4)(void);
102 void (*write_cr4)(unsigned long);
Rusty Russelld3561b72006-12-07 02:14:07 +0100103
Glauber de Oliveira Costa4c9890c2008-01-30 13:33:19 +0100104#ifdef CONFIG_X86_64
105 unsigned long (*read_cr8)(void);
106 void (*write_cr8)(unsigned long);
107#endif
108
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200109 /* Segment descriptor handling */
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100110 void (*load_tr_desc)(void);
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100111 void (*load_gdt)(const struct desc_ptr *);
112 void (*load_idt)(const struct desc_ptr *);
113 void (*store_gdt)(struct desc_ptr *);
114 void (*store_idt)(struct desc_ptr *);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100115 void (*set_ldt)(const void *desc, unsigned entries);
116 unsigned long (*store_tr)(void);
117 void (*load_tls)(struct thread_struct *t, unsigned int cpu);
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100118 void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
119 const void *desc);
Rusty Russell90a0a062007-05-02 19:27:10 +0200120 void (*write_gdt_entry)(struct desc_struct *,
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100121 int entrynum, const void *desc, int size);
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100122 void (*write_idt_entry)(gate_desc *,
123 int entrynum, const gate_desc *gate);
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100124 void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t);
Rusty Russelld3561b72006-12-07 02:14:07 +0100125
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100126 void (*set_iopl_mask)(unsigned mask);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700127
128 void (*wbinvd)(void);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100129 void (*io_delay)(void);
Rusty Russelld3561b72006-12-07 02:14:07 +0100130
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700131 /* cpuid emulation, mostly so that caps bits can be disabled */
132 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
133 unsigned int *ecx, unsigned int *edx);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200134
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700135 /* MSR, PMC and TSR operations.
136 err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */
137 u64 (*read_msr)(unsigned int msr, int *err);
Glauber de Oliveira Costac9dcda52008-01-30 13:31:07 +0100138 int (*write_msr)(unsigned int msr, unsigned low, unsigned high);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700139
140 u64 (*read_tsc)(void);
Glauber de Oliveira Costab8d1fae2008-01-30 13:31:07 +0100141 u64 (*read_pmc)(int counter);
Glauber de Oliveira Costae5aaac42008-01-30 13:32:05 +0100142 unsigned long long (*read_tscp)(unsigned int *aux);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700143
144 /* These two are jmp to, not actually called. */
Glauber de Oliveira Costa6abcd982008-01-30 13:30:33 +0100145 void (*irq_enable_syscall_ret)(void);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700146 void (*iret)(void);
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700147
Glauber de Oliveira Costae801f862008-01-30 13:32:08 +0100148 void (*swapgs)(void);
149
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700150 struct pv_lazy_ops lazy_mode;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700151};
152
153struct pv_irq_ops {
154 void (*init_IRQ)(void);
155
156 /*
157 * Get/set interrupt state. save_fl and restore_fl are only
158 * expected to use X86_EFLAGS_IF; all other bits
159 * returned from save_fl are undefined, and may be ignored by
160 * restore_fl.
161 */
162 unsigned long (*save_fl)(void);
163 void (*restore_fl)(unsigned long);
164 void (*irq_disable)(void);
165 void (*irq_enable)(void);
166 void (*safe_halt)(void);
167 void (*halt)(void);
168};
169
170struct pv_apic_ops {
Rusty Russell13623d72006-12-07 02:14:08 +0100171#ifdef CONFIG_X86_LOCAL_APIC
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200172 /*
173 * Direct APIC operations, principally for VMI. Ideally
174 * these shouldn't be in this interface.
175 */
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100176 void (*apic_write)(unsigned long reg, u32 v);
177 void (*apic_write_atomic)(unsigned long reg, u32 v);
178 u32 (*apic_read)(unsigned long reg);
Zachary Amsdenbbab4f32007-02-13 13:26:21 +0100179 void (*setup_boot_clock)(void);
180 void (*setup_secondary_clock)(void);
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200181
182 void (*startup_ipi_hook)(int phys_apicid,
183 unsigned long start_eip,
184 unsigned long start_esp);
Rusty Russell13623d72006-12-07 02:14:08 +0100185#endif
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700186};
187
188struct pv_mmu_ops {
189 /*
190 * Called before/after init_mm pagetable setup. setup_start
191 * may reset %cr3, and may pre-install parts of the pagetable;
192 * pagetable setup is expected to preserve any existing
193 * mapping.
194 */
195 void (*pagetable_setup_start)(pgd_t *pgd_base);
196 void (*pagetable_setup_done)(pgd_t *pgd_base);
197
198 unsigned long (*read_cr2)(void);
199 void (*write_cr2)(unsigned long);
200
201 unsigned long (*read_cr3)(void);
202 void (*write_cr3)(unsigned long);
203
204 /*
205 * Hooks for intercepting the creation/use/destruction of an
206 * mm_struct.
207 */
208 void (*activate_mm)(struct mm_struct *prev,
209 struct mm_struct *next);
210 void (*dup_mmap)(struct mm_struct *oldmm,
211 struct mm_struct *mm);
212 void (*exit_mmap)(struct mm_struct *mm);
213
Rusty Russell13623d72006-12-07 02:14:08 +0100214
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200215 /* TLB operations */
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100216 void (*flush_tlb_user)(void);
217 void (*flush_tlb_kernel)(void);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200218 void (*flush_tlb_single)(unsigned long addr);
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200219 void (*flush_tlb_others)(const cpumask_t *cpus, struct mm_struct *mm,
220 unsigned long va);
Rusty Russellda181a82006-12-07 02:14:08 +0100221
Jeremy Fitzhardingeeba00452008-06-25 00:19:12 -0400222 /* Hooks for allocating and freeing a pagetable top-level */
223 int (*pgd_alloc)(struct mm_struct *mm);
224 void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
225
226 /*
227 * Hooks for allocating/releasing pagetable pages when they're
228 * attached to a pagetable
229 */
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700230 void (*alloc_pte)(struct mm_struct *mm, u32 pfn);
231 void (*alloc_pmd)(struct mm_struct *mm, u32 pfn);
232 void (*alloc_pmd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count);
Jeremy Fitzhardinge2761fa02008-03-17 16:37:02 -0700233 void (*alloc_pud)(struct mm_struct *mm, u32 pfn);
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700234 void (*release_pte)(u32 pfn);
235 void (*release_pmd)(u32 pfn);
Jeremy Fitzhardinge2761fa02008-03-17 16:37:02 -0700236 void (*release_pud)(u32 pfn);
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100237
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200238 /* Pagetable manipulation functions */
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100239 void (*set_pte)(pte_t *ptep, pte_t pteval);
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200240 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
241 pte_t *ptep, pte_t pteval);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100242 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
Joe Perches49cd7402008-03-23 01:03:00 -0700243 void (*pte_update)(struct mm_struct *mm, unsigned long addr,
244 pte_t *ptep);
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200245 void (*pte_update_defer)(struct mm_struct *mm,
246 unsigned long addr, pte_t *ptep);
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200247
Jeremy Fitzhardinge08b882c2008-06-16 04:30:01 -0700248 pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
249 pte_t *ptep);
250 void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
251 pte_t *ptep, pte_t pte);
252
Jeremy Fitzhardinge5b8dd1e2008-01-30 13:33:15 +0100253 pteval_t (*pte_val)(pte_t);
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100254 pteval_t (*pte_flags)(pte_t);
Jeremy Fitzhardinge5b8dd1e2008-01-30 13:33:15 +0100255 pte_t (*make_pte)(pteval_t pte);
256
257 pgdval_t (*pgd_val)(pgd_t);
258 pgd_t (*make_pgd)(pgdval_t pgd);
259
260#if PAGETABLE_LEVELS >= 3
Rusty Russellda181a82006-12-07 02:14:08 +0100261#ifdef CONFIG_X86_PAE
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100262 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700263 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr,
264 pte_t *ptep, pte_t pte);
Joe Perches49cd7402008-03-23 01:03:00 -0700265 void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
266 pte_t *ptep);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100267 void (*pmd_clear)(pmd_t *pmdp);
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200268
Jeremy Fitzhardinge5b8dd1e2008-01-30 13:33:15 +0100269#endif /* CONFIG_X86_PAE */
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200270
Jeremy Fitzhardinge5b8dd1e2008-01-30 13:33:15 +0100271 void (*set_pud)(pud_t *pudp, pud_t pudval);
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200272
Jeremy Fitzhardinge5b8dd1e2008-01-30 13:33:15 +0100273 pmdval_t (*pmd_val)(pmd_t);
274 pmd_t (*make_pmd)(pmdval_t pmd);
275
276#if PAGETABLE_LEVELS == 4
277 pudval_t (*pud_val)(pud_t);
278 pud_t (*make_pud)(pudval_t pud);
Eduardo Habkost90422192008-01-30 13:33:20 +0100279
280 void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
Jeremy Fitzhardinge5b8dd1e2008-01-30 13:33:15 +0100281#endif /* PAGETABLE_LEVELS == 4 */
282#endif /* PAGETABLE_LEVELS >= 3 */
Rusty Russellda181a82006-12-07 02:14:08 +0100283
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700284#ifdef CONFIG_HIGHPTE
285 void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
286#endif
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700287
288 struct pv_lazy_ops lazy_mode;
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700289
290 /* dom0 ops */
291
292 /* Sometimes the physical address is a pfn, and sometimes its
293 an mfn. We can tell which is which from the index. */
294 void (*set_fixmap)(unsigned /* enum fixed_addresses */ idx,
295 unsigned long phys, pgprot_t flags);
Rusty Russelld3561b72006-12-07 02:14:07 +0100296};
297
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700298/* This contains all the paravirt structures: we get a convenient
299 * number for each function using the offset which we use to indicate
300 * what to patch. */
Joe Perches49cd7402008-03-23 01:03:00 -0700301struct paravirt_patch_template {
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700302 struct pv_init_ops pv_init_ops;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700303 struct pv_time_ops pv_time_ops;
304 struct pv_cpu_ops pv_cpu_ops;
305 struct pv_irq_ops pv_irq_ops;
306 struct pv_apic_ops pv_apic_ops;
307 struct pv_mmu_ops pv_mmu_ops;
308};
309
310extern struct pv_info pv_info;
311extern struct pv_init_ops pv_init_ops;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700312extern struct pv_time_ops pv_time_ops;
313extern struct pv_cpu_ops pv_cpu_ops;
314extern struct pv_irq_ops pv_irq_ops;
315extern struct pv_apic_ops pv_apic_ops;
316extern struct pv_mmu_ops pv_mmu_ops;
Rusty Russelld3561b72006-12-07 02:14:07 +0100317
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200318#define PARAVIRT_PATCH(x) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700319 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200320
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700321#define paravirt_type(op) \
322 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
323 [paravirt_opptr] "m" (op)
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200324#define paravirt_clobber(clobber) \
325 [paravirt_clobber] "i" (clobber)
326
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200327/*
328 * Generate some code, and mark it as patchable by the
329 * apply_paravirt() alternate instruction patcher.
330 */
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200331#define _paravirt_alt(insn_string, type, clobber) \
332 "771:\n\t" insn_string "\n" "772:\n" \
333 ".pushsection .parainstructions,\"a\"\n" \
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100334 _ASM_ALIGN "\n" \
335 _ASM_PTR " 771b\n" \
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200336 " .byte " type "\n" \
337 " .byte 772b-771b\n" \
338 " .short " clobber "\n" \
339 ".popsection\n"
340
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200341/* Generate patchable code, with the default asm parameters. */
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200342#define paravirt_alt(insn_string) \
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200343 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
344
Glauber de Oliveira Costa2f485ef2008-01-30 13:32:10 +0100345/* Simple instruction patching code. */
346#define DEF_NATIVE(ops, name, code) \
347 extern const char start_##ops##_##name[], end_##ops##_##name[]; \
348 asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
349
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200350unsigned paravirt_patch_nop(void);
351unsigned paravirt_patch_ignore(unsigned len);
Andi Kleenab144f52007-08-10 22:31:03 +0200352unsigned paravirt_patch_call(void *insnbuf,
353 const void *target, u16 tgt_clobbers,
354 unsigned long addr, u16 site_clobbers,
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200355 unsigned len);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700356unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
Andi Kleenab144f52007-08-10 22:31:03 +0200357 unsigned long addr, unsigned len);
358unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
359 unsigned long addr, unsigned len);
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200360
Andi Kleenab144f52007-08-10 22:31:03 +0200361unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200362 const char *start, const char *end);
363
Glauber de Oliveira Costa2f485ef2008-01-30 13:32:10 +0100364unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
365 unsigned long addr, unsigned len);
366
Jeremy Fitzhardinged5729292007-07-17 18:37:04 -0700367int paravirt_disable_iospace(void);
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200368
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200369/*
370 * This generates an indirect call based on the operation type number.
371 * The type number, computed in PARAVIRT_PATCH, is derived from the
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700372 * offset into the paravirt_patch_template structure, and can therefore be
373 * freely converted back into a structure offset.
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200374 */
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700375#define PARAVIRT_CALL "call *%[paravirt_opptr];"
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200376
377/*
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700378 * These macros are intended to wrap calls through one of the paravirt
379 * ops structs, so that they can be later identified and patched at
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200380 * runtime.
381 *
382 * Normally, a call to a pv_op function is a simple indirect call:
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100383 * (pv_op_struct.operations)(args...).
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200384 *
385 * Unfortunately, this is a relatively slow operation for modern CPUs,
386 * because it cannot necessarily determine what the destination
387 * address is. In this case, the address is a runtime constant, so at
388 * the very least we can patch the call to e a simple direct call, or
389 * ideally, patch an inline implementation into the callsite. (Direct
390 * calls are essentially free, because the call and return addresses
391 * are completely predictable.)
392 *
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100393 * For i386, these macros rely on the standard gcc "regparm(3)" calling
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200394 * convention, in which the first three arguments are placed in %eax,
395 * %edx, %ecx (in that order), and the remaining arguments are placed
396 * on the stack. All caller-save registers (eax,edx,ecx) are expected
397 * to be modified (either clobbered or used for return values).
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100398 * X86_64, on the other hand, already specifies a register-based calling
399 * conventions, returning at %rax, with parameteres going on %rdi, %rsi,
400 * %rdx, and %rcx. Note that for this reason, x86_64 does not need any
401 * special handling for dealing with 4 arguments, unlike i386.
402 * However, x86_64 also have to clobber all caller saved registers, which
403 * unfortunately, are quite a bit (r8 - r11)
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200404 *
405 * The call instruction itself is marked by placing its start address
406 * and size into the .parainstructions section, so that
407 * apply_paravirt() in arch/i386/kernel/alternative.c can do the
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700408 * appropriate patching under the control of the backend pv_init_ops
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200409 * implementation.
410 *
411 * Unfortunately there's no way to get gcc to generate the args setup
412 * for the call, and then allow the call itself to be generated by an
413 * inline asm. Because of this, we must do the complete arg setup and
414 * return value handling from within these macros. This is fairly
415 * cumbersome.
416 *
417 * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments.
418 * It could be extended to more arguments, but there would be little
419 * to be gained from that. For each number of arguments, there are
420 * the two VCALL and CALL variants for void and non-void functions.
421 *
422 * When there is a return value, the invoker of the macro must specify
423 * the return type. The macro then uses sizeof() on that type to
424 * determine whether its a 32 or 64 bit value, and places the return
425 * in the right register(s) (just %eax for 32-bit, and %edx:%eax for
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100426 * 64-bit). For x86_64 machines, it just returns at %rax regardless of
427 * the return value size.
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200428 *
429 * 64-bit arguments are passed as a pair of adjacent 32-bit arguments
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100430 * i386 also passes 64-bit arguments as a pair of adjacent 32-bit arguments
431 * in low,high order
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200432 *
433 * Small structures are passed and returned in registers. The macro
434 * calling convention can't directly deal with this, so the wrapper
435 * functions must do this.
436 *
437 * These PVOP_* macros are only defined within this header. This
438 * means that all uses must be wrapped in inline functions. This also
439 * makes sure the incoming and outgoing types are always correct.
440 */
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100441#ifdef CONFIG_X86_32
442#define PVOP_VCALL_ARGS unsigned long __eax, __edx, __ecx
443#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
444#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
445 "=c" (__ecx)
446#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
447#define EXTRA_CLOBBERS
448#define VEXTRA_CLOBBERS
449#else
450#define PVOP_VCALL_ARGS unsigned long __edi, __esi, __edx, __ecx
451#define PVOP_CALL_ARGS PVOP_VCALL_ARGS, __eax
452#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
453 "=S" (__esi), "=d" (__edx), \
454 "=c" (__ecx)
455
456#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
457
458#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
459#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
460#endif
461
Jeremy Fitzhardinge97349132008-06-25 00:19:14 -0400462#ifdef CONFIG_PARAVIRT_DEBUG
463#define PVOP_TEST_NULL(op) BUG_ON(op == NULL)
464#else
465#define PVOP_TEST_NULL(op) ((void)op)
466#endif
467
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200468#define __PVOP_CALL(rettype, op, pre, post, ...) \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200469 ({ \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200470 rettype __ret; \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100471 PVOP_CALL_ARGS; \
Jeremy Fitzhardinge97349132008-06-25 00:19:14 -0400472 PVOP_TEST_NULL(op); \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100473 /* This is 32-bit specific, but is okay in 64-bit */ \
474 /* since this condition will never hold */ \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200475 if (sizeof(rettype) > sizeof(unsigned long)) { \
476 asm volatile(pre \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200477 paravirt_alt(PARAVIRT_CALL) \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200478 post \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100479 : PVOP_CALL_CLOBBERS \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200480 : paravirt_type(op), \
481 paravirt_clobber(CLBR_ANY), \
482 ##__VA_ARGS__ \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100483 : "memory", "cc" EXTRA_CLOBBERS); \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200484 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
485 } else { \
486 asm volatile(pre \
487 paravirt_alt(PARAVIRT_CALL) \
488 post \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100489 : PVOP_CALL_CLOBBERS \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200490 : paravirt_type(op), \
491 paravirt_clobber(CLBR_ANY), \
492 ##__VA_ARGS__ \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100493 : "memory", "cc" EXTRA_CLOBBERS); \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200494 __ret = (rettype)__eax; \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200495 } \
496 __ret; \
497 })
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200498#define __PVOP_VCALL(op, pre, post, ...) \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200499 ({ \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100500 PVOP_VCALL_ARGS; \
Jeremy Fitzhardinge97349132008-06-25 00:19:14 -0400501 PVOP_TEST_NULL(op); \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200502 asm volatile(pre \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200503 paravirt_alt(PARAVIRT_CALL) \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200504 post \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100505 : PVOP_VCALL_CLOBBERS \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200506 : paravirt_type(op), \
507 paravirt_clobber(CLBR_ANY), \
508 ##__VA_ARGS__ \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100509 : "memory", "cc" VEXTRA_CLOBBERS); \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200510 })
511
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200512#define PVOP_CALL0(rettype, op) \
513 __PVOP_CALL(rettype, op, "", "")
514#define PVOP_VCALL0(op) \
515 __PVOP_VCALL(op, "", "")
516
517#define PVOP_CALL1(rettype, op, arg1) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100518 __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200519#define PVOP_VCALL1(op, arg1) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100520 __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200521
522#define PVOP_CALL2(rettype, op, arg1, arg2) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100523 __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
524 "1" ((unsigned long)(arg2)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200525#define PVOP_VCALL2(op, arg1, arg2) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100526 __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
527 "1" ((unsigned long)(arg2)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200528
529#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100530 __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
531 "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200532#define PVOP_VCALL3(op, arg1, arg2, arg3) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100533 __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
534 "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200535
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100536/* This is the only difference in x86_64. We can make it much simpler */
537#ifdef CONFIG_X86_32
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200538#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
539 __PVOP_CALL(rettype, op, \
540 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
541 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
542 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
543#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
544 __PVOP_VCALL(op, \
545 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
546 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
547 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100548#else
549#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
550 __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
551 "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \
552 "3"((unsigned long)(arg4)))
553#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
554 __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
555 "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \
556 "3"((unsigned long)(arg4)))
557#endif
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200558
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200559static inline int paravirt_enabled(void)
560{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700561 return pv_info.paravirt_enabled;
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200562}
Rusty Russelld3561b72006-12-07 02:14:07 +0100563
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100564static inline void load_sp0(struct tss_struct *tss,
Rusty Russelld3561b72006-12-07 02:14:07 +0100565 struct thread_struct *thread)
566{
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100567 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
Rusty Russelld3561b72006-12-07 02:14:07 +0100568}
569
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700570#define ARCH_SETUP pv_init_ops.arch_setup();
Rusty Russelld3561b72006-12-07 02:14:07 +0100571static inline unsigned long get_wallclock(void)
572{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700573 return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
Rusty Russelld3561b72006-12-07 02:14:07 +0100574}
575
576static inline int set_wallclock(unsigned long nowtime)
577{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700578 return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
Rusty Russelld3561b72006-12-07 02:14:07 +0100579}
580
Zachary Amsdene30fab32007-03-05 00:30:39 -0800581static inline void (*choose_time_init(void))(void)
Rusty Russelld3561b72006-12-07 02:14:07 +0100582{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700583 return pv_time_ops.time_init;
Rusty Russelld3561b72006-12-07 02:14:07 +0100584}
585
586/* The paravirtualized CPUID instruction. */
587static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
588 unsigned int *ecx, unsigned int *edx)
589{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700590 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
Rusty Russelld3561b72006-12-07 02:14:07 +0100591}
592
593/*
594 * These special macros can be used to get or set a debugging register
595 */
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200596static inline unsigned long paravirt_get_debugreg(int reg)
597{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700598 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200599}
600#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
601static inline void set_debugreg(unsigned long val, int reg)
602{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700603 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200604}
Rusty Russelld3561b72006-12-07 02:14:07 +0100605
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200606static inline void clts(void)
607{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700608 PVOP_VCALL0(pv_cpu_ops.clts);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200609}
Rusty Russelld3561b72006-12-07 02:14:07 +0100610
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200611static inline unsigned long read_cr0(void)
612{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700613 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200614}
Rusty Russelld3561b72006-12-07 02:14:07 +0100615
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200616static inline void write_cr0(unsigned long x)
617{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700618 PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200619}
Rusty Russelld3561b72006-12-07 02:14:07 +0100620
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200621static inline unsigned long read_cr2(void)
622{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700623 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200624}
Rusty Russelld3561b72006-12-07 02:14:07 +0100625
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200626static inline void write_cr2(unsigned long x)
627{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700628 PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200629}
Rusty Russelld3561b72006-12-07 02:14:07 +0100630
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200631static inline unsigned long read_cr3(void)
632{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700633 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200634}
635
636static inline void write_cr3(unsigned long x)
637{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700638 PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200639}
640
641static inline unsigned long read_cr4(void)
642{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700643 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200644}
645static inline unsigned long read_cr4_safe(void)
646{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700647 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200648}
649
650static inline void write_cr4(unsigned long x)
651{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700652 PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200653}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200654
Glauber de Oliveira Costa94ea03c2008-01-30 13:33:19 +0100655#ifdef CONFIG_X86_64
Glauber de Oliveira Costa4c9890c2008-01-30 13:33:19 +0100656static inline unsigned long read_cr8(void)
657{
658 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
659}
660
661static inline void write_cr8(unsigned long x)
662{
663 PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
664}
Glauber de Oliveira Costa94ea03c2008-01-30 13:33:19 +0100665#endif
Glauber de Oliveira Costa4c9890c2008-01-30 13:33:19 +0100666
Rusty Russelld3561b72006-12-07 02:14:07 +0100667static inline void raw_safe_halt(void)
668{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700669 PVOP_VCALL0(pv_irq_ops.safe_halt);
Rusty Russelld3561b72006-12-07 02:14:07 +0100670}
671
672static inline void halt(void)
673{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700674 PVOP_VCALL0(pv_irq_ops.safe_halt);
Rusty Russelld3561b72006-12-07 02:14:07 +0100675}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200676
677static inline void wbinvd(void)
678{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700679 PVOP_VCALL0(pv_cpu_ops.wbinvd);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200680}
Rusty Russelld3561b72006-12-07 02:14:07 +0100681
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700682#define get_kernel_rpl() (pv_info.kernel_rpl)
Rusty Russelld3561b72006-12-07 02:14:07 +0100683
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200684static inline u64 paravirt_read_msr(unsigned msr, int *err)
685{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700686 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200687}
688static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
689{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700690 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200691}
692
Rusty Russell90a0a062007-05-02 19:27:10 +0200693/* These should all do BUG_ON(_err), but our headers are too tangled. */
Joe Perches49cd7402008-03-23 01:03:00 -0700694#define rdmsr(msr, val1, val2) \
695do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200696 int _err; \
697 u64 _l = paravirt_read_msr(msr, &_err); \
698 val1 = (u32)_l; \
699 val2 = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700700} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100701
Joe Perches49cd7402008-03-23 01:03:00 -0700702#define wrmsr(msr, val1, val2) \
703do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200704 paravirt_write_msr(msr, val1, val2); \
Joe Perches49cd7402008-03-23 01:03:00 -0700705} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100706
Joe Perches49cd7402008-03-23 01:03:00 -0700707#define rdmsrl(msr, val) \
708do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200709 int _err; \
710 val = paravirt_read_msr(msr, &_err); \
Joe Perches49cd7402008-03-23 01:03:00 -0700711} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100712
Joe Perches49cd7402008-03-23 01:03:00 -0700713#define wrmsrl(msr, val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
714#define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b)
Rusty Russelld3561b72006-12-07 02:14:07 +0100715
716/* rdmsr with exception handling */
Joe Perches49cd7402008-03-23 01:03:00 -0700717#define rdmsr_safe(msr, a, b) \
718({ \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200719 int _err; \
720 u64 _l = paravirt_read_msr(msr, &_err); \
721 (*a) = (u32)_l; \
722 (*b) = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700723 _err; \
724})
Rusty Russelld3561b72006-12-07 02:14:07 +0100725
Andi Kleen1de87bd2008-03-22 10:59:28 +0100726static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
727{
728 int err;
729
730 *p = paravirt_read_msr(msr, &err);
731 return err;
732}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200733
734static inline u64 paravirt_read_tsc(void)
735{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700736 return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200737}
Rusty Russelld3561b72006-12-07 02:14:07 +0100738
Joe Perches49cd7402008-03-23 01:03:00 -0700739#define rdtscl(low) \
740do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200741 u64 _l = paravirt_read_tsc(); \
742 low = (int)_l; \
Joe Perches49cd7402008-03-23 01:03:00 -0700743} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100744
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200745#define rdtscll(val) (val = paravirt_read_tsc())
Rusty Russelld3561b72006-12-07 02:14:07 +0100746
Jeremy Fitzhardinge688340e2007-07-17 18:37:04 -0700747static inline unsigned long long paravirt_sched_clock(void)
748{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700749 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
Jeremy Fitzhardinge688340e2007-07-17 18:37:04 -0700750}
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700751#define calculate_cpu_khz() (pv_time_ops.get_cpu_khz())
Zachary Amsden6cb9a832007-03-05 00:30:35 -0800752
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200753static inline unsigned long long paravirt_read_pmc(int counter)
754{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700755 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200756}
757
Joe Perches49cd7402008-03-23 01:03:00 -0700758#define rdpmc(counter, low, high) \
759do { \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200760 u64 _l = paravirt_read_pmc(counter); \
761 low = (u32)_l; \
762 high = _l >> 32; \
Joe Perches49cd7402008-03-23 01:03:00 -0700763} while (0)
Rusty Russelld3561b72006-12-07 02:14:07 +0100764
Glauber de Oliveira Costae5aaac42008-01-30 13:32:05 +0100765static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
766{
767 return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
768}
769
770#define rdtscp(low, high, aux) \
771do { \
772 int __aux; \
773 unsigned long __val = paravirt_rdtscp(&__aux); \
774 (low) = (u32)__val; \
775 (high) = (u32)(__val >> 32); \
776 (aux) = __aux; \
777} while (0)
778
779#define rdtscpll(val, aux) \
780do { \
781 unsigned long __aux; \
782 val = paravirt_rdtscp(&__aux); \
783 (aux) = __aux; \
784} while (0)
785
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200786static inline void load_TR_desc(void)
787{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700788 PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200789}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100790static inline void load_gdt(const struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200791{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700792 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200793}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100794static inline void load_idt(const struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200795{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700796 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200797}
798static inline void set_ldt(const void *addr, unsigned entries)
799{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700800 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200801}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100802static inline void store_gdt(struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200803{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700804 PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200805}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100806static inline void store_idt(struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200807{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700808 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200809}
810static inline unsigned long paravirt_store_tr(void)
811{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700812 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200813}
814#define store_tr(tr) ((tr) = paravirt_store_tr())
815static inline void load_TLS(struct thread_struct *t, unsigned cpu)
816{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700817 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200818}
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100819
820static inline void write_ldt_entry(struct desc_struct *dt, int entry,
821 const void *desc)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200822{
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100823 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200824}
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100825
826static inline void write_gdt_entry(struct desc_struct *dt, int entry,
827 void *desc, int type)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200828{
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100829 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200830}
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100831
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100832static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200833{
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100834 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200835}
836static inline void set_iopl_mask(unsigned mask)
837{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700838 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200839}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200840
Rusty Russelld3561b72006-12-07 02:14:07 +0100841/* The paravirtualized I/O functions */
Joe Perches49cd7402008-03-23 01:03:00 -0700842static inline void slow_down_io(void)
843{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700844 pv_cpu_ops.io_delay();
Rusty Russelld3561b72006-12-07 02:14:07 +0100845#ifdef REALLY_SLOW_IO
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700846 pv_cpu_ops.io_delay();
847 pv_cpu_ops.io_delay();
848 pv_cpu_ops.io_delay();
Rusty Russelld3561b72006-12-07 02:14:07 +0100849#endif
850}
851
Rusty Russell13623d72006-12-07 02:14:08 +0100852#ifdef CONFIG_X86_LOCAL_APIC
853/*
854 * Basic functions accessing APICs.
855 */
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100856static inline void apic_write(unsigned long reg, u32 v)
Rusty Russell13623d72006-12-07 02:14:08 +0100857{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700858 PVOP_VCALL2(pv_apic_ops.apic_write, reg, v);
Rusty Russell13623d72006-12-07 02:14:08 +0100859}
860
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100861static inline void apic_write_atomic(unsigned long reg, u32 v)
Rusty Russell13623d72006-12-07 02:14:08 +0100862{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700863 PVOP_VCALL2(pv_apic_ops.apic_write_atomic, reg, v);
Rusty Russell13623d72006-12-07 02:14:08 +0100864}
865
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100866static inline u32 apic_read(unsigned long reg)
Rusty Russell13623d72006-12-07 02:14:08 +0100867{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700868 return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg);
Rusty Russell13623d72006-12-07 02:14:08 +0100869}
Zachary Amsdenbbab4f32007-02-13 13:26:21 +0100870
871static inline void setup_boot_clock(void)
872{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700873 PVOP_VCALL0(pv_apic_ops.setup_boot_clock);
Zachary Amsdenbbab4f32007-02-13 13:26:21 +0100874}
875
876static inline void setup_secondary_clock(void)
877{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700878 PVOP_VCALL0(pv_apic_ops.setup_secondary_clock);
Zachary Amsdenbbab4f32007-02-13 13:26:21 +0100879}
Rusty Russell13623d72006-12-07 02:14:08 +0100880#endif
881
Jeremy Fitzhardinge6996d3b2007-07-17 18:37:03 -0700882static inline void paravirt_post_allocator_init(void)
883{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700884 if (pv_init_ops.post_allocator_init)
885 (*pv_init_ops.post_allocator_init)();
Jeremy Fitzhardinge6996d3b2007-07-17 18:37:03 -0700886}
887
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200888static inline void paravirt_pagetable_setup_start(pgd_t *base)
889{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700890 (*pv_mmu_ops.pagetable_setup_start)(base);
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200891}
892
893static inline void paravirt_pagetable_setup_done(pgd_t *base)
894{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700895 (*pv_mmu_ops.pagetable_setup_done)(base);
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200896}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200897
Zachary Amsdenae5da272007-02-13 13:26:21 +0100898#ifdef CONFIG_SMP
899static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
900 unsigned long start_esp)
901{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700902 PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
903 phys_apicid, start_eip, start_esp);
Zachary Amsdenae5da272007-02-13 13:26:21 +0100904}
905#endif
Rusty Russell13623d72006-12-07 02:14:08 +0100906
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200907static inline void paravirt_activate_mm(struct mm_struct *prev,
908 struct mm_struct *next)
909{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700910 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200911}
912
913static inline void arch_dup_mmap(struct mm_struct *oldmm,
914 struct mm_struct *mm)
915{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700916 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200917}
918
919static inline void arch_exit_mmap(struct mm_struct *mm)
920{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700921 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200922}
923
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200924static inline void __flush_tlb(void)
925{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700926 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200927}
928static inline void __flush_tlb_global(void)
929{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700930 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200931}
932static inline void __flush_tlb_single(unsigned long addr)
933{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700934 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200935}
Rusty Russellda181a82006-12-07 02:14:08 +0100936
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200937static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
938 unsigned long va)
939{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700940 PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va);
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200941}
942
Jeremy Fitzhardingeeba00452008-06-25 00:19:12 -0400943static inline int paravirt_pgd_alloc(struct mm_struct *mm)
944{
945 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
946}
947
948static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
949{
950 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
951}
952
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700953static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200954{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700955 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200956}
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700957static inline void paravirt_release_pte(unsigned pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200958{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700959 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200960}
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100961
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700962static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200963{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700964 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200965}
966
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700967static inline void paravirt_alloc_pmd_clone(unsigned pfn, unsigned clonepfn,
968 unsigned start, unsigned count)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200969{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700970 PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200971}
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700972static inline void paravirt_release_pmd(unsigned pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200973{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700974 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200975}
976
Jeremy Fitzhardinge2761fa02008-03-17 16:37:02 -0700977static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned pfn)
978{
979 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
980}
981static inline void paravirt_release_pud(unsigned pfn)
982{
983 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
984}
985
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +0200986#ifdef CONFIG_HIGHPTE
987static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
988{
989 unsigned long ret;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700990 ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type);
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +0200991 return (void *)ret;
992}
993#endif
994
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200995static inline void pte_update(struct mm_struct *mm, unsigned long addr,
996 pte_t *ptep)
997{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700998 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200999}
1000
1001static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
1002 pte_t *ptep)
1003{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001004 PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001005}
1006
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +01001007static inline pte_t __pte(pteval_t val)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001008{
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +01001009 pteval_t ret;
1010
1011 if (sizeof(pteval_t) > sizeof(long))
1012 ret = PVOP_CALL2(pteval_t,
1013 pv_mmu_ops.make_pte,
1014 val, (u64)val >> 32);
1015 else
1016 ret = PVOP_CALL1(pteval_t,
1017 pv_mmu_ops.make_pte,
1018 val);
1019
Jeremy Fitzhardingec8e53932008-01-30 13:32:57 +01001020 return (pte_t) { .pte = ret };
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001021}
1022
Jeremy Fitzhardinge773221f2008-01-30 13:33:15 +01001023static inline pteval_t pte_val(pte_t pte)
1024{
1025 pteval_t ret;
1026
1027 if (sizeof(pteval_t) > sizeof(long))
1028 ret = PVOP_CALL2(pteval_t, pv_mmu_ops.pte_val,
1029 pte.pte, (u64)pte.pte >> 32);
1030 else
1031 ret = PVOP_CALL1(pteval_t, pv_mmu_ops.pte_val,
1032 pte.pte);
1033
1034 return ret;
1035}
1036
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +01001037static inline pteval_t pte_flags(pte_t pte)
1038{
1039 pteval_t ret;
1040
1041 if (sizeof(pteval_t) > sizeof(long))
1042 ret = PVOP_CALL2(pteval_t, pv_mmu_ops.pte_flags,
1043 pte.pte, (u64)pte.pte >> 32);
1044 else
1045 ret = PVOP_CALL1(pteval_t, pv_mmu_ops.pte_flags,
1046 pte.pte);
1047
1048 return ret;
1049}
1050
Jeremy Fitzhardingeef385032008-01-30 13:33:15 +01001051static inline pgd_t __pgd(pgdval_t val)
1052{
1053 pgdval_t ret;
1054
1055 if (sizeof(pgdval_t) > sizeof(long))
1056 ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.make_pgd,
1057 val, (u64)val >> 32);
1058 else
1059 ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.make_pgd,
1060 val);
1061
1062 return (pgd_t) { ret };
1063}
1064
1065static inline pgdval_t pgd_val(pgd_t pgd)
1066{
1067 pgdval_t ret;
1068
1069 if (sizeof(pgdval_t) > sizeof(long))
1070 ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.pgd_val,
1071 pgd.pgd, (u64)pgd.pgd >> 32);
1072 else
1073 ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.pgd_val,
1074 pgd.pgd);
1075
1076 return ret;
1077}
1078
Jeremy Fitzhardinge08b882c2008-06-16 04:30:01 -07001079#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1080static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
1081 pte_t *ptep)
1082{
1083 pteval_t ret;
1084
1085 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
1086 mm, addr, ptep);
1087
1088 return (pte_t) { .pte = ret };
1089}
1090
1091static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
1092 pte_t *ptep, pte_t pte)
1093{
1094 if (sizeof(pteval_t) > sizeof(long))
1095 /* 5 arg words */
1096 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
1097 else
1098 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
1099 mm, addr, ptep, pte.pte);
1100}
1101
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +01001102static inline void set_pte(pte_t *ptep, pte_t pte)
1103{
1104 if (sizeof(pteval_t) > sizeof(long))
1105 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
1106 pte.pte, (u64)pte.pte >> 32);
1107 else
1108 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
1109 pte.pte);
1110}
1111
1112static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
1113 pte_t *ptep, pte_t pte)
1114{
1115 if (sizeof(pteval_t) > sizeof(long))
1116 /* 5 arg words */
1117 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
1118 else
1119 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
1120}
1121
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +01001122static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
1123{
1124 pmdval_t val = native_pmd_val(pmd);
1125
1126 if (sizeof(pmdval_t) > sizeof(long))
1127 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
1128 else
1129 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
1130}
1131
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +01001132#if PAGETABLE_LEVELS >= 3
1133static inline pmd_t __pmd(pmdval_t val)
1134{
1135 pmdval_t ret;
1136
1137 if (sizeof(pmdval_t) > sizeof(long))
1138 ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.make_pmd,
1139 val, (u64)val >> 32);
1140 else
1141 ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.make_pmd,
1142 val);
1143
1144 return (pmd_t) { ret };
1145}
1146
1147static inline pmdval_t pmd_val(pmd_t pmd)
1148{
1149 pmdval_t ret;
1150
1151 if (sizeof(pmdval_t) > sizeof(long))
1152 ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.pmd_val,
1153 pmd.pmd, (u64)pmd.pmd >> 32);
1154 else
1155 ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.pmd_val,
1156 pmd.pmd);
1157
1158 return ret;
1159}
1160
1161static inline void set_pud(pud_t *pudp, pud_t pud)
1162{
1163 pudval_t val = native_pud_val(pud);
1164
1165 if (sizeof(pudval_t) > sizeof(long))
1166 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
1167 val, (u64)val >> 32);
1168 else
1169 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
1170 val);
1171}
Eduardo Habkost90422192008-01-30 13:33:20 +01001172#if PAGETABLE_LEVELS == 4
1173static inline pud_t __pud(pudval_t val)
1174{
1175 pudval_t ret;
1176
1177 if (sizeof(pudval_t) > sizeof(long))
1178 ret = PVOP_CALL2(pudval_t, pv_mmu_ops.make_pud,
1179 val, (u64)val >> 32);
1180 else
1181 ret = PVOP_CALL1(pudval_t, pv_mmu_ops.make_pud,
1182 val);
1183
1184 return (pud_t) { ret };
1185}
1186
1187static inline pudval_t pud_val(pud_t pud)
1188{
1189 pudval_t ret;
1190
1191 if (sizeof(pudval_t) > sizeof(long))
1192 ret = PVOP_CALL2(pudval_t, pv_mmu_ops.pud_val,
1193 pud.pud, (u64)pud.pud >> 32);
1194 else
1195 ret = PVOP_CALL1(pudval_t, pv_mmu_ops.pud_val,
1196 pud.pud);
1197
1198 return ret;
1199}
1200
1201static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
1202{
1203 pgdval_t val = native_pgd_val(pgd);
1204
1205 if (sizeof(pgdval_t) > sizeof(long))
1206 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
1207 val, (u64)val >> 32);
1208 else
1209 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
1210 val);
1211}
1212
1213static inline void pgd_clear(pgd_t *pgdp)
1214{
1215 set_pgd(pgdp, __pgd(0));
1216}
1217
1218static inline void pud_clear(pud_t *pudp)
1219{
1220 set_pud(pudp, __pud(0));
1221}
1222
1223#endif /* PAGETABLE_LEVELS == 4 */
1224
Glauber de Oliveira Costa1fe91512008-01-30 13:33:19 +01001225#endif /* PAGETABLE_LEVELS >= 3 */
1226
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +01001227#ifdef CONFIG_X86_PAE
1228/* Special-case pte-setting operations for PAE, which can't update a
1229 64-bit pte atomically */
1230static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
1231{
1232 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
1233 pte.pte, pte.pte >> 32);
1234}
1235
1236static inline void set_pte_present(struct mm_struct *mm, unsigned long addr,
1237 pte_t *ptep, pte_t pte)
1238{
1239 /* 5 arg words */
1240 pv_mmu_ops.set_pte_present(mm, addr, ptep, pte);
1241}
1242
1243static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
1244 pte_t *ptep)
1245{
1246 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
1247}
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +01001248
1249static inline void pmd_clear(pmd_t *pmdp)
1250{
1251 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
1252}
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +01001253#else /* !CONFIG_X86_PAE */
1254static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
1255{
1256 set_pte(ptep, pte);
1257}
1258
1259static inline void set_pte_present(struct mm_struct *mm, unsigned long addr,
1260 pte_t *ptep, pte_t pte)
1261{
1262 set_pte(ptep, pte);
1263}
1264
1265static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
1266 pte_t *ptep)
1267{
1268 set_pte_at(mm, addr, ptep, __pte(0));
1269}
Jeremy Fitzhardinge60b3f622008-01-30 13:33:15 +01001270
1271static inline void pmd_clear(pmd_t *pmdp)
1272{
1273 set_pmd(pmdp, __pmd(0));
1274}
Jeremy Fitzhardinge4eed80c2008-01-30 13:33:15 +01001275#endif /* CONFIG_X86_PAE */
1276
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001277/* Lazy mode for batching updates / context switch */
1278enum paravirt_lazy_mode {
1279 PARAVIRT_LAZY_NONE,
1280 PARAVIRT_LAZY_MMU,
1281 PARAVIRT_LAZY_CPU,
1282};
1283
1284enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
1285void paravirt_enter_lazy_cpu(void);
1286void paravirt_leave_lazy_cpu(void);
1287void paravirt_enter_lazy_mmu(void);
1288void paravirt_leave_lazy_mmu(void);
1289void paravirt_leave_lazy(enum paravirt_lazy_mode mode);
1290
Zachary Amsden9226d122007-02-13 13:26:21 +01001291#define __HAVE_ARCH_ENTER_LAZY_CPU_MODE
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001292static inline void arch_enter_lazy_cpu_mode(void)
1293{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001294 PVOP_VCALL0(pv_cpu_ops.lazy_mode.enter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001295}
1296
1297static inline void arch_leave_lazy_cpu_mode(void)
1298{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001299 PVOP_VCALL0(pv_cpu_ops.lazy_mode.leave);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001300}
1301
1302static inline void arch_flush_lazy_cpu_mode(void)
1303{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001304 if (unlikely(paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU)) {
1305 arch_leave_lazy_cpu_mode();
1306 arch_enter_lazy_cpu_mode();
1307 }
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001308}
1309
Zachary Amsden9226d122007-02-13 13:26:21 +01001310
1311#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001312static inline void arch_enter_lazy_mmu_mode(void)
1313{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001314 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001315}
1316
1317static inline void arch_leave_lazy_mmu_mode(void)
1318{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001319 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001320}
1321
1322static inline void arch_flush_lazy_mmu_mode(void)
1323{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001324 if (unlikely(paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU)) {
1325 arch_leave_lazy_mmu_mode();
1326 arch_enter_lazy_mmu_mode();
1327 }
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001328}
Zachary Amsden9226d122007-02-13 13:26:21 +01001329
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -07001330static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
1331 unsigned long phys, pgprot_t flags)
1332{
1333 pv_mmu_ops.set_fixmap(idx, phys, flags);
1334}
1335
Jeremy Fitzhardinge45876232007-05-02 19:27:13 +02001336void _paravirt_nop(void);
1337#define paravirt_nop ((void *)_paravirt_nop)
1338
Rusty Russell139ec7c2006-12-07 02:14:08 +01001339/* These all sit in the .parainstructions section to tell us what to patch. */
Jeremy Fitzhardinge98de0322007-05-02 19:27:14 +02001340struct paravirt_patch_site {
Rusty Russell139ec7c2006-12-07 02:14:08 +01001341 u8 *instr; /* original instructions */
1342 u8 instrtype; /* type of this instruction */
1343 u8 len; /* length of original instruction */
1344 u16 clobbers; /* what registers you may clobber */
1345};
1346
Jeremy Fitzhardinge98de0322007-05-02 19:27:14 +02001347extern struct paravirt_patch_site __parainstructions[],
1348 __parainstructions_end[];
1349
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001350#ifdef CONFIG_X86_32
1351#define PV_SAVE_REGS "pushl %%ecx; pushl %%edx;"
1352#define PV_RESTORE_REGS "popl %%edx; popl %%ecx"
1353#define PV_FLAGS_ARG "0"
1354#define PV_EXTRA_CLOBBERS
1355#define PV_VEXTRA_CLOBBERS
1356#else
1357/* We save some registers, but all of them, that's too much. We clobber all
1358 * caller saved registers but the argument parameter */
1359#define PV_SAVE_REGS "pushq %%rdi;"
1360#define PV_RESTORE_REGS "popq %%rdi;"
1361#define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx"
1362#define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx"
1363#define PV_FLAGS_ARG "D"
1364#endif
1365
Rusty Russell139ec7c2006-12-07 02:14:08 +01001366static inline unsigned long __raw_local_save_flags(void)
1367{
1368 unsigned long f;
1369
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001370 asm volatile(paravirt_alt(PV_SAVE_REGS
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001371 PARAVIRT_CALL
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001372 PV_RESTORE_REGS)
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001373 : "=a"(f)
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001374 : paravirt_type(pv_irq_ops.save_fl),
Jeremy Fitzhardinge42c24fa2007-05-02 19:27:14 +02001375 paravirt_clobber(CLBR_EAX)
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001376 : "memory", "cc" PV_VEXTRA_CLOBBERS);
Rusty Russell139ec7c2006-12-07 02:14:08 +01001377 return f;
1378}
1379
1380static inline void raw_local_irq_restore(unsigned long f)
1381{
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001382 asm volatile(paravirt_alt(PV_SAVE_REGS
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001383 PARAVIRT_CALL
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001384 PV_RESTORE_REGS)
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001385 : "=a"(f)
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001386 : PV_FLAGS_ARG(f),
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001387 paravirt_type(pv_irq_ops.restore_fl),
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001388 paravirt_clobber(CLBR_EAX)
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001389 : "memory", "cc" PV_EXTRA_CLOBBERS);
Rusty Russell139ec7c2006-12-07 02:14:08 +01001390}
1391
1392static inline void raw_local_irq_disable(void)
1393{
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001394 asm volatile(paravirt_alt(PV_SAVE_REGS
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001395 PARAVIRT_CALL
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001396 PV_RESTORE_REGS)
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001397 :
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001398 : paravirt_type(pv_irq_ops.irq_disable),
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001399 paravirt_clobber(CLBR_EAX)
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001400 : "memory", "eax", "cc" PV_EXTRA_CLOBBERS);
Rusty Russell139ec7c2006-12-07 02:14:08 +01001401}
1402
1403static inline void raw_local_irq_enable(void)
1404{
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001405 asm volatile(paravirt_alt(PV_SAVE_REGS
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001406 PARAVIRT_CALL
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001407 PV_RESTORE_REGS)
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001408 :
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001409 : paravirt_type(pv_irq_ops.irq_enable),
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001410 paravirt_clobber(CLBR_EAX)
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001411 : "memory", "eax", "cc" PV_EXTRA_CLOBBERS);
Rusty Russell139ec7c2006-12-07 02:14:08 +01001412}
1413
1414static inline unsigned long __raw_local_irq_save(void)
1415{
1416 unsigned long f;
1417
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001418 f = __raw_local_save_flags();
1419 raw_local_irq_disable();
Rusty Russell139ec7c2006-12-07 02:14:08 +01001420 return f;
1421}
1422
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +02001423/* Make sure as little as possible of this mess escapes. */
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001424#undef PARAVIRT_CALL
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +02001425#undef __PVOP_CALL
1426#undef __PVOP_VCALL
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001427#undef PVOP_VCALL0
1428#undef PVOP_CALL0
1429#undef PVOP_VCALL1
1430#undef PVOP_CALL1
1431#undef PVOP_VCALL2
1432#undef PVOP_CALL2
1433#undef PVOP_VCALL3
1434#undef PVOP_CALL3
1435#undef PVOP_VCALL4
1436#undef PVOP_CALL4
Rusty Russell139ec7c2006-12-07 02:14:08 +01001437
Rusty Russelld3561b72006-12-07 02:14:07 +01001438#else /* __ASSEMBLY__ */
1439
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001440#define _PVSITE(ptype, clobbers, ops, word, algn) \
Rusty Russell139ec7c2006-12-07 02:14:08 +01001441771:; \
1442 ops; \
1443772:; \
1444 .pushsection .parainstructions,"a"; \
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001445 .align algn; \
1446 word 771b; \
Rusty Russell139ec7c2006-12-07 02:14:08 +01001447 .byte ptype; \
1448 .byte 772b-771b; \
1449 .short clobbers; \
1450 .popsection
1451
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001452
1453#ifdef CONFIG_X86_64
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001454#define PV_SAVE_REGS pushq %rax; pushq %rdi; pushq %rcx; pushq %rdx
1455#define PV_RESTORE_REGS popq %rdx; popq %rcx; popq %rdi; popq %rax
1456#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001457#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001458#define PARA_INDIRECT(addr) *addr(%rip)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001459#else
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001460#define PV_SAVE_REGS pushl %eax; pushl %edi; pushl %ecx; pushl %edx
1461#define PV_RESTORE_REGS popl %edx; popl %ecx; popl %edi; popl %eax
1462#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001463#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001464#define PARA_INDIRECT(addr) *%cs:addr
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001465#endif
1466
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001467#define INTERRUPT_RETURN \
1468 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001469 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
Rusty Russell139ec7c2006-12-07 02:14:08 +01001470
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001471#define DISABLE_INTERRUPTS(clobbers) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001472 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001473 PV_SAVE_REGS; \
1474 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001475 PV_RESTORE_REGS;) \
Rusty Russell139ec7c2006-12-07 02:14:08 +01001476
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001477#define ENABLE_INTERRUPTS(clobbers) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001478 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001479 PV_SAVE_REGS; \
1480 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001481 PV_RESTORE_REGS;)
Rusty Russell139ec7c2006-12-07 02:14:08 +01001482
Glauber de Oliveira Costa6abcd982008-01-30 13:30:33 +01001483#define ENABLE_INTERRUPTS_SYSCALL_RET \
1484 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_syscall_ret),\
1485 CLBR_NONE, \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001486 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_syscall_ret))
Rusty Russell139ec7c2006-12-07 02:14:08 +01001487
Glauber de Oliveira Costa2e47d3e2008-01-30 13:32:07 +01001488
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001489#ifdef CONFIG_X86_32
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001490#define GET_CR0_INTO_EAX \
1491 push %ecx; push %edx; \
1492 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
Jeremy Fitzhardinge42c24fa2007-05-02 19:27:14 +02001493 pop %edx; pop %ecx
Glauber de Oliveira Costa4a8c4c42008-01-30 13:32:07 +01001494#else
Glauber de Oliveira Costae801f862008-01-30 13:32:08 +01001495#define SWAPGS \
1496 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
1497 PV_SAVE_REGS; \
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001498 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs); \
Glauber de Oliveira Costae801f862008-01-30 13:32:08 +01001499 PV_RESTORE_REGS \
1500 )
1501
Jeremy Fitzhardinge491eccb2008-06-25 00:19:15 -04001502#define GET_CR2_INTO_RCX \
1503 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
1504 movq %rax, %rcx; \
Glauber de Oliveira Costa4a8c4c42008-01-30 13:32:07 +01001505 xorq %rax, %rax;
1506
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001507#endif
Rusty Russell139ec7c2006-12-07 02:14:08 +01001508
Rusty Russelld3561b72006-12-07 02:14:07 +01001509#endif /* __ASSEMBLY__ */
1510#endif /* CONFIG_PARAVIRT */
1511#endif /* __ASM_PARAVIRT_H */