blob: 854c19364da35def30f5c629fe8eb67a17b9afff [file] [log] [blame]
Rusty Russelld3561b72006-12-07 02:14:07 +01001#ifndef __ASM_PARAVIRT_H
2#define __ASM_PARAVIRT_H
3/* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +02005
6#ifdef CONFIG_PARAVIRT
Rusty Russellda181a82006-12-07 02:14:08 +01007#include <asm/page.h>
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01008#include <asm/asm.h>
Rusty Russelld3561b72006-12-07 02:14:07 +01009
Rusty Russell139ec7c2006-12-07 02:14:08 +010010/* Bitmask of what can be clobbered: usually at least eax. */
11#define CLBR_NONE 0x0
12#define CLBR_EAX 0x1
13#define CLBR_ECX 0x2
14#define CLBR_EDX 0x4
15#define CLBR_ANY 0x7
16
Rusty Russelld3561b72006-12-07 02:14:07 +010017#ifndef __ASSEMBLY__
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +020018#include <linux/types.h>
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +020019#include <linux/cpumask.h>
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +020020#include <asm/kmap_types.h>
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +010021#include <asm/desc_defs.h>
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +020022
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +020023struct page;
Rusty Russelld3561b72006-12-07 02:14:07 +010024struct thread_struct;
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +010025struct desc_ptr;
Rusty Russelld3561b72006-12-07 02:14:07 +010026struct tss_struct;
Rusty Russellda181a82006-12-07 02:14:08 +010027struct mm_struct;
Rusty Russell90a0a062007-05-02 19:27:10 +020028struct desc_struct;
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020029
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070030/* general info */
31struct pv_info {
Rusty Russelld3561b72006-12-07 02:14:07 +010032 unsigned int kernel_rpl;
Jeremy Fitzhardinge5311ab62007-05-02 19:27:13 +020033 int shared_kernel_pmd;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070034 int paravirt_enabled;
Rusty Russelld3561b72006-12-07 02:14:07 +010035 const char *name;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070036};
Rusty Russelld3561b72006-12-07 02:14:07 +010037
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070038struct pv_init_ops {
Rusty Russell139ec7c2006-12-07 02:14:08 +010039 /*
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070040 * Patch may replace one of the defined code sequences with
41 * arbitrary code, subject to the same register constraints.
42 * This generally means the code is not free to clobber any
43 * registers other than EAX. The patch function should return
44 * the number of bytes of code generated, as we nop pad the
45 * rest in generic code.
Rusty Russell139ec7c2006-12-07 02:14:08 +010046 */
Andi Kleenab144f52007-08-10 22:31:03 +020047 unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
48 unsigned long addr, unsigned len);
Rusty Russell139ec7c2006-12-07 02:14:08 +010049
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020050 /* Basic arch-specific setup */
Rusty Russelld3561b72006-12-07 02:14:07 +010051 void (*arch_setup)(void);
52 char *(*memory_setup)(void);
Jeremy Fitzhardinge6996d3b2007-07-17 18:37:03 -070053 void (*post_allocator_init)(void);
54
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020055 /* Print a banner to identify the environment */
Rusty Russelld3561b72006-12-07 02:14:07 +010056 void (*banner)(void);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070057};
58
59
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -070060struct pv_lazy_ops {
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070061 /* Set deferred update mode, used for batching operations. */
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -070062 void (*enter)(void);
63 void (*leave)(void);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070064};
65
66struct pv_time_ops {
67 void (*time_init)(void);
Rusty Russelld3561b72006-12-07 02:14:07 +010068
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020069 /* Set and set time of day */
Rusty Russelld3561b72006-12-07 02:14:07 +010070 unsigned long (*get_wallclock)(void);
71 int (*set_wallclock)(unsigned long);
Rusty Russelld3561b72006-12-07 02:14:07 +010072
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070073 unsigned long long (*sched_clock)(void);
74 unsigned long (*get_cpu_khz)(void);
75};
Rusty Russelld3561b72006-12-07 02:14:07 +010076
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -070077struct pv_cpu_ops {
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020078 /* hooks for various privileged instructions */
Andi Kleen1a1eecd2007-02-13 13:26:25 +010079 unsigned long (*get_debugreg)(int regno);
80 void (*set_debugreg)(int regno, unsigned long value);
Rusty Russelld3561b72006-12-07 02:14:07 +010081
Andi Kleen1a1eecd2007-02-13 13:26:25 +010082 void (*clts)(void);
Rusty Russelld3561b72006-12-07 02:14:07 +010083
Andi Kleen1a1eecd2007-02-13 13:26:25 +010084 unsigned long (*read_cr0)(void);
85 void (*write_cr0)(unsigned long);
Rusty Russelld3561b72006-12-07 02:14:07 +010086
Andi Kleen1a1eecd2007-02-13 13:26:25 +010087 unsigned long (*read_cr4_safe)(void);
88 unsigned long (*read_cr4)(void);
89 void (*write_cr4)(unsigned long);
Rusty Russelld3561b72006-12-07 02:14:07 +010090
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +020091 /* Segment descriptor handling */
Andi Kleen1a1eecd2007-02-13 13:26:25 +010092 void (*load_tr_desc)(void);
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +010093 void (*load_gdt)(const struct desc_ptr *);
94 void (*load_idt)(const struct desc_ptr *);
95 void (*store_gdt)(struct desc_ptr *);
96 void (*store_idt)(struct desc_ptr *);
Andi Kleen1a1eecd2007-02-13 13:26:25 +010097 void (*set_ldt)(const void *desc, unsigned entries);
98 unsigned long (*store_tr)(void);
99 void (*load_tls)(struct thread_struct *t, unsigned int cpu);
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100100 void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
101 const void *desc);
Rusty Russell90a0a062007-05-02 19:27:10 +0200102 void (*write_gdt_entry)(struct desc_struct *,
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100103 int entrynum, const void *desc, int size);
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100104 void (*write_idt_entry)(gate_desc *,
105 int entrynum, const gate_desc *gate);
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100106 void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t);
Rusty Russelld3561b72006-12-07 02:14:07 +0100107
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100108 void (*set_iopl_mask)(unsigned mask);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700109
110 void (*wbinvd)(void);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100111 void (*io_delay)(void);
Rusty Russelld3561b72006-12-07 02:14:07 +0100112
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700113 /* cpuid emulation, mostly so that caps bits can be disabled */
114 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
115 unsigned int *ecx, unsigned int *edx);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200116
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700117 /* MSR, PMC and TSR operations.
118 err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */
119 u64 (*read_msr)(unsigned int msr, int *err);
Glauber de Oliveira Costac9dcda52008-01-30 13:31:07 +0100120 int (*write_msr)(unsigned int msr, unsigned low, unsigned high);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700121
122 u64 (*read_tsc)(void);
Glauber de Oliveira Costab8d1fae2008-01-30 13:31:07 +0100123 u64 (*read_pmc)(int counter);
Glauber de Oliveira Costae5aaac42008-01-30 13:32:05 +0100124 unsigned long long (*read_tscp)(unsigned int *aux);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700125
126 /* These two are jmp to, not actually called. */
Glauber de Oliveira Costa6abcd982008-01-30 13:30:33 +0100127 void (*irq_enable_syscall_ret)(void);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700128 void (*iret)(void);
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700129
130 struct pv_lazy_ops lazy_mode;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700131};
132
133struct pv_irq_ops {
134 void (*init_IRQ)(void);
135
136 /*
137 * Get/set interrupt state. save_fl and restore_fl are only
138 * expected to use X86_EFLAGS_IF; all other bits
139 * returned from save_fl are undefined, and may be ignored by
140 * restore_fl.
141 */
142 unsigned long (*save_fl)(void);
143 void (*restore_fl)(unsigned long);
144 void (*irq_disable)(void);
145 void (*irq_enable)(void);
146 void (*safe_halt)(void);
147 void (*halt)(void);
148};
149
150struct pv_apic_ops {
Rusty Russell13623d72006-12-07 02:14:08 +0100151#ifdef CONFIG_X86_LOCAL_APIC
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200152 /*
153 * Direct APIC operations, principally for VMI. Ideally
154 * these shouldn't be in this interface.
155 */
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100156 void (*apic_write)(unsigned long reg, u32 v);
157 void (*apic_write_atomic)(unsigned long reg, u32 v);
158 u32 (*apic_read)(unsigned long reg);
Zachary Amsdenbbab4f32007-02-13 13:26:21 +0100159 void (*setup_boot_clock)(void);
160 void (*setup_secondary_clock)(void);
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200161
162 void (*startup_ipi_hook)(int phys_apicid,
163 unsigned long start_eip,
164 unsigned long start_esp);
Rusty Russell13623d72006-12-07 02:14:08 +0100165#endif
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700166};
167
168struct pv_mmu_ops {
169 /*
170 * Called before/after init_mm pagetable setup. setup_start
171 * may reset %cr3, and may pre-install parts of the pagetable;
172 * pagetable setup is expected to preserve any existing
173 * mapping.
174 */
175 void (*pagetable_setup_start)(pgd_t *pgd_base);
176 void (*pagetable_setup_done)(pgd_t *pgd_base);
177
178 unsigned long (*read_cr2)(void);
179 void (*write_cr2)(unsigned long);
180
181 unsigned long (*read_cr3)(void);
182 void (*write_cr3)(unsigned long);
183
184 /*
185 * Hooks for intercepting the creation/use/destruction of an
186 * mm_struct.
187 */
188 void (*activate_mm)(struct mm_struct *prev,
189 struct mm_struct *next);
190 void (*dup_mmap)(struct mm_struct *oldmm,
191 struct mm_struct *mm);
192 void (*exit_mmap)(struct mm_struct *mm);
193
Rusty Russell13623d72006-12-07 02:14:08 +0100194
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200195 /* TLB operations */
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100196 void (*flush_tlb_user)(void);
197 void (*flush_tlb_kernel)(void);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200198 void (*flush_tlb_single)(unsigned long addr);
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200199 void (*flush_tlb_others)(const cpumask_t *cpus, struct mm_struct *mm,
200 unsigned long va);
Rusty Russellda181a82006-12-07 02:14:08 +0100201
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200202 /* Hooks for allocating/releasing pagetable pages */
Jeremy Fitzhardingefdb4c332007-07-17 18:37:03 -0700203 void (*alloc_pt)(struct mm_struct *mm, u32 pfn);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100204 void (*alloc_pd)(u32 pfn);
205 void (*alloc_pd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count);
206 void (*release_pt)(u32 pfn);
207 void (*release_pd)(u32 pfn);
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100208
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200209 /* Pagetable manipulation functions */
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100210 void (*set_pte)(pte_t *ptep, pte_t pteval);
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200211 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
212 pte_t *ptep, pte_t pteval);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100213 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200214 void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200215 void (*pte_update_defer)(struct mm_struct *mm,
216 unsigned long addr, pte_t *ptep);
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200217
Rusty Russellda181a82006-12-07 02:14:08 +0100218#ifdef CONFIG_X86_PAE
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100219 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700220 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr,
221 pte_t *ptep, pte_t pte);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100222 void (*set_pud)(pud_t *pudp, pud_t pudval);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700223 void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
Andi Kleen1a1eecd2007-02-13 13:26:25 +0100224 void (*pmd_clear)(pmd_t *pmdp);
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200225
226 unsigned long long (*pte_val)(pte_t);
227 unsigned long long (*pmd_val)(pmd_t);
228 unsigned long long (*pgd_val)(pgd_t);
229
230 pte_t (*make_pte)(unsigned long long pte);
231 pmd_t (*make_pmd)(unsigned long long pmd);
232 pgd_t (*make_pgd)(unsigned long long pgd);
233#else
234 unsigned long (*pte_val)(pte_t);
235 unsigned long (*pgd_val)(pgd_t);
236
237 pte_t (*make_pte)(unsigned long pte);
238 pgd_t (*make_pgd)(unsigned long pgd);
Rusty Russellda181a82006-12-07 02:14:08 +0100239#endif
240
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700241#ifdef CONFIG_HIGHPTE
242 void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
243#endif
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700244
245 struct pv_lazy_ops lazy_mode;
Rusty Russelld3561b72006-12-07 02:14:07 +0100246};
247
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700248/* This contains all the paravirt structures: we get a convenient
249 * number for each function using the offset which we use to indicate
250 * what to patch. */
251struct paravirt_patch_template
252{
253 struct pv_init_ops pv_init_ops;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700254 struct pv_time_ops pv_time_ops;
255 struct pv_cpu_ops pv_cpu_ops;
256 struct pv_irq_ops pv_irq_ops;
257 struct pv_apic_ops pv_apic_ops;
258 struct pv_mmu_ops pv_mmu_ops;
259};
260
261extern struct pv_info pv_info;
262extern struct pv_init_ops pv_init_ops;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700263extern struct pv_time_ops pv_time_ops;
264extern struct pv_cpu_ops pv_cpu_ops;
265extern struct pv_irq_ops pv_irq_ops;
266extern struct pv_apic_ops pv_apic_ops;
267extern struct pv_mmu_ops pv_mmu_ops;
Rusty Russelld3561b72006-12-07 02:14:07 +0100268
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200269#define PARAVIRT_PATCH(x) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700270 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200271
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700272#define paravirt_type(op) \
273 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
274 [paravirt_opptr] "m" (op)
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200275#define paravirt_clobber(clobber) \
276 [paravirt_clobber] "i" (clobber)
277
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200278/*
279 * Generate some code, and mark it as patchable by the
280 * apply_paravirt() alternate instruction patcher.
281 */
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200282#define _paravirt_alt(insn_string, type, clobber) \
283 "771:\n\t" insn_string "\n" "772:\n" \
284 ".pushsection .parainstructions,\"a\"\n" \
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +0100285 _ASM_ALIGN "\n" \
286 _ASM_PTR " 771b\n" \
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200287 " .byte " type "\n" \
288 " .byte 772b-771b\n" \
289 " .short " clobber "\n" \
290 ".popsection\n"
291
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200292/* Generate patchable code, with the default asm parameters. */
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200293#define paravirt_alt(insn_string) \
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +0200294 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
295
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200296unsigned paravirt_patch_nop(void);
297unsigned paravirt_patch_ignore(unsigned len);
Andi Kleenab144f52007-08-10 22:31:03 +0200298unsigned paravirt_patch_call(void *insnbuf,
299 const void *target, u16 tgt_clobbers,
300 unsigned long addr, u16 site_clobbers,
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200301 unsigned len);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700302unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
Andi Kleenab144f52007-08-10 22:31:03 +0200303 unsigned long addr, unsigned len);
304unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
305 unsigned long addr, unsigned len);
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200306
Andi Kleenab144f52007-08-10 22:31:03 +0200307unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200308 const char *start, const char *end);
309
Jeremy Fitzhardinged5729292007-07-17 18:37:04 -0700310int paravirt_disable_iospace(void);
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200311
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200312/*
313 * This generates an indirect call based on the operation type number.
314 * The type number, computed in PARAVIRT_PATCH, is derived from the
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700315 * offset into the paravirt_patch_template structure, and can therefore be
316 * freely converted back into a structure offset.
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200317 */
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700318#define PARAVIRT_CALL "call *%[paravirt_opptr];"
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200319
320/*
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700321 * These macros are intended to wrap calls through one of the paravirt
322 * ops structs, so that they can be later identified and patched at
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200323 * runtime.
324 *
325 * Normally, a call to a pv_op function is a simple indirect call:
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100326 * (pv_op_struct.operations)(args...).
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200327 *
328 * Unfortunately, this is a relatively slow operation for modern CPUs,
329 * because it cannot necessarily determine what the destination
330 * address is. In this case, the address is a runtime constant, so at
331 * the very least we can patch the call to e a simple direct call, or
332 * ideally, patch an inline implementation into the callsite. (Direct
333 * calls are essentially free, because the call and return addresses
334 * are completely predictable.)
335 *
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100336 * For i386, these macros rely on the standard gcc "regparm(3)" calling
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200337 * convention, in which the first three arguments are placed in %eax,
338 * %edx, %ecx (in that order), and the remaining arguments are placed
339 * on the stack. All caller-save registers (eax,edx,ecx) are expected
340 * to be modified (either clobbered or used for return values).
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100341 * X86_64, on the other hand, already specifies a register-based calling
342 * conventions, returning at %rax, with parameteres going on %rdi, %rsi,
343 * %rdx, and %rcx. Note that for this reason, x86_64 does not need any
344 * special handling for dealing with 4 arguments, unlike i386.
345 * However, x86_64 also have to clobber all caller saved registers, which
346 * unfortunately, are quite a bit (r8 - r11)
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200347 *
348 * The call instruction itself is marked by placing its start address
349 * and size into the .parainstructions section, so that
350 * apply_paravirt() in arch/i386/kernel/alternative.c can do the
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700351 * appropriate patching under the control of the backend pv_init_ops
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200352 * implementation.
353 *
354 * Unfortunately there's no way to get gcc to generate the args setup
355 * for the call, and then allow the call itself to be generated by an
356 * inline asm. Because of this, we must do the complete arg setup and
357 * return value handling from within these macros. This is fairly
358 * cumbersome.
359 *
360 * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments.
361 * It could be extended to more arguments, but there would be little
362 * to be gained from that. For each number of arguments, there are
363 * the two VCALL and CALL variants for void and non-void functions.
364 *
365 * When there is a return value, the invoker of the macro must specify
366 * the return type. The macro then uses sizeof() on that type to
367 * determine whether its a 32 or 64 bit value, and places the return
368 * in the right register(s) (just %eax for 32-bit, and %edx:%eax for
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100369 * 64-bit). For x86_64 machines, it just returns at %rax regardless of
370 * the return value size.
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200371 *
372 * 64-bit arguments are passed as a pair of adjacent 32-bit arguments
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100373 * i386 also passes 64-bit arguments as a pair of adjacent 32-bit arguments
374 * in low,high order
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +0200375 *
376 * Small structures are passed and returned in registers. The macro
377 * calling convention can't directly deal with this, so the wrapper
378 * functions must do this.
379 *
380 * These PVOP_* macros are only defined within this header. This
381 * means that all uses must be wrapped in inline functions. This also
382 * makes sure the incoming and outgoing types are always correct.
383 */
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100384#ifdef CONFIG_X86_32
385#define PVOP_VCALL_ARGS unsigned long __eax, __edx, __ecx
386#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
387#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
388 "=c" (__ecx)
389#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
390#define EXTRA_CLOBBERS
391#define VEXTRA_CLOBBERS
392#else
393#define PVOP_VCALL_ARGS unsigned long __edi, __esi, __edx, __ecx
394#define PVOP_CALL_ARGS PVOP_VCALL_ARGS, __eax
395#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
396 "=S" (__esi), "=d" (__edx), \
397 "=c" (__ecx)
398
399#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
400
401#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
402#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
403#endif
404
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200405#define __PVOP_CALL(rettype, op, pre, post, ...) \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200406 ({ \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200407 rettype __ret; \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100408 PVOP_CALL_ARGS; \
409 /* This is 32-bit specific, but is okay in 64-bit */ \
410 /* since this condition will never hold */ \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200411 if (sizeof(rettype) > sizeof(unsigned long)) { \
412 asm volatile(pre \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200413 paravirt_alt(PARAVIRT_CALL) \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200414 post \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100415 : PVOP_CALL_CLOBBERS \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200416 : paravirt_type(op), \
417 paravirt_clobber(CLBR_ANY), \
418 ##__VA_ARGS__ \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100419 : "memory", "cc" EXTRA_CLOBBERS); \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200420 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
421 } else { \
422 asm volatile(pre \
423 paravirt_alt(PARAVIRT_CALL) \
424 post \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100425 : PVOP_CALL_CLOBBERS \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200426 : paravirt_type(op), \
427 paravirt_clobber(CLBR_ANY), \
428 ##__VA_ARGS__ \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100429 : "memory", "cc" EXTRA_CLOBBERS); \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200430 __ret = (rettype)__eax; \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200431 } \
432 __ret; \
433 })
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200434#define __PVOP_VCALL(op, pre, post, ...) \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200435 ({ \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100436 PVOP_VCALL_ARGS; \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200437 asm volatile(pre \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200438 paravirt_alt(PARAVIRT_CALL) \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200439 post \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100440 : PVOP_VCALL_CLOBBERS \
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200441 : paravirt_type(op), \
442 paravirt_clobber(CLBR_ANY), \
443 ##__VA_ARGS__ \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100444 : "memory", "cc" VEXTRA_CLOBBERS); \
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200445 })
446
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200447#define PVOP_CALL0(rettype, op) \
448 __PVOP_CALL(rettype, op, "", "")
449#define PVOP_VCALL0(op) \
450 __PVOP_VCALL(op, "", "")
451
452#define PVOP_CALL1(rettype, op, arg1) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100453 __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200454#define PVOP_VCALL1(op, arg1) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100455 __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200456
457#define PVOP_CALL2(rettype, op, arg1, arg2) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100458 __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
459 "1" ((unsigned long)(arg2)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200460#define PVOP_VCALL2(op, arg1, arg2) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100461 __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
462 "1" ((unsigned long)(arg2)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200463
464#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100465 __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
466 "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200467#define PVOP_VCALL3(op, arg1, arg2, arg3) \
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100468 __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
469 "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200470
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100471/* This is the only difference in x86_64. We can make it much simpler */
472#ifdef CONFIG_X86_32
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200473#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
474 __PVOP_CALL(rettype, op, \
475 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
476 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
477 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
478#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
479 __PVOP_VCALL(op, \
480 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
481 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
482 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
Glauber de Oliveira Costaa4746362008-01-30 13:32:05 +0100483#else
484#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
485 __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
486 "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \
487 "3"((unsigned long)(arg4)))
488#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
489 __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
490 "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \
491 "3"((unsigned long)(arg4)))
492#endif
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +0200493
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200494static inline int paravirt_enabled(void)
495{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700496 return pv_info.paravirt_enabled;
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200497}
Rusty Russelld3561b72006-12-07 02:14:07 +0100498
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100499static inline void load_sp0(struct tss_struct *tss,
Rusty Russelld3561b72006-12-07 02:14:07 +0100500 struct thread_struct *thread)
501{
H. Peter Anvinfaca6222008-01-30 13:31:02 +0100502 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
Rusty Russelld3561b72006-12-07 02:14:07 +0100503}
504
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700505#define ARCH_SETUP pv_init_ops.arch_setup();
Rusty Russelld3561b72006-12-07 02:14:07 +0100506static inline unsigned long get_wallclock(void)
507{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700508 return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
Rusty Russelld3561b72006-12-07 02:14:07 +0100509}
510
511static inline int set_wallclock(unsigned long nowtime)
512{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700513 return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
Rusty Russelld3561b72006-12-07 02:14:07 +0100514}
515
Zachary Amsdene30fab32007-03-05 00:30:39 -0800516static inline void (*choose_time_init(void))(void)
Rusty Russelld3561b72006-12-07 02:14:07 +0100517{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700518 return pv_time_ops.time_init;
Rusty Russelld3561b72006-12-07 02:14:07 +0100519}
520
521/* The paravirtualized CPUID instruction. */
522static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
523 unsigned int *ecx, unsigned int *edx)
524{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700525 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
Rusty Russelld3561b72006-12-07 02:14:07 +0100526}
527
528/*
529 * These special macros can be used to get or set a debugging register
530 */
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200531static inline unsigned long paravirt_get_debugreg(int reg)
532{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700533 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200534}
535#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
536static inline void set_debugreg(unsigned long val, int reg)
537{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700538 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200539}
Rusty Russelld3561b72006-12-07 02:14:07 +0100540
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200541static inline void clts(void)
542{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700543 PVOP_VCALL0(pv_cpu_ops.clts);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200544}
Rusty Russelld3561b72006-12-07 02:14:07 +0100545
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200546static inline unsigned long read_cr0(void)
547{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700548 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200549}
Rusty Russelld3561b72006-12-07 02:14:07 +0100550
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200551static inline void write_cr0(unsigned long x)
552{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700553 PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200554}
Rusty Russelld3561b72006-12-07 02:14:07 +0100555
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200556static inline unsigned long read_cr2(void)
557{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700558 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200559}
Rusty Russelld3561b72006-12-07 02:14:07 +0100560
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200561static inline void write_cr2(unsigned long x)
562{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700563 PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200564}
Rusty Russelld3561b72006-12-07 02:14:07 +0100565
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200566static inline unsigned long read_cr3(void)
567{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700568 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200569}
570
571static inline void write_cr3(unsigned long x)
572{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700573 PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200574}
575
576static inline unsigned long read_cr4(void)
577{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700578 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200579}
580static inline unsigned long read_cr4_safe(void)
581{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700582 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200583}
584
585static inline void write_cr4(unsigned long x)
586{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700587 PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200588}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200589
Rusty Russelld3561b72006-12-07 02:14:07 +0100590static inline void raw_safe_halt(void)
591{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700592 PVOP_VCALL0(pv_irq_ops.safe_halt);
Rusty Russelld3561b72006-12-07 02:14:07 +0100593}
594
595static inline void halt(void)
596{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700597 PVOP_VCALL0(pv_irq_ops.safe_halt);
Rusty Russelld3561b72006-12-07 02:14:07 +0100598}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200599
600static inline void wbinvd(void)
601{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700602 PVOP_VCALL0(pv_cpu_ops.wbinvd);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200603}
Rusty Russelld3561b72006-12-07 02:14:07 +0100604
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700605#define get_kernel_rpl() (pv_info.kernel_rpl)
Rusty Russelld3561b72006-12-07 02:14:07 +0100606
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200607static inline u64 paravirt_read_msr(unsigned msr, int *err)
608{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700609 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200610}
611static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
612{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700613 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200614}
615
Rusty Russell90a0a062007-05-02 19:27:10 +0200616/* These should all do BUG_ON(_err), but our headers are too tangled. */
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200617#define rdmsr(msr,val1,val2) do { \
618 int _err; \
619 u64 _l = paravirt_read_msr(msr, &_err); \
620 val1 = (u32)_l; \
621 val2 = _l >> 32; \
Rusty Russelld3561b72006-12-07 02:14:07 +0100622} while(0)
623
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200624#define wrmsr(msr,val1,val2) do { \
625 paravirt_write_msr(msr, val1, val2); \
Rusty Russelld3561b72006-12-07 02:14:07 +0100626} while(0)
627
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200628#define rdmsrl(msr,val) do { \
629 int _err; \
630 val = paravirt_read_msr(msr, &_err); \
Rusty Russelld3561b72006-12-07 02:14:07 +0100631} while(0)
632
Björn Steinbrinkb9e36142007-06-25 23:04:37 +0200633#define wrmsrl(msr,val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200634#define wrmsr_safe(msr,a,b) paravirt_write_msr(msr, a, b)
Rusty Russelld3561b72006-12-07 02:14:07 +0100635
636/* rdmsr with exception handling */
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200637#define rdmsr_safe(msr,a,b) ({ \
638 int _err; \
639 u64 _l = paravirt_read_msr(msr, &_err); \
640 (*a) = (u32)_l; \
641 (*b) = _l >> 32; \
Rusty Russelld3561b72006-12-07 02:14:07 +0100642 _err; })
643
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200644
645static inline u64 paravirt_read_tsc(void)
646{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700647 return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200648}
Rusty Russelld3561b72006-12-07 02:14:07 +0100649
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200650#define rdtscl(low) do { \
651 u64 _l = paravirt_read_tsc(); \
652 low = (int)_l; \
Rusty Russelld3561b72006-12-07 02:14:07 +0100653} while(0)
654
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200655#define rdtscll(val) (val = paravirt_read_tsc())
Rusty Russelld3561b72006-12-07 02:14:07 +0100656
Jeremy Fitzhardinge688340e2007-07-17 18:37:04 -0700657static inline unsigned long long paravirt_sched_clock(void)
658{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700659 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
Jeremy Fitzhardinge688340e2007-07-17 18:37:04 -0700660}
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700661#define calculate_cpu_khz() (pv_time_ops.get_cpu_khz())
Zachary Amsden6cb9a832007-03-05 00:30:35 -0800662
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200663static inline unsigned long long paravirt_read_pmc(int counter)
664{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700665 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200666}
667
668#define rdpmc(counter,low,high) do { \
669 u64 _l = paravirt_read_pmc(counter); \
670 low = (u32)_l; \
671 high = _l >> 32; \
Rusty Russelld3561b72006-12-07 02:14:07 +0100672} while(0)
673
Glauber de Oliveira Costae5aaac42008-01-30 13:32:05 +0100674static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
675{
676 return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
677}
678
679#define rdtscp(low, high, aux) \
680do { \
681 int __aux; \
682 unsigned long __val = paravirt_rdtscp(&__aux); \
683 (low) = (u32)__val; \
684 (high) = (u32)(__val >> 32); \
685 (aux) = __aux; \
686} while (0)
687
688#define rdtscpll(val, aux) \
689do { \
690 unsigned long __aux; \
691 val = paravirt_rdtscp(&__aux); \
692 (aux) = __aux; \
693} while (0)
694
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200695static inline void load_TR_desc(void)
696{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700697 PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200698}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100699static inline void load_gdt(const struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200700{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700701 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200702}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100703static inline void load_idt(const struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200704{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700705 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200706}
707static inline void set_ldt(const void *addr, unsigned entries)
708{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700709 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200710}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100711static inline void store_gdt(struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200712{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700713 PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200714}
Glauber de Oliveira Costa6b68f012008-01-30 13:31:12 +0100715static inline void store_idt(struct desc_ptr *dtr)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200716{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700717 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200718}
719static inline unsigned long paravirt_store_tr(void)
720{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700721 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200722}
723#define store_tr(tr) ((tr) = paravirt_store_tr())
724static inline void load_TLS(struct thread_struct *t, unsigned cpu)
725{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700726 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200727}
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100728
729static inline void write_ldt_entry(struct desc_struct *dt, int entry,
730 const void *desc)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200731{
Glauber de Oliveira Costa75b8bb32008-01-30 13:31:13 +0100732 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200733}
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100734
735static inline void write_gdt_entry(struct desc_struct *dt, int entry,
736 void *desc, int type)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200737{
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100738 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200739}
Glauber de Oliveira Costa014b15b2008-01-30 13:31:13 +0100740
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100741static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200742{
Glauber de Oliveira Costa8d947342008-01-30 13:31:12 +0100743 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200744}
745static inline void set_iopl_mask(unsigned mask)
746{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700747 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200748}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200749
Rusty Russelld3561b72006-12-07 02:14:07 +0100750/* The paravirtualized I/O functions */
751static inline void slow_down_io(void) {
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700752 pv_cpu_ops.io_delay();
Rusty Russelld3561b72006-12-07 02:14:07 +0100753#ifdef REALLY_SLOW_IO
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700754 pv_cpu_ops.io_delay();
755 pv_cpu_ops.io_delay();
756 pv_cpu_ops.io_delay();
Rusty Russelld3561b72006-12-07 02:14:07 +0100757#endif
758}
759
Rusty Russell13623d72006-12-07 02:14:08 +0100760#ifdef CONFIG_X86_LOCAL_APIC
761/*
762 * Basic functions accessing APICs.
763 */
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100764static inline void apic_write(unsigned long reg, u32 v)
Rusty Russell13623d72006-12-07 02:14:08 +0100765{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700766 PVOP_VCALL2(pv_apic_ops.apic_write, reg, v);
Rusty Russell13623d72006-12-07 02:14:08 +0100767}
768
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100769static inline void apic_write_atomic(unsigned long reg, u32 v)
Rusty Russell13623d72006-12-07 02:14:08 +0100770{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700771 PVOP_VCALL2(pv_apic_ops.apic_write_atomic, reg, v);
Rusty Russell13623d72006-12-07 02:14:08 +0100772}
773
Thomas Gleixner42e0a9aa2008-01-30 13:30:15 +0100774static inline u32 apic_read(unsigned long reg)
Rusty Russell13623d72006-12-07 02:14:08 +0100775{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700776 return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg);
Rusty Russell13623d72006-12-07 02:14:08 +0100777}
Zachary Amsdenbbab4f32007-02-13 13:26:21 +0100778
779static inline void setup_boot_clock(void)
780{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700781 PVOP_VCALL0(pv_apic_ops.setup_boot_clock);
Zachary Amsdenbbab4f32007-02-13 13:26:21 +0100782}
783
784static inline void setup_secondary_clock(void)
785{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700786 PVOP_VCALL0(pv_apic_ops.setup_secondary_clock);
Zachary Amsdenbbab4f32007-02-13 13:26:21 +0100787}
Rusty Russell13623d72006-12-07 02:14:08 +0100788#endif
789
Jeremy Fitzhardinge6996d3b2007-07-17 18:37:03 -0700790static inline void paravirt_post_allocator_init(void)
791{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700792 if (pv_init_ops.post_allocator_init)
793 (*pv_init_ops.post_allocator_init)();
Jeremy Fitzhardinge6996d3b2007-07-17 18:37:03 -0700794}
795
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200796static inline void paravirt_pagetable_setup_start(pgd_t *base)
797{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700798 (*pv_mmu_ops.pagetable_setup_start)(base);
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200799}
800
801static inline void paravirt_pagetable_setup_done(pgd_t *base)
802{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700803 (*pv_mmu_ops.pagetable_setup_done)(base);
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200804}
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200805
Zachary Amsdenae5da272007-02-13 13:26:21 +0100806#ifdef CONFIG_SMP
807static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
808 unsigned long start_esp)
809{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700810 PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
811 phys_apicid, start_eip, start_esp);
Zachary Amsdenae5da272007-02-13 13:26:21 +0100812}
813#endif
Rusty Russell13623d72006-12-07 02:14:08 +0100814
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200815static inline void paravirt_activate_mm(struct mm_struct *prev,
816 struct mm_struct *next)
817{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700818 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200819}
820
821static inline void arch_dup_mmap(struct mm_struct *oldmm,
822 struct mm_struct *mm)
823{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700824 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200825}
826
827static inline void arch_exit_mmap(struct mm_struct *mm)
828{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700829 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200830}
831
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200832static inline void __flush_tlb(void)
833{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700834 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200835}
836static inline void __flush_tlb_global(void)
837{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700838 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200839}
840static inline void __flush_tlb_single(unsigned long addr)
841{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700842 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200843}
Rusty Russellda181a82006-12-07 02:14:08 +0100844
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200845static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
846 unsigned long va)
847{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700848 PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va);
Jeremy Fitzhardinged4c10472007-05-02 19:27:15 +0200849}
850
Jeremy Fitzhardingefdb4c332007-07-17 18:37:03 -0700851static inline void paravirt_alloc_pt(struct mm_struct *mm, unsigned pfn)
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200852{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700853 PVOP_VCALL2(pv_mmu_ops.alloc_pt, mm, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200854}
855static inline void paravirt_release_pt(unsigned pfn)
856{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700857 PVOP_VCALL1(pv_mmu_ops.release_pt, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200858}
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100859
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200860static inline void paravirt_alloc_pd(unsigned pfn)
861{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700862 PVOP_VCALL1(pv_mmu_ops.alloc_pd, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200863}
864
865static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn,
866 unsigned start, unsigned count)
867{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700868 PVOP_VCALL4(pv_mmu_ops.alloc_pd_clone, pfn, clonepfn, start, count);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200869}
870static inline void paravirt_release_pd(unsigned pfn)
871{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700872 PVOP_VCALL1(pv_mmu_ops.release_pd, pfn);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200873}
874
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +0200875#ifdef CONFIG_HIGHPTE
876static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
877{
878 unsigned long ret;
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700879 ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type);
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +0200880 return (void *)ret;
881}
882#endif
883
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200884static inline void pte_update(struct mm_struct *mm, unsigned long addr,
885 pte_t *ptep)
886{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700887 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200888}
889
890static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
891 pte_t *ptep)
892{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700893 PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200894}
895
896#ifdef CONFIG_X86_PAE
897static inline pte_t __pte(unsigned long long val)
898{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700899 unsigned long long ret = PVOP_CALL2(unsigned long long,
900 pv_mmu_ops.make_pte,
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200901 val, val >> 32);
902 return (pte_t) { ret, ret >> 32 };
903}
904
905static inline pmd_t __pmd(unsigned long long val)
906{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700907 return (pmd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pmd,
908 val, val >> 32) };
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200909}
910
911static inline pgd_t __pgd(unsigned long long val)
912{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700913 return (pgd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pgd,
914 val, val >> 32) };
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200915}
916
917static inline unsigned long long pte_val(pte_t x)
918{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700919 return PVOP_CALL2(unsigned long long, pv_mmu_ops.pte_val,
920 x.pte_low, x.pte_high);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200921}
922
923static inline unsigned long long pmd_val(pmd_t x)
924{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700925 return PVOP_CALL2(unsigned long long, pv_mmu_ops.pmd_val,
926 x.pmd, x.pmd >> 32);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200927}
928
929static inline unsigned long long pgd_val(pgd_t x)
930{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700931 return PVOP_CALL2(unsigned long long, pv_mmu_ops.pgd_val,
932 x.pgd, x.pgd >> 32);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200933}
Zachary Amsdenc119ecc2007-02-13 13:26:21 +0100934
Rusty Russellda181a82006-12-07 02:14:08 +0100935static inline void set_pte(pte_t *ptep, pte_t pteval)
936{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700937 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, pteval.pte_low, pteval.pte_high);
Rusty Russellda181a82006-12-07 02:14:08 +0100938}
939
Jeremy Fitzhardinge3dc494e2007-05-02 19:27:13 +0200940static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
941 pte_t *ptep, pte_t pteval)
Rusty Russellda181a82006-12-07 02:14:08 +0100942{
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200943 /* 5 arg words */
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700944 pv_mmu_ops.set_pte_at(mm, addr, ptep, pteval);
Rusty Russellda181a82006-12-07 02:14:08 +0100945}
946
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200947static inline void set_pte_atomic(pte_t *ptep, pte_t pteval)
948{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700949 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
950 pteval.pte_low, pteval.pte_high);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200951}
952
953static inline void set_pte_present(struct mm_struct *mm, unsigned long addr,
954 pte_t *ptep, pte_t pte)
955{
956 /* 5 arg words */
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700957 pv_mmu_ops.set_pte_present(mm, addr, ptep, pte);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200958}
959
Rusty Russellda181a82006-12-07 02:14:08 +0100960static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval)
961{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700962 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp,
963 pmdval.pmd, pmdval.pmd >> 32);
Rusty Russellda181a82006-12-07 02:14:08 +0100964}
965
966static inline void set_pud(pud_t *pudp, pud_t pudval)
967{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700968 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
969 pudval.pgd.pgd, pudval.pgd.pgd >> 32);
Rusty Russellda181a82006-12-07 02:14:08 +0100970}
971
972static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
973{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700974 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
Rusty Russellda181a82006-12-07 02:14:08 +0100975}
976
977static inline void pmd_clear(pmd_t *pmdp)
978{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700979 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
Rusty Russellda181a82006-12-07 02:14:08 +0100980}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200981
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200982#else /* !CONFIG_X86_PAE */
Jeremy Fitzhardinge4cdd9c82007-05-02 19:27:15 +0200983
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200984static inline pte_t __pte(unsigned long val)
985{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700986 return (pte_t) { PVOP_CALL1(unsigned long, pv_mmu_ops.make_pte, val) };
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200987}
988
989static inline pgd_t __pgd(unsigned long val)
990{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700991 return (pgd_t) { PVOP_CALL1(unsigned long, pv_mmu_ops.make_pgd, val) };
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200992}
993
994static inline unsigned long pte_val(pte_t x)
995{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700996 return PVOP_CALL1(unsigned long, pv_mmu_ops.pte_val, x.pte_low);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +0200997}
998
999static inline unsigned long pgd_val(pgd_t x)
1000{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001001 return PVOP_CALL1(unsigned long, pv_mmu_ops.pgd_val, x.pgd);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001002}
1003
1004static inline void set_pte(pte_t *ptep, pte_t pteval)
1005{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001006 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, pteval.pte_low);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001007}
1008
1009static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
1010 pte_t *ptep, pte_t pteval)
1011{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001012 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pteval.pte_low);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001013}
1014
1015static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval)
1016{
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001017 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, pmdval.pud.pgd.pgd);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001018}
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001019#endif /* CONFIG_X86_PAE */
Rusty Russellda181a82006-12-07 02:14:08 +01001020
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001021/* Lazy mode for batching updates / context switch */
1022enum paravirt_lazy_mode {
1023 PARAVIRT_LAZY_NONE,
1024 PARAVIRT_LAZY_MMU,
1025 PARAVIRT_LAZY_CPU,
1026};
1027
1028enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
1029void paravirt_enter_lazy_cpu(void);
1030void paravirt_leave_lazy_cpu(void);
1031void paravirt_enter_lazy_mmu(void);
1032void paravirt_leave_lazy_mmu(void);
1033void paravirt_leave_lazy(enum paravirt_lazy_mode mode);
1034
Zachary Amsden9226d122007-02-13 13:26:21 +01001035#define __HAVE_ARCH_ENTER_LAZY_CPU_MODE
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001036static inline void arch_enter_lazy_cpu_mode(void)
1037{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001038 PVOP_VCALL0(pv_cpu_ops.lazy_mode.enter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001039}
1040
1041static inline void arch_leave_lazy_cpu_mode(void)
1042{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001043 PVOP_VCALL0(pv_cpu_ops.lazy_mode.leave);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001044}
1045
1046static inline void arch_flush_lazy_cpu_mode(void)
1047{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001048 if (unlikely(paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU)) {
1049 arch_leave_lazy_cpu_mode();
1050 arch_enter_lazy_cpu_mode();
1051 }
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001052}
1053
Zachary Amsden9226d122007-02-13 13:26:21 +01001054
1055#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001056static inline void arch_enter_lazy_mmu_mode(void)
1057{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001058 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001059}
1060
1061static inline void arch_leave_lazy_mmu_mode(void)
1062{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001063 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001064}
1065
1066static inline void arch_flush_lazy_mmu_mode(void)
1067{
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -07001068 if (unlikely(paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU)) {
1069 arch_leave_lazy_mmu_mode();
1070 arch_enter_lazy_mmu_mode();
1071 }
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001072}
Zachary Amsden9226d122007-02-13 13:26:21 +01001073
Jeremy Fitzhardinge45876232007-05-02 19:27:13 +02001074void _paravirt_nop(void);
1075#define paravirt_nop ((void *)_paravirt_nop)
1076
Rusty Russell139ec7c2006-12-07 02:14:08 +01001077/* These all sit in the .parainstructions section to tell us what to patch. */
Jeremy Fitzhardinge98de0322007-05-02 19:27:14 +02001078struct paravirt_patch_site {
Rusty Russell139ec7c2006-12-07 02:14:08 +01001079 u8 *instr; /* original instructions */
1080 u8 instrtype; /* type of this instruction */
1081 u8 len; /* length of original instruction */
1082 u16 clobbers; /* what registers you may clobber */
1083};
1084
Jeremy Fitzhardinge98de0322007-05-02 19:27:14 +02001085extern struct paravirt_patch_site __parainstructions[],
1086 __parainstructions_end[];
1087
Rusty Russell139ec7c2006-12-07 02:14:08 +01001088static inline unsigned long __raw_local_save_flags(void)
1089{
1090 unsigned long f;
1091
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001092 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
1093 PARAVIRT_CALL
1094 "popl %%edx; popl %%ecx")
1095 : "=a"(f)
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001096 : paravirt_type(pv_irq_ops.save_fl),
Jeremy Fitzhardinge42c24fa2007-05-02 19:27:14 +02001097 paravirt_clobber(CLBR_EAX)
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001098 : "memory", "cc");
Rusty Russell139ec7c2006-12-07 02:14:08 +01001099 return f;
1100}
1101
1102static inline void raw_local_irq_restore(unsigned long f)
1103{
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001104 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
1105 PARAVIRT_CALL
1106 "popl %%edx; popl %%ecx")
1107 : "=a"(f)
1108 : "0"(f),
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001109 paravirt_type(pv_irq_ops.restore_fl),
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001110 paravirt_clobber(CLBR_EAX)
1111 : "memory", "cc");
Rusty Russell139ec7c2006-12-07 02:14:08 +01001112}
1113
1114static inline void raw_local_irq_disable(void)
1115{
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001116 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
1117 PARAVIRT_CALL
1118 "popl %%edx; popl %%ecx")
1119 :
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001120 : paravirt_type(pv_irq_ops.irq_disable),
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001121 paravirt_clobber(CLBR_EAX)
1122 : "memory", "eax", "cc");
Rusty Russell139ec7c2006-12-07 02:14:08 +01001123}
1124
1125static inline void raw_local_irq_enable(void)
1126{
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001127 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
1128 PARAVIRT_CALL
1129 "popl %%edx; popl %%ecx")
1130 :
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001131 : paravirt_type(pv_irq_ops.irq_enable),
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001132 paravirt_clobber(CLBR_EAX)
1133 : "memory", "eax", "cc");
Rusty Russell139ec7c2006-12-07 02:14:08 +01001134}
1135
1136static inline unsigned long __raw_local_irq_save(void)
1137{
1138 unsigned long f;
1139
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001140 f = __raw_local_save_flags();
1141 raw_local_irq_disable();
Rusty Russell139ec7c2006-12-07 02:14:08 +01001142 return f;
1143}
1144
Jeremy Fitzhardinge294688c2007-05-02 19:27:14 +02001145/* Make sure as little as possible of this mess escapes. */
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001146#undef PARAVIRT_CALL
Jeremy Fitzhardinge1a45b7a2007-05-02 19:27:15 +02001147#undef __PVOP_CALL
1148#undef __PVOP_VCALL
Jeremy Fitzhardingef8822f42007-05-02 19:27:14 +02001149#undef PVOP_VCALL0
1150#undef PVOP_CALL0
1151#undef PVOP_VCALL1
1152#undef PVOP_CALL1
1153#undef PVOP_VCALL2
1154#undef PVOP_CALL2
1155#undef PVOP_VCALL3
1156#undef PVOP_CALL3
1157#undef PVOP_VCALL4
1158#undef PVOP_CALL4
Rusty Russell139ec7c2006-12-07 02:14:08 +01001159
Rusty Russelld3561b72006-12-07 02:14:07 +01001160#else /* __ASSEMBLY__ */
1161
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001162#define _PVSITE(ptype, clobbers, ops, word, algn) \
Rusty Russell139ec7c2006-12-07 02:14:08 +01001163771:; \
1164 ops; \
1165772:; \
1166 .pushsection .parainstructions,"a"; \
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001167 .align algn; \
1168 word 771b; \
Rusty Russell139ec7c2006-12-07 02:14:08 +01001169 .byte ptype; \
1170 .byte 772b-771b; \
1171 .short clobbers; \
1172 .popsection
1173
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001174
1175#ifdef CONFIG_X86_64
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001176#define PV_SAVE_REGS pushq %rax; pushq %rdi; pushq %rcx; pushq %rdx
1177#define PV_RESTORE_REGS popq %rdx; popq %rcx; popq %rdi; popq %rax
1178#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001179#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
1180#else
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001181#define PV_SAVE_REGS pushl %eax; pushl %edi; pushl %ecx; pushl %edx
1182#define PV_RESTORE_REGS popl %edx; popl %ecx; popl %edi; popl %eax
1183#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
Glauber de Oliveira Costa658be9d2008-01-30 13:32:06 +01001184#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
1185#endif
1186
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001187#define INTERRUPT_RETURN \
1188 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \
1189 jmp *%cs:pv_cpu_ops+PV_CPU_iret)
Rusty Russell139ec7c2006-12-07 02:14:08 +01001190
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001191#define DISABLE_INTERRUPTS(clobbers) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001192 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001193 PV_SAVE_REGS; \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001194 call *%cs:pv_irq_ops+PV_IRQ_irq_disable; \
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001195 PV_RESTORE_REGS;) \
Rusty Russell139ec7c2006-12-07 02:14:08 +01001196
Jeremy Fitzhardinged5822032007-05-02 19:27:14 +02001197#define ENABLE_INTERRUPTS(clobbers) \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001198 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001199 PV_SAVE_REGS; \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001200 call *%cs:pv_irq_ops+PV_IRQ_irq_enable; \
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001201 PV_RESTORE_REGS;)
Rusty Russell139ec7c2006-12-07 02:14:08 +01001202
Glauber de Oliveira Costa6abcd982008-01-30 13:30:33 +01001203#define ENABLE_INTERRUPTS_SYSCALL_RET \
1204 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_syscall_ret),\
1205 CLBR_NONE, \
1206 jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_syscall_ret)
Rusty Russell139ec7c2006-12-07 02:14:08 +01001207
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001208#ifdef CONFIG_X86_32
Rusty Russell139ec7c2006-12-07 02:14:08 +01001209#define GET_CR0_INTO_EAX \
Jeremy Fitzhardinge42c24fa2007-05-02 19:27:14 +02001210 push %ecx; push %edx; \
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -07001211 call *pv_cpu_ops+PV_CPU_read_cr0; \
Jeremy Fitzhardinge42c24fa2007-05-02 19:27:14 +02001212 pop %edx; pop %ecx
Glauber de Oliveira Costa6057fc82008-01-30 13:32:06 +01001213#endif
Rusty Russell139ec7c2006-12-07 02:14:08 +01001214
Rusty Russelld3561b72006-12-07 02:14:07 +01001215#endif /* __ASSEMBLY__ */
1216#endif /* CONFIG_PARAVIRT */
1217#endif /* __ASM_PARAVIRT_H */