Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 1 | #ifndef __ASM_PARAVIRT_H |
| 2 | #define __ASM_PARAVIRT_H |
| 3 | /* Various instructions on x86 need to be replaced for |
| 4 | * para-virtualization: those hooks are defined here. */ |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 5 | |
| 6 | #ifdef CONFIG_PARAVIRT |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 7 | #include <asm/page.h> |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 8 | #include <asm/asm.h> |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 9 | |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 10 | /* Bitmask of what can be clobbered: usually at least eax. */ |
Glauber de Oliveira Costa | 21438f7 | 2008-01-30 13:32:09 +0100 | [diff] [blame^] | 11 | #define CLBR_NONE 0 |
| 12 | #define CLBR_EAX (1 << 0) |
| 13 | #define CLBR_ECX (1 << 1) |
| 14 | #define CLBR_EDX (1 << 2) |
| 15 | |
| 16 | #ifdef CONFIG_X86_64 |
| 17 | #define CLBR_RSI (1 << 3) |
| 18 | #define CLBR_RDI (1 << 4) |
| 19 | #define CLBR_R8 (1 << 5) |
| 20 | #define CLBR_R9 (1 << 6) |
| 21 | #define CLBR_R10 (1 << 7) |
| 22 | #define CLBR_R11 (1 << 8) |
| 23 | #define CLBR_ANY ((1 << 9) - 1) |
| 24 | #include <asm/desc_defs.h> |
| 25 | #else |
| 26 | /* CLBR_ANY should match all regs platform has. For i386, that's just it */ |
| 27 | #define CLBR_ANY ((1 << 3) - 1) |
| 28 | #endif /* X86_64 */ |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 29 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 30 | #ifndef __ASSEMBLY__ |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 31 | #include <linux/types.h> |
Jeremy Fitzhardinge | d4c1047 | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 32 | #include <linux/cpumask.h> |
Jeremy Fitzhardinge | ce6234b | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 33 | #include <asm/kmap_types.h> |
Glauber de Oliveira Costa | 8d94734 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 34 | #include <asm/desc_defs.h> |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 35 | |
Jeremy Fitzhardinge | ce6234b | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 36 | struct page; |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 37 | struct thread_struct; |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 38 | struct desc_ptr; |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 39 | struct tss_struct; |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 40 | struct mm_struct; |
Rusty Russell | 90a0a06 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 41 | struct desc_struct; |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 42 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 43 | /* general info */ |
| 44 | struct pv_info { |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 45 | unsigned int kernel_rpl; |
Jeremy Fitzhardinge | 5311ab6 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 46 | int shared_kernel_pmd; |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 47 | int paravirt_enabled; |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 48 | const char *name; |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 49 | }; |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 50 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 51 | struct pv_init_ops { |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 52 | /* |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 53 | * Patch may replace one of the defined code sequences with |
| 54 | * arbitrary code, subject to the same register constraints. |
| 55 | * This generally means the code is not free to clobber any |
| 56 | * registers other than EAX. The patch function should return |
| 57 | * the number of bytes of code generated, as we nop pad the |
| 58 | * rest in generic code. |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 59 | */ |
Andi Kleen | ab144f5 | 2007-08-10 22:31:03 +0200 | [diff] [blame] | 60 | unsigned (*patch)(u8 type, u16 clobber, void *insnbuf, |
| 61 | unsigned long addr, unsigned len); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 62 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 63 | /* Basic arch-specific setup */ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 64 | void (*arch_setup)(void); |
| 65 | char *(*memory_setup)(void); |
Jeremy Fitzhardinge | 6996d3b | 2007-07-17 18:37:03 -0700 | [diff] [blame] | 66 | void (*post_allocator_init)(void); |
| 67 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 68 | /* Print a banner to identify the environment */ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 69 | void (*banner)(void); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 70 | }; |
| 71 | |
| 72 | |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 73 | struct pv_lazy_ops { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 74 | /* Set deferred update mode, used for batching operations. */ |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 75 | void (*enter)(void); |
| 76 | void (*leave)(void); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 77 | }; |
| 78 | |
| 79 | struct pv_time_ops { |
| 80 | void (*time_init)(void); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 81 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 82 | /* Set and set time of day */ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 83 | unsigned long (*get_wallclock)(void); |
| 84 | int (*set_wallclock)(unsigned long); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 85 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 86 | unsigned long long (*sched_clock)(void); |
| 87 | unsigned long (*get_cpu_khz)(void); |
| 88 | }; |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 89 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 90 | struct pv_cpu_ops { |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 91 | /* hooks for various privileged instructions */ |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 92 | unsigned long (*get_debugreg)(int regno); |
| 93 | void (*set_debugreg)(int regno, unsigned long value); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 94 | |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 95 | void (*clts)(void); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 96 | |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 97 | unsigned long (*read_cr0)(void); |
| 98 | void (*write_cr0)(unsigned long); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 99 | |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 100 | unsigned long (*read_cr4_safe)(void); |
| 101 | unsigned long (*read_cr4)(void); |
| 102 | void (*write_cr4)(unsigned long); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 103 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 104 | /* Segment descriptor handling */ |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 105 | void (*load_tr_desc)(void); |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 106 | void (*load_gdt)(const struct desc_ptr *); |
| 107 | void (*load_idt)(const struct desc_ptr *); |
| 108 | void (*store_gdt)(struct desc_ptr *); |
| 109 | void (*store_idt)(struct desc_ptr *); |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 110 | void (*set_ldt)(const void *desc, unsigned entries); |
| 111 | unsigned long (*store_tr)(void); |
| 112 | void (*load_tls)(struct thread_struct *t, unsigned int cpu); |
Glauber de Oliveira Costa | 75b8bb3 | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 113 | void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum, |
| 114 | const void *desc); |
Rusty Russell | 90a0a06 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 115 | void (*write_gdt_entry)(struct desc_struct *, |
Glauber de Oliveira Costa | 014b15b | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 116 | int entrynum, const void *desc, int size); |
Glauber de Oliveira Costa | 8d94734 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 117 | void (*write_idt_entry)(gate_desc *, |
| 118 | int entrynum, const gate_desc *gate); |
H. Peter Anvin | faca622 | 2008-01-30 13:31:02 +0100 | [diff] [blame] | 119 | void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 120 | |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 121 | void (*set_iopl_mask)(unsigned mask); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 122 | |
| 123 | void (*wbinvd)(void); |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 124 | void (*io_delay)(void); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 125 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 126 | /* cpuid emulation, mostly so that caps bits can be disabled */ |
| 127 | void (*cpuid)(unsigned int *eax, unsigned int *ebx, |
| 128 | unsigned int *ecx, unsigned int *edx); |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 129 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 130 | /* MSR, PMC and TSR operations. |
| 131 | err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */ |
| 132 | u64 (*read_msr)(unsigned int msr, int *err); |
Glauber de Oliveira Costa | c9dcda5 | 2008-01-30 13:31:07 +0100 | [diff] [blame] | 133 | int (*write_msr)(unsigned int msr, unsigned low, unsigned high); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 134 | |
| 135 | u64 (*read_tsc)(void); |
Glauber de Oliveira Costa | b8d1fae | 2008-01-30 13:31:07 +0100 | [diff] [blame] | 136 | u64 (*read_pmc)(int counter); |
Glauber de Oliveira Costa | e5aaac4 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 137 | unsigned long long (*read_tscp)(unsigned int *aux); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 138 | |
| 139 | /* These two are jmp to, not actually called. */ |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 140 | void (*irq_enable_syscall_ret)(void); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 141 | void (*iret)(void); |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 142 | |
Glauber de Oliveira Costa | e801f86 | 2008-01-30 13:32:08 +0100 | [diff] [blame] | 143 | void (*swapgs)(void); |
| 144 | |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 145 | struct pv_lazy_ops lazy_mode; |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 146 | }; |
| 147 | |
| 148 | struct pv_irq_ops { |
| 149 | void (*init_IRQ)(void); |
| 150 | |
| 151 | /* |
| 152 | * Get/set interrupt state. save_fl and restore_fl are only |
| 153 | * expected to use X86_EFLAGS_IF; all other bits |
| 154 | * returned from save_fl are undefined, and may be ignored by |
| 155 | * restore_fl. |
| 156 | */ |
| 157 | unsigned long (*save_fl)(void); |
| 158 | void (*restore_fl)(unsigned long); |
| 159 | void (*irq_disable)(void); |
| 160 | void (*irq_enable)(void); |
| 161 | void (*safe_halt)(void); |
| 162 | void (*halt)(void); |
| 163 | }; |
| 164 | |
| 165 | struct pv_apic_ops { |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 166 | #ifdef CONFIG_X86_LOCAL_APIC |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 167 | /* |
| 168 | * Direct APIC operations, principally for VMI. Ideally |
| 169 | * these shouldn't be in this interface. |
| 170 | */ |
Thomas Gleixner | 42e0a9aa | 2008-01-30 13:30:15 +0100 | [diff] [blame] | 171 | void (*apic_write)(unsigned long reg, u32 v); |
| 172 | void (*apic_write_atomic)(unsigned long reg, u32 v); |
| 173 | u32 (*apic_read)(unsigned long reg); |
Zachary Amsden | bbab4f3 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 174 | void (*setup_boot_clock)(void); |
| 175 | void (*setup_secondary_clock)(void); |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 176 | |
| 177 | void (*startup_ipi_hook)(int phys_apicid, |
| 178 | unsigned long start_eip, |
| 179 | unsigned long start_esp); |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 180 | #endif |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 181 | }; |
| 182 | |
| 183 | struct pv_mmu_ops { |
| 184 | /* |
| 185 | * Called before/after init_mm pagetable setup. setup_start |
| 186 | * may reset %cr3, and may pre-install parts of the pagetable; |
| 187 | * pagetable setup is expected to preserve any existing |
| 188 | * mapping. |
| 189 | */ |
| 190 | void (*pagetable_setup_start)(pgd_t *pgd_base); |
| 191 | void (*pagetable_setup_done)(pgd_t *pgd_base); |
| 192 | |
| 193 | unsigned long (*read_cr2)(void); |
| 194 | void (*write_cr2)(unsigned long); |
| 195 | |
| 196 | unsigned long (*read_cr3)(void); |
| 197 | void (*write_cr3)(unsigned long); |
| 198 | |
| 199 | /* |
| 200 | * Hooks for intercepting the creation/use/destruction of an |
| 201 | * mm_struct. |
| 202 | */ |
| 203 | void (*activate_mm)(struct mm_struct *prev, |
| 204 | struct mm_struct *next); |
| 205 | void (*dup_mmap)(struct mm_struct *oldmm, |
| 206 | struct mm_struct *mm); |
| 207 | void (*exit_mmap)(struct mm_struct *mm); |
| 208 | |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 209 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 210 | /* TLB operations */ |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 211 | void (*flush_tlb_user)(void); |
| 212 | void (*flush_tlb_kernel)(void); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 213 | void (*flush_tlb_single)(unsigned long addr); |
Jeremy Fitzhardinge | d4c1047 | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 214 | void (*flush_tlb_others)(const cpumask_t *cpus, struct mm_struct *mm, |
| 215 | unsigned long va); |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 216 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 217 | /* Hooks for allocating/releasing pagetable pages */ |
Jeremy Fitzhardinge | fdb4c33 | 2007-07-17 18:37:03 -0700 | [diff] [blame] | 218 | void (*alloc_pt)(struct mm_struct *mm, u32 pfn); |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 219 | void (*alloc_pd)(u32 pfn); |
| 220 | void (*alloc_pd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count); |
| 221 | void (*release_pt)(u32 pfn); |
| 222 | void (*release_pd)(u32 pfn); |
Zachary Amsden | c119ecc | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 223 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 224 | /* Pagetable manipulation functions */ |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 225 | void (*set_pte)(pte_t *ptep, pte_t pteval); |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 226 | void (*set_pte_at)(struct mm_struct *mm, unsigned long addr, |
| 227 | pte_t *ptep, pte_t pteval); |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 228 | void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 229 | void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 230 | void (*pte_update_defer)(struct mm_struct *mm, |
| 231 | unsigned long addr, pte_t *ptep); |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 232 | |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 233 | #ifdef CONFIG_X86_PAE |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 234 | void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 235 | void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, |
| 236 | pte_t *ptep, pte_t pte); |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 237 | void (*set_pud)(pud_t *pudp, pud_t pudval); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 238 | void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); |
Andi Kleen | 1a1eecd | 2007-02-13 13:26:25 +0100 | [diff] [blame] | 239 | void (*pmd_clear)(pmd_t *pmdp); |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 240 | |
| 241 | unsigned long long (*pte_val)(pte_t); |
| 242 | unsigned long long (*pmd_val)(pmd_t); |
| 243 | unsigned long long (*pgd_val)(pgd_t); |
| 244 | |
| 245 | pte_t (*make_pte)(unsigned long long pte); |
| 246 | pmd_t (*make_pmd)(unsigned long long pmd); |
| 247 | pgd_t (*make_pgd)(unsigned long long pgd); |
| 248 | #else |
| 249 | unsigned long (*pte_val)(pte_t); |
| 250 | unsigned long (*pgd_val)(pgd_t); |
| 251 | |
| 252 | pte_t (*make_pte)(unsigned long pte); |
| 253 | pgd_t (*make_pgd)(unsigned long pgd); |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 254 | #endif |
| 255 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 256 | #ifdef CONFIG_HIGHPTE |
| 257 | void *(*kmap_atomic_pte)(struct page *page, enum km_type type); |
| 258 | #endif |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 259 | |
| 260 | struct pv_lazy_ops lazy_mode; |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 261 | }; |
| 262 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 263 | /* This contains all the paravirt structures: we get a convenient |
| 264 | * number for each function using the offset which we use to indicate |
| 265 | * what to patch. */ |
| 266 | struct paravirt_patch_template |
| 267 | { |
| 268 | struct pv_init_ops pv_init_ops; |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 269 | struct pv_time_ops pv_time_ops; |
| 270 | struct pv_cpu_ops pv_cpu_ops; |
| 271 | struct pv_irq_ops pv_irq_ops; |
| 272 | struct pv_apic_ops pv_apic_ops; |
| 273 | struct pv_mmu_ops pv_mmu_ops; |
| 274 | }; |
| 275 | |
| 276 | extern struct pv_info pv_info; |
| 277 | extern struct pv_init_ops pv_init_ops; |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 278 | extern struct pv_time_ops pv_time_ops; |
| 279 | extern struct pv_cpu_ops pv_cpu_ops; |
| 280 | extern struct pv_irq_ops pv_irq_ops; |
| 281 | extern struct pv_apic_ops pv_apic_ops; |
| 282 | extern struct pv_mmu_ops pv_mmu_ops; |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 283 | |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 284 | #define PARAVIRT_PATCH(x) \ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 285 | (offsetof(struct paravirt_patch_template, x) / sizeof(void *)) |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 286 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 287 | #define paravirt_type(op) \ |
| 288 | [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \ |
| 289 | [paravirt_opptr] "m" (op) |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 290 | #define paravirt_clobber(clobber) \ |
| 291 | [paravirt_clobber] "i" (clobber) |
| 292 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 293 | /* |
| 294 | * Generate some code, and mark it as patchable by the |
| 295 | * apply_paravirt() alternate instruction patcher. |
| 296 | */ |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 297 | #define _paravirt_alt(insn_string, type, clobber) \ |
| 298 | "771:\n\t" insn_string "\n" "772:\n" \ |
| 299 | ".pushsection .parainstructions,\"a\"\n" \ |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 300 | _ASM_ALIGN "\n" \ |
| 301 | _ASM_PTR " 771b\n" \ |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 302 | " .byte " type "\n" \ |
| 303 | " .byte 772b-771b\n" \ |
| 304 | " .short " clobber "\n" \ |
| 305 | ".popsection\n" |
| 306 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 307 | /* Generate patchable code, with the default asm parameters. */ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 308 | #define paravirt_alt(insn_string) \ |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 309 | _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]") |
| 310 | |
Jeremy Fitzhardinge | 63f7027 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 311 | unsigned paravirt_patch_nop(void); |
| 312 | unsigned paravirt_patch_ignore(unsigned len); |
Andi Kleen | ab144f5 | 2007-08-10 22:31:03 +0200 | [diff] [blame] | 313 | unsigned paravirt_patch_call(void *insnbuf, |
| 314 | const void *target, u16 tgt_clobbers, |
| 315 | unsigned long addr, u16 site_clobbers, |
Jeremy Fitzhardinge | 63f7027 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 316 | unsigned len); |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 317 | unsigned paravirt_patch_jmp(void *insnbuf, const void *target, |
Andi Kleen | ab144f5 | 2007-08-10 22:31:03 +0200 | [diff] [blame] | 318 | unsigned long addr, unsigned len); |
| 319 | unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, |
| 320 | unsigned long addr, unsigned len); |
Jeremy Fitzhardinge | 63f7027 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 321 | |
Andi Kleen | ab144f5 | 2007-08-10 22:31:03 +0200 | [diff] [blame] | 322 | unsigned paravirt_patch_insns(void *insnbuf, unsigned len, |
Jeremy Fitzhardinge | 63f7027 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 323 | const char *start, const char *end); |
| 324 | |
Jeremy Fitzhardinge | d572929 | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 325 | int paravirt_disable_iospace(void); |
Jeremy Fitzhardinge | 63f7027 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 326 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 327 | /* |
| 328 | * This generates an indirect call based on the operation type number. |
| 329 | * The type number, computed in PARAVIRT_PATCH, is derived from the |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 330 | * offset into the paravirt_patch_template structure, and can therefore be |
| 331 | * freely converted back into a structure offset. |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 332 | */ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 333 | #define PARAVIRT_CALL "call *%[paravirt_opptr];" |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 334 | |
| 335 | /* |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 336 | * These macros are intended to wrap calls through one of the paravirt |
| 337 | * ops structs, so that they can be later identified and patched at |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 338 | * runtime. |
| 339 | * |
| 340 | * Normally, a call to a pv_op function is a simple indirect call: |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 341 | * (pv_op_struct.operations)(args...). |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 342 | * |
| 343 | * Unfortunately, this is a relatively slow operation for modern CPUs, |
| 344 | * because it cannot necessarily determine what the destination |
| 345 | * address is. In this case, the address is a runtime constant, so at |
| 346 | * the very least we can patch the call to e a simple direct call, or |
| 347 | * ideally, patch an inline implementation into the callsite. (Direct |
| 348 | * calls are essentially free, because the call and return addresses |
| 349 | * are completely predictable.) |
| 350 | * |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 351 | * For i386, these macros rely on the standard gcc "regparm(3)" calling |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 352 | * convention, in which the first three arguments are placed in %eax, |
| 353 | * %edx, %ecx (in that order), and the remaining arguments are placed |
| 354 | * on the stack. All caller-save registers (eax,edx,ecx) are expected |
| 355 | * to be modified (either clobbered or used for return values). |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 356 | * X86_64, on the other hand, already specifies a register-based calling |
| 357 | * conventions, returning at %rax, with parameteres going on %rdi, %rsi, |
| 358 | * %rdx, and %rcx. Note that for this reason, x86_64 does not need any |
| 359 | * special handling for dealing with 4 arguments, unlike i386. |
| 360 | * However, x86_64 also have to clobber all caller saved registers, which |
| 361 | * unfortunately, are quite a bit (r8 - r11) |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 362 | * |
| 363 | * The call instruction itself is marked by placing its start address |
| 364 | * and size into the .parainstructions section, so that |
| 365 | * apply_paravirt() in arch/i386/kernel/alternative.c can do the |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 366 | * appropriate patching under the control of the backend pv_init_ops |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 367 | * implementation. |
| 368 | * |
| 369 | * Unfortunately there's no way to get gcc to generate the args setup |
| 370 | * for the call, and then allow the call itself to be generated by an |
| 371 | * inline asm. Because of this, we must do the complete arg setup and |
| 372 | * return value handling from within these macros. This is fairly |
| 373 | * cumbersome. |
| 374 | * |
| 375 | * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments. |
| 376 | * It could be extended to more arguments, but there would be little |
| 377 | * to be gained from that. For each number of arguments, there are |
| 378 | * the two VCALL and CALL variants for void and non-void functions. |
| 379 | * |
| 380 | * When there is a return value, the invoker of the macro must specify |
| 381 | * the return type. The macro then uses sizeof() on that type to |
| 382 | * determine whether its a 32 or 64 bit value, and places the return |
| 383 | * in the right register(s) (just %eax for 32-bit, and %edx:%eax for |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 384 | * 64-bit). For x86_64 machines, it just returns at %rax regardless of |
| 385 | * the return value size. |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 386 | * |
| 387 | * 64-bit arguments are passed as a pair of adjacent 32-bit arguments |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 388 | * i386 also passes 64-bit arguments as a pair of adjacent 32-bit arguments |
| 389 | * in low,high order |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 390 | * |
| 391 | * Small structures are passed and returned in registers. The macro |
| 392 | * calling convention can't directly deal with this, so the wrapper |
| 393 | * functions must do this. |
| 394 | * |
| 395 | * These PVOP_* macros are only defined within this header. This |
| 396 | * means that all uses must be wrapped in inline functions. This also |
| 397 | * makes sure the incoming and outgoing types are always correct. |
| 398 | */ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 399 | #ifdef CONFIG_X86_32 |
| 400 | #define PVOP_VCALL_ARGS unsigned long __eax, __edx, __ecx |
| 401 | #define PVOP_CALL_ARGS PVOP_VCALL_ARGS |
| 402 | #define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \ |
| 403 | "=c" (__ecx) |
| 404 | #define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS |
| 405 | #define EXTRA_CLOBBERS |
| 406 | #define VEXTRA_CLOBBERS |
| 407 | #else |
| 408 | #define PVOP_VCALL_ARGS unsigned long __edi, __esi, __edx, __ecx |
| 409 | #define PVOP_CALL_ARGS PVOP_VCALL_ARGS, __eax |
| 410 | #define PVOP_VCALL_CLOBBERS "=D" (__edi), \ |
| 411 | "=S" (__esi), "=d" (__edx), \ |
| 412 | "=c" (__ecx) |
| 413 | |
| 414 | #define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax) |
| 415 | |
| 416 | #define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11" |
| 417 | #define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11" |
| 418 | #endif |
| 419 | |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 420 | #define __PVOP_CALL(rettype, op, pre, post, ...) \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 421 | ({ \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 422 | rettype __ret; \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 423 | PVOP_CALL_ARGS; \ |
| 424 | /* This is 32-bit specific, but is okay in 64-bit */ \ |
| 425 | /* since this condition will never hold */ \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 426 | if (sizeof(rettype) > sizeof(unsigned long)) { \ |
| 427 | asm volatile(pre \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 428 | paravirt_alt(PARAVIRT_CALL) \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 429 | post \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 430 | : PVOP_CALL_CLOBBERS \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 431 | : paravirt_type(op), \ |
| 432 | paravirt_clobber(CLBR_ANY), \ |
| 433 | ##__VA_ARGS__ \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 434 | : "memory", "cc" EXTRA_CLOBBERS); \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 435 | __ret = (rettype)((((u64)__edx) << 32) | __eax); \ |
| 436 | } else { \ |
| 437 | asm volatile(pre \ |
| 438 | paravirt_alt(PARAVIRT_CALL) \ |
| 439 | post \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 440 | : PVOP_CALL_CLOBBERS \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 441 | : paravirt_type(op), \ |
| 442 | paravirt_clobber(CLBR_ANY), \ |
| 443 | ##__VA_ARGS__ \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 444 | : "memory", "cc" EXTRA_CLOBBERS); \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 445 | __ret = (rettype)__eax; \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 446 | } \ |
| 447 | __ret; \ |
| 448 | }) |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 449 | #define __PVOP_VCALL(op, pre, post, ...) \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 450 | ({ \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 451 | PVOP_VCALL_ARGS; \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 452 | asm volatile(pre \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 453 | paravirt_alt(PARAVIRT_CALL) \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 454 | post \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 455 | : PVOP_VCALL_CLOBBERS \ |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 456 | : paravirt_type(op), \ |
| 457 | paravirt_clobber(CLBR_ANY), \ |
| 458 | ##__VA_ARGS__ \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 459 | : "memory", "cc" VEXTRA_CLOBBERS); \ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 460 | }) |
| 461 | |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 462 | #define PVOP_CALL0(rettype, op) \ |
| 463 | __PVOP_CALL(rettype, op, "", "") |
| 464 | #define PVOP_VCALL0(op) \ |
| 465 | __PVOP_VCALL(op, "", "") |
| 466 | |
| 467 | #define PVOP_CALL1(rettype, op, arg1) \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 468 | __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1))) |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 469 | #define PVOP_VCALL1(op, arg1) \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 470 | __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1))) |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 471 | |
| 472 | #define PVOP_CALL2(rettype, op, arg1, arg2) \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 473 | __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \ |
| 474 | "1" ((unsigned long)(arg2))) |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 475 | #define PVOP_VCALL2(op, arg1, arg2) \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 476 | __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \ |
| 477 | "1" ((unsigned long)(arg2))) |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 478 | |
| 479 | #define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 480 | __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \ |
| 481 | "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3))) |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 482 | #define PVOP_VCALL3(op, arg1, arg2, arg3) \ |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 483 | __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \ |
| 484 | "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3))) |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 485 | |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 486 | /* This is the only difference in x86_64. We can make it much simpler */ |
| 487 | #ifdef CONFIG_X86_32 |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 488 | #define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \ |
| 489 | __PVOP_CALL(rettype, op, \ |
| 490 | "push %[_arg4];", "lea 4(%%esp),%%esp;", \ |
| 491 | "0" ((u32)(arg1)), "1" ((u32)(arg2)), \ |
| 492 | "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4))) |
| 493 | #define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \ |
| 494 | __PVOP_VCALL(op, \ |
| 495 | "push %[_arg4];", "lea 4(%%esp),%%esp;", \ |
| 496 | "0" ((u32)(arg1)), "1" ((u32)(arg2)), \ |
| 497 | "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4))) |
Glauber de Oliveira Costa | a474636 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 498 | #else |
| 499 | #define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \ |
| 500 | __PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \ |
| 501 | "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \ |
| 502 | "3"((unsigned long)(arg4))) |
| 503 | #define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \ |
| 504 | __PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \ |
| 505 | "1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \ |
| 506 | "3"((unsigned long)(arg4))) |
| 507 | #endif |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 508 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 509 | static inline int paravirt_enabled(void) |
| 510 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 511 | return pv_info.paravirt_enabled; |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 512 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 513 | |
H. Peter Anvin | faca622 | 2008-01-30 13:31:02 +0100 | [diff] [blame] | 514 | static inline void load_sp0(struct tss_struct *tss, |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 515 | struct thread_struct *thread) |
| 516 | { |
H. Peter Anvin | faca622 | 2008-01-30 13:31:02 +0100 | [diff] [blame] | 517 | PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 518 | } |
| 519 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 520 | #define ARCH_SETUP pv_init_ops.arch_setup(); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 521 | static inline unsigned long get_wallclock(void) |
| 522 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 523 | return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 524 | } |
| 525 | |
| 526 | static inline int set_wallclock(unsigned long nowtime) |
| 527 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 528 | return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 529 | } |
| 530 | |
Zachary Amsden | e30fab3 | 2007-03-05 00:30:39 -0800 | [diff] [blame] | 531 | static inline void (*choose_time_init(void))(void) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 532 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 533 | return pv_time_ops.time_init; |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 534 | } |
| 535 | |
| 536 | /* The paravirtualized CPUID instruction. */ |
| 537 | static inline void __cpuid(unsigned int *eax, unsigned int *ebx, |
| 538 | unsigned int *ecx, unsigned int *edx) |
| 539 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 540 | PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 541 | } |
| 542 | |
| 543 | /* |
| 544 | * These special macros can be used to get or set a debugging register |
| 545 | */ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 546 | static inline unsigned long paravirt_get_debugreg(int reg) |
| 547 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 548 | return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 549 | } |
| 550 | #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) |
| 551 | static inline void set_debugreg(unsigned long val, int reg) |
| 552 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 553 | PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 554 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 555 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 556 | static inline void clts(void) |
| 557 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 558 | PVOP_VCALL0(pv_cpu_ops.clts); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 559 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 560 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 561 | static inline unsigned long read_cr0(void) |
| 562 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 563 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 564 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 565 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 566 | static inline void write_cr0(unsigned long x) |
| 567 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 568 | PVOP_VCALL1(pv_cpu_ops.write_cr0, x); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 569 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 570 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 571 | static inline unsigned long read_cr2(void) |
| 572 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 573 | return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 574 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 575 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 576 | static inline void write_cr2(unsigned long x) |
| 577 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 578 | PVOP_VCALL1(pv_mmu_ops.write_cr2, x); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 579 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 580 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 581 | static inline unsigned long read_cr3(void) |
| 582 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 583 | return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 584 | } |
| 585 | |
| 586 | static inline void write_cr3(unsigned long x) |
| 587 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 588 | PVOP_VCALL1(pv_mmu_ops.write_cr3, x); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 589 | } |
| 590 | |
| 591 | static inline unsigned long read_cr4(void) |
| 592 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 593 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 594 | } |
| 595 | static inline unsigned long read_cr4_safe(void) |
| 596 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 597 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 598 | } |
| 599 | |
| 600 | static inline void write_cr4(unsigned long x) |
| 601 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 602 | PVOP_VCALL1(pv_cpu_ops.write_cr4, x); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 603 | } |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 604 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 605 | static inline void raw_safe_halt(void) |
| 606 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 607 | PVOP_VCALL0(pv_irq_ops.safe_halt); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 608 | } |
| 609 | |
| 610 | static inline void halt(void) |
| 611 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 612 | PVOP_VCALL0(pv_irq_ops.safe_halt); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 613 | } |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 614 | |
| 615 | static inline void wbinvd(void) |
| 616 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 617 | PVOP_VCALL0(pv_cpu_ops.wbinvd); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 618 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 619 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 620 | #define get_kernel_rpl() (pv_info.kernel_rpl) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 621 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 622 | static inline u64 paravirt_read_msr(unsigned msr, int *err) |
| 623 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 624 | return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 625 | } |
| 626 | static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) |
| 627 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 628 | return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 629 | } |
| 630 | |
Rusty Russell | 90a0a06 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 631 | /* These should all do BUG_ON(_err), but our headers are too tangled. */ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 632 | #define rdmsr(msr,val1,val2) do { \ |
| 633 | int _err; \ |
| 634 | u64 _l = paravirt_read_msr(msr, &_err); \ |
| 635 | val1 = (u32)_l; \ |
| 636 | val2 = _l >> 32; \ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 637 | } while(0) |
| 638 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 639 | #define wrmsr(msr,val1,val2) do { \ |
| 640 | paravirt_write_msr(msr, val1, val2); \ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 641 | } while(0) |
| 642 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 643 | #define rdmsrl(msr,val) do { \ |
| 644 | int _err; \ |
| 645 | val = paravirt_read_msr(msr, &_err); \ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 646 | } while(0) |
| 647 | |
Björn Steinbrink | b9e3614 | 2007-06-25 23:04:37 +0200 | [diff] [blame] | 648 | #define wrmsrl(msr,val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 649 | #define wrmsr_safe(msr,a,b) paravirt_write_msr(msr, a, b) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 650 | |
| 651 | /* rdmsr with exception handling */ |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 652 | #define rdmsr_safe(msr,a,b) ({ \ |
| 653 | int _err; \ |
| 654 | u64 _l = paravirt_read_msr(msr, &_err); \ |
| 655 | (*a) = (u32)_l; \ |
| 656 | (*b) = _l >> 32; \ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 657 | _err; }) |
| 658 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 659 | |
| 660 | static inline u64 paravirt_read_tsc(void) |
| 661 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 662 | return PVOP_CALL0(u64, pv_cpu_ops.read_tsc); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 663 | } |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 664 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 665 | #define rdtscl(low) do { \ |
| 666 | u64 _l = paravirt_read_tsc(); \ |
| 667 | low = (int)_l; \ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 668 | } while(0) |
| 669 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 670 | #define rdtscll(val) (val = paravirt_read_tsc()) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 671 | |
Jeremy Fitzhardinge | 688340e | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 672 | static inline unsigned long long paravirt_sched_clock(void) |
| 673 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 674 | return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); |
Jeremy Fitzhardinge | 688340e | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 675 | } |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 676 | #define calculate_cpu_khz() (pv_time_ops.get_cpu_khz()) |
Zachary Amsden | 6cb9a83 | 2007-03-05 00:30:35 -0800 | [diff] [blame] | 677 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 678 | static inline unsigned long long paravirt_read_pmc(int counter) |
| 679 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 680 | return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 681 | } |
| 682 | |
| 683 | #define rdpmc(counter,low,high) do { \ |
| 684 | u64 _l = paravirt_read_pmc(counter); \ |
| 685 | low = (u32)_l; \ |
| 686 | high = _l >> 32; \ |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 687 | } while(0) |
| 688 | |
Glauber de Oliveira Costa | e5aaac4 | 2008-01-30 13:32:05 +0100 | [diff] [blame] | 689 | static inline unsigned long long paravirt_rdtscp(unsigned int *aux) |
| 690 | { |
| 691 | return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux); |
| 692 | } |
| 693 | |
| 694 | #define rdtscp(low, high, aux) \ |
| 695 | do { \ |
| 696 | int __aux; \ |
| 697 | unsigned long __val = paravirt_rdtscp(&__aux); \ |
| 698 | (low) = (u32)__val; \ |
| 699 | (high) = (u32)(__val >> 32); \ |
| 700 | (aux) = __aux; \ |
| 701 | } while (0) |
| 702 | |
| 703 | #define rdtscpll(val, aux) \ |
| 704 | do { \ |
| 705 | unsigned long __aux; \ |
| 706 | val = paravirt_rdtscp(&__aux); \ |
| 707 | (aux) = __aux; \ |
| 708 | } while (0) |
| 709 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 710 | static inline void load_TR_desc(void) |
| 711 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 712 | PVOP_VCALL0(pv_cpu_ops.load_tr_desc); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 713 | } |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 714 | static inline void load_gdt(const struct desc_ptr *dtr) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 715 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 716 | PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 717 | } |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 718 | static inline void load_idt(const struct desc_ptr *dtr) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 719 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 720 | PVOP_VCALL1(pv_cpu_ops.load_idt, dtr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 721 | } |
| 722 | static inline void set_ldt(const void *addr, unsigned entries) |
| 723 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 724 | PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 725 | } |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 726 | static inline void store_gdt(struct desc_ptr *dtr) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 727 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 728 | PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 729 | } |
Glauber de Oliveira Costa | 6b68f01 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 730 | static inline void store_idt(struct desc_ptr *dtr) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 731 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 732 | PVOP_VCALL1(pv_cpu_ops.store_idt, dtr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 733 | } |
| 734 | static inline unsigned long paravirt_store_tr(void) |
| 735 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 736 | return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 737 | } |
| 738 | #define store_tr(tr) ((tr) = paravirt_store_tr()) |
| 739 | static inline void load_TLS(struct thread_struct *t, unsigned cpu) |
| 740 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 741 | PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 742 | } |
Glauber de Oliveira Costa | 75b8bb3 | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 743 | |
| 744 | static inline void write_ldt_entry(struct desc_struct *dt, int entry, |
| 745 | const void *desc) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 746 | { |
Glauber de Oliveira Costa | 75b8bb3 | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 747 | PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 748 | } |
Glauber de Oliveira Costa | 014b15b | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 749 | |
| 750 | static inline void write_gdt_entry(struct desc_struct *dt, int entry, |
| 751 | void *desc, int type) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 752 | { |
Glauber de Oliveira Costa | 014b15b | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 753 | PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 754 | } |
Glauber de Oliveira Costa | 014b15b | 2008-01-30 13:31:13 +0100 | [diff] [blame] | 755 | |
Glauber de Oliveira Costa | 8d94734 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 756 | static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 757 | { |
Glauber de Oliveira Costa | 8d94734 | 2008-01-30 13:31:12 +0100 | [diff] [blame] | 758 | PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 759 | } |
| 760 | static inline void set_iopl_mask(unsigned mask) |
| 761 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 762 | PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 763 | } |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 764 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 765 | /* The paravirtualized I/O functions */ |
| 766 | static inline void slow_down_io(void) { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 767 | pv_cpu_ops.io_delay(); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 768 | #ifdef REALLY_SLOW_IO |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 769 | pv_cpu_ops.io_delay(); |
| 770 | pv_cpu_ops.io_delay(); |
| 771 | pv_cpu_ops.io_delay(); |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 772 | #endif |
| 773 | } |
| 774 | |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 775 | #ifdef CONFIG_X86_LOCAL_APIC |
| 776 | /* |
| 777 | * Basic functions accessing APICs. |
| 778 | */ |
Thomas Gleixner | 42e0a9aa | 2008-01-30 13:30:15 +0100 | [diff] [blame] | 779 | static inline void apic_write(unsigned long reg, u32 v) |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 780 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 781 | PVOP_VCALL2(pv_apic_ops.apic_write, reg, v); |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 782 | } |
| 783 | |
Thomas Gleixner | 42e0a9aa | 2008-01-30 13:30:15 +0100 | [diff] [blame] | 784 | static inline void apic_write_atomic(unsigned long reg, u32 v) |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 785 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 786 | PVOP_VCALL2(pv_apic_ops.apic_write_atomic, reg, v); |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 787 | } |
| 788 | |
Thomas Gleixner | 42e0a9aa | 2008-01-30 13:30:15 +0100 | [diff] [blame] | 789 | static inline u32 apic_read(unsigned long reg) |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 790 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 791 | return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg); |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 792 | } |
Zachary Amsden | bbab4f3 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 793 | |
| 794 | static inline void setup_boot_clock(void) |
| 795 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 796 | PVOP_VCALL0(pv_apic_ops.setup_boot_clock); |
Zachary Amsden | bbab4f3 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 797 | } |
| 798 | |
| 799 | static inline void setup_secondary_clock(void) |
| 800 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 801 | PVOP_VCALL0(pv_apic_ops.setup_secondary_clock); |
Zachary Amsden | bbab4f3 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 802 | } |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 803 | #endif |
| 804 | |
Jeremy Fitzhardinge | 6996d3b | 2007-07-17 18:37:03 -0700 | [diff] [blame] | 805 | static inline void paravirt_post_allocator_init(void) |
| 806 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 807 | if (pv_init_ops.post_allocator_init) |
| 808 | (*pv_init_ops.post_allocator_init)(); |
Jeremy Fitzhardinge | 6996d3b | 2007-07-17 18:37:03 -0700 | [diff] [blame] | 809 | } |
| 810 | |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 811 | static inline void paravirt_pagetable_setup_start(pgd_t *base) |
| 812 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 813 | (*pv_mmu_ops.pagetable_setup_start)(base); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 814 | } |
| 815 | |
| 816 | static inline void paravirt_pagetable_setup_done(pgd_t *base) |
| 817 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 818 | (*pv_mmu_ops.pagetable_setup_done)(base); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 819 | } |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 820 | |
Zachary Amsden | ae5da27 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 821 | #ifdef CONFIG_SMP |
| 822 | static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, |
| 823 | unsigned long start_esp) |
| 824 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 825 | PVOP_VCALL3(pv_apic_ops.startup_ipi_hook, |
| 826 | phys_apicid, start_eip, start_esp); |
Zachary Amsden | ae5da27 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 827 | } |
| 828 | #endif |
Rusty Russell | 13623d7 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 829 | |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 830 | static inline void paravirt_activate_mm(struct mm_struct *prev, |
| 831 | struct mm_struct *next) |
| 832 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 833 | PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next); |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 834 | } |
| 835 | |
| 836 | static inline void arch_dup_mmap(struct mm_struct *oldmm, |
| 837 | struct mm_struct *mm) |
| 838 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 839 | PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm); |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 840 | } |
| 841 | |
| 842 | static inline void arch_exit_mmap(struct mm_struct *mm) |
| 843 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 844 | PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm); |
Jeremy Fitzhardinge | d6dd61c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 845 | } |
| 846 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 847 | static inline void __flush_tlb(void) |
| 848 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 849 | PVOP_VCALL0(pv_mmu_ops.flush_tlb_user); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 850 | } |
| 851 | static inline void __flush_tlb_global(void) |
| 852 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 853 | PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 854 | } |
| 855 | static inline void __flush_tlb_single(unsigned long addr) |
| 856 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 857 | PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 858 | } |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 859 | |
Jeremy Fitzhardinge | d4c1047 | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 860 | static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, |
| 861 | unsigned long va) |
| 862 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 863 | PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va); |
Jeremy Fitzhardinge | d4c1047 | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 864 | } |
| 865 | |
Jeremy Fitzhardinge | fdb4c33 | 2007-07-17 18:37:03 -0700 | [diff] [blame] | 866 | static inline void paravirt_alloc_pt(struct mm_struct *mm, unsigned pfn) |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 867 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 868 | PVOP_VCALL2(pv_mmu_ops.alloc_pt, mm, pfn); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 869 | } |
| 870 | static inline void paravirt_release_pt(unsigned pfn) |
| 871 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 872 | PVOP_VCALL1(pv_mmu_ops.release_pt, pfn); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 873 | } |
Zachary Amsden | c119ecc | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 874 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 875 | static inline void paravirt_alloc_pd(unsigned pfn) |
| 876 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 877 | PVOP_VCALL1(pv_mmu_ops.alloc_pd, pfn); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 878 | } |
| 879 | |
| 880 | static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn, |
| 881 | unsigned start, unsigned count) |
| 882 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 883 | PVOP_VCALL4(pv_mmu_ops.alloc_pd_clone, pfn, clonepfn, start, count); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 884 | } |
| 885 | static inline void paravirt_release_pd(unsigned pfn) |
| 886 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 887 | PVOP_VCALL1(pv_mmu_ops.release_pd, pfn); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 888 | } |
| 889 | |
Jeremy Fitzhardinge | ce6234b | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 890 | #ifdef CONFIG_HIGHPTE |
| 891 | static inline void *kmap_atomic_pte(struct page *page, enum km_type type) |
| 892 | { |
| 893 | unsigned long ret; |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 894 | ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type); |
Jeremy Fitzhardinge | ce6234b | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 895 | return (void *)ret; |
| 896 | } |
| 897 | #endif |
| 898 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 899 | static inline void pte_update(struct mm_struct *mm, unsigned long addr, |
| 900 | pte_t *ptep) |
| 901 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 902 | PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 903 | } |
| 904 | |
| 905 | static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr, |
| 906 | pte_t *ptep) |
| 907 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 908 | PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 909 | } |
| 910 | |
| 911 | #ifdef CONFIG_X86_PAE |
| 912 | static inline pte_t __pte(unsigned long long val) |
| 913 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 914 | unsigned long long ret = PVOP_CALL2(unsigned long long, |
| 915 | pv_mmu_ops.make_pte, |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 916 | val, val >> 32); |
| 917 | return (pte_t) { ret, ret >> 32 }; |
| 918 | } |
| 919 | |
| 920 | static inline pmd_t __pmd(unsigned long long val) |
| 921 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 922 | return (pmd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pmd, |
| 923 | val, val >> 32) }; |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 924 | } |
| 925 | |
| 926 | static inline pgd_t __pgd(unsigned long long val) |
| 927 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 928 | return (pgd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pgd, |
| 929 | val, val >> 32) }; |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 930 | } |
| 931 | |
| 932 | static inline unsigned long long pte_val(pte_t x) |
| 933 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 934 | return PVOP_CALL2(unsigned long long, pv_mmu_ops.pte_val, |
| 935 | x.pte_low, x.pte_high); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 936 | } |
| 937 | |
| 938 | static inline unsigned long long pmd_val(pmd_t x) |
| 939 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 940 | return PVOP_CALL2(unsigned long long, pv_mmu_ops.pmd_val, |
| 941 | x.pmd, x.pmd >> 32); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 942 | } |
| 943 | |
| 944 | static inline unsigned long long pgd_val(pgd_t x) |
| 945 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 946 | return PVOP_CALL2(unsigned long long, pv_mmu_ops.pgd_val, |
| 947 | x.pgd, x.pgd >> 32); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 948 | } |
Zachary Amsden | c119ecc | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 949 | |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 950 | static inline void set_pte(pte_t *ptep, pte_t pteval) |
| 951 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 952 | PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, pteval.pte_low, pteval.pte_high); |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 953 | } |
| 954 | |
Jeremy Fitzhardinge | 3dc494e | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 955 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
| 956 | pte_t *ptep, pte_t pteval) |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 957 | { |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 958 | /* 5 arg words */ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 959 | pv_mmu_ops.set_pte_at(mm, addr, ptep, pteval); |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 960 | } |
| 961 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 962 | static inline void set_pte_atomic(pte_t *ptep, pte_t pteval) |
| 963 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 964 | PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, |
| 965 | pteval.pte_low, pteval.pte_high); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 966 | } |
| 967 | |
| 968 | static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, |
| 969 | pte_t *ptep, pte_t pte) |
| 970 | { |
| 971 | /* 5 arg words */ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 972 | pv_mmu_ops.set_pte_present(mm, addr, ptep, pte); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 973 | } |
| 974 | |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 975 | static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) |
| 976 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 977 | PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, |
| 978 | pmdval.pmd, pmdval.pmd >> 32); |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 979 | } |
| 980 | |
| 981 | static inline void set_pud(pud_t *pudp, pud_t pudval) |
| 982 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 983 | PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, |
| 984 | pudval.pgd.pgd, pudval.pgd.pgd >> 32); |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 985 | } |
| 986 | |
| 987 | static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
| 988 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 989 | PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 990 | } |
| 991 | |
| 992 | static inline void pmd_clear(pmd_t *pmdp) |
| 993 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 994 | PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 995 | } |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 996 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 997 | #else /* !CONFIG_X86_PAE */ |
Jeremy Fitzhardinge | 4cdd9c8 | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 998 | |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 999 | static inline pte_t __pte(unsigned long val) |
| 1000 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1001 | return (pte_t) { PVOP_CALL1(unsigned long, pv_mmu_ops.make_pte, val) }; |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1002 | } |
| 1003 | |
| 1004 | static inline pgd_t __pgd(unsigned long val) |
| 1005 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1006 | return (pgd_t) { PVOP_CALL1(unsigned long, pv_mmu_ops.make_pgd, val) }; |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1007 | } |
| 1008 | |
| 1009 | static inline unsigned long pte_val(pte_t x) |
| 1010 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1011 | return PVOP_CALL1(unsigned long, pv_mmu_ops.pte_val, x.pte_low); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1012 | } |
| 1013 | |
| 1014 | static inline unsigned long pgd_val(pgd_t x) |
| 1015 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1016 | return PVOP_CALL1(unsigned long, pv_mmu_ops.pgd_val, x.pgd); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1017 | } |
| 1018 | |
| 1019 | static inline void set_pte(pte_t *ptep, pte_t pteval) |
| 1020 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1021 | PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, pteval.pte_low); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1022 | } |
| 1023 | |
| 1024 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
| 1025 | pte_t *ptep, pte_t pteval) |
| 1026 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1027 | PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pteval.pte_low); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1028 | } |
| 1029 | |
| 1030 | static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) |
| 1031 | { |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1032 | PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, pmdval.pud.pgd.pgd); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1033 | } |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1034 | #endif /* CONFIG_X86_PAE */ |
Rusty Russell | da181a8 | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1035 | |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1036 | /* Lazy mode for batching updates / context switch */ |
| 1037 | enum paravirt_lazy_mode { |
| 1038 | PARAVIRT_LAZY_NONE, |
| 1039 | PARAVIRT_LAZY_MMU, |
| 1040 | PARAVIRT_LAZY_CPU, |
| 1041 | }; |
| 1042 | |
| 1043 | enum paravirt_lazy_mode paravirt_get_lazy_mode(void); |
| 1044 | void paravirt_enter_lazy_cpu(void); |
| 1045 | void paravirt_leave_lazy_cpu(void); |
| 1046 | void paravirt_enter_lazy_mmu(void); |
| 1047 | void paravirt_leave_lazy_mmu(void); |
| 1048 | void paravirt_leave_lazy(enum paravirt_lazy_mode mode); |
| 1049 | |
Zachary Amsden | 9226d12 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 1050 | #define __HAVE_ARCH_ENTER_LAZY_CPU_MODE |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1051 | static inline void arch_enter_lazy_cpu_mode(void) |
| 1052 | { |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1053 | PVOP_VCALL0(pv_cpu_ops.lazy_mode.enter); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1054 | } |
| 1055 | |
| 1056 | static inline void arch_leave_lazy_cpu_mode(void) |
| 1057 | { |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1058 | PVOP_VCALL0(pv_cpu_ops.lazy_mode.leave); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1059 | } |
| 1060 | |
| 1061 | static inline void arch_flush_lazy_cpu_mode(void) |
| 1062 | { |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1063 | if (unlikely(paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU)) { |
| 1064 | arch_leave_lazy_cpu_mode(); |
| 1065 | arch_enter_lazy_cpu_mode(); |
| 1066 | } |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1067 | } |
| 1068 | |
Zachary Amsden | 9226d12 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 1069 | |
| 1070 | #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1071 | static inline void arch_enter_lazy_mmu_mode(void) |
| 1072 | { |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1073 | PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1074 | } |
| 1075 | |
| 1076 | static inline void arch_leave_lazy_mmu_mode(void) |
| 1077 | { |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1078 | PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave); |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1079 | } |
| 1080 | |
| 1081 | static inline void arch_flush_lazy_mmu_mode(void) |
| 1082 | { |
Jeremy Fitzhardinge | 8965c1c | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1083 | if (unlikely(paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU)) { |
| 1084 | arch_leave_lazy_mmu_mode(); |
| 1085 | arch_enter_lazy_mmu_mode(); |
| 1086 | } |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1087 | } |
Zachary Amsden | 9226d12 | 2007-02-13 13:26:21 +0100 | [diff] [blame] | 1088 | |
Jeremy Fitzhardinge | 4587623 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 1089 | void _paravirt_nop(void); |
| 1090 | #define paravirt_nop ((void *)_paravirt_nop) |
| 1091 | |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1092 | /* These all sit in the .parainstructions section to tell us what to patch. */ |
Jeremy Fitzhardinge | 98de032 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1093 | struct paravirt_patch_site { |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1094 | u8 *instr; /* original instructions */ |
| 1095 | u8 instrtype; /* type of this instruction */ |
| 1096 | u8 len; /* length of original instruction */ |
| 1097 | u16 clobbers; /* what registers you may clobber */ |
| 1098 | }; |
| 1099 | |
Jeremy Fitzhardinge | 98de032 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1100 | extern struct paravirt_patch_site __parainstructions[], |
| 1101 | __parainstructions_end[]; |
| 1102 | |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1103 | #ifdef CONFIG_X86_32 |
| 1104 | #define PV_SAVE_REGS "pushl %%ecx; pushl %%edx;" |
| 1105 | #define PV_RESTORE_REGS "popl %%edx; popl %%ecx" |
| 1106 | #define PV_FLAGS_ARG "0" |
| 1107 | #define PV_EXTRA_CLOBBERS |
| 1108 | #define PV_VEXTRA_CLOBBERS |
| 1109 | #else |
| 1110 | /* We save some registers, but all of them, that's too much. We clobber all |
| 1111 | * caller saved registers but the argument parameter */ |
| 1112 | #define PV_SAVE_REGS "pushq %%rdi;" |
| 1113 | #define PV_RESTORE_REGS "popq %%rdi;" |
| 1114 | #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx" |
| 1115 | #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx" |
| 1116 | #define PV_FLAGS_ARG "D" |
| 1117 | #endif |
| 1118 | |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1119 | static inline unsigned long __raw_local_save_flags(void) |
| 1120 | { |
| 1121 | unsigned long f; |
| 1122 | |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1123 | asm volatile(paravirt_alt(PV_SAVE_REGS |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1124 | PARAVIRT_CALL |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1125 | PV_RESTORE_REGS) |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1126 | : "=a"(f) |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1127 | : paravirt_type(pv_irq_ops.save_fl), |
Jeremy Fitzhardinge | 42c24fa | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1128 | paravirt_clobber(CLBR_EAX) |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1129 | : "memory", "cc" PV_VEXTRA_CLOBBERS); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1130 | return f; |
| 1131 | } |
| 1132 | |
| 1133 | static inline void raw_local_irq_restore(unsigned long f) |
| 1134 | { |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1135 | asm volatile(paravirt_alt(PV_SAVE_REGS |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1136 | PARAVIRT_CALL |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1137 | PV_RESTORE_REGS) |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1138 | : "=a"(f) |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1139 | : PV_FLAGS_ARG(f), |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1140 | paravirt_type(pv_irq_ops.restore_fl), |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1141 | paravirt_clobber(CLBR_EAX) |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1142 | : "memory", "cc" PV_EXTRA_CLOBBERS); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1143 | } |
| 1144 | |
| 1145 | static inline void raw_local_irq_disable(void) |
| 1146 | { |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1147 | asm volatile(paravirt_alt(PV_SAVE_REGS |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1148 | PARAVIRT_CALL |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1149 | PV_RESTORE_REGS) |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1150 | : |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1151 | : paravirt_type(pv_irq_ops.irq_disable), |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1152 | paravirt_clobber(CLBR_EAX) |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1153 | : "memory", "eax", "cc" PV_EXTRA_CLOBBERS); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1154 | } |
| 1155 | |
| 1156 | static inline void raw_local_irq_enable(void) |
| 1157 | { |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1158 | asm volatile(paravirt_alt(PV_SAVE_REGS |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1159 | PARAVIRT_CALL |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1160 | PV_RESTORE_REGS) |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1161 | : |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1162 | : paravirt_type(pv_irq_ops.irq_enable), |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1163 | paravirt_clobber(CLBR_EAX) |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1164 | : "memory", "eax", "cc" PV_EXTRA_CLOBBERS); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1165 | } |
| 1166 | |
| 1167 | static inline unsigned long __raw_local_irq_save(void) |
| 1168 | { |
| 1169 | unsigned long f; |
| 1170 | |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1171 | f = __raw_local_save_flags(); |
| 1172 | raw_local_irq_disable(); |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1173 | return f; |
| 1174 | } |
| 1175 | |
Jeremy Fitzhardinge | 294688c | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1176 | /* Make sure as little as possible of this mess escapes. */ |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1177 | #undef PARAVIRT_CALL |
Jeremy Fitzhardinge | 1a45b7a | 2007-05-02 19:27:15 +0200 | [diff] [blame] | 1178 | #undef __PVOP_CALL |
| 1179 | #undef __PVOP_VCALL |
Jeremy Fitzhardinge | f8822f4 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1180 | #undef PVOP_VCALL0 |
| 1181 | #undef PVOP_CALL0 |
| 1182 | #undef PVOP_VCALL1 |
| 1183 | #undef PVOP_CALL1 |
| 1184 | #undef PVOP_VCALL2 |
| 1185 | #undef PVOP_CALL2 |
| 1186 | #undef PVOP_VCALL3 |
| 1187 | #undef PVOP_CALL3 |
| 1188 | #undef PVOP_VCALL4 |
| 1189 | #undef PVOP_CALL4 |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1190 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 1191 | #else /* __ASSEMBLY__ */ |
| 1192 | |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1193 | #define _PVSITE(ptype, clobbers, ops, word, algn) \ |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1194 | 771:; \ |
| 1195 | ops; \ |
| 1196 | 772:; \ |
| 1197 | .pushsection .parainstructions,"a"; \ |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1198 | .align algn; \ |
| 1199 | word 771b; \ |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1200 | .byte ptype; \ |
| 1201 | .byte 772b-771b; \ |
| 1202 | .short clobbers; \ |
| 1203 | .popsection |
| 1204 | |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1205 | |
| 1206 | #ifdef CONFIG_X86_64 |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1207 | #define PV_SAVE_REGS pushq %rax; pushq %rdi; pushq %rcx; pushq %rdx |
| 1208 | #define PV_RESTORE_REGS popq %rdx; popq %rcx; popq %rdi; popq %rax |
| 1209 | #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8) |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1210 | #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8) |
| 1211 | #else |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1212 | #define PV_SAVE_REGS pushl %eax; pushl %edi; pushl %ecx; pushl %edx |
| 1213 | #define PV_RESTORE_REGS popl %edx; popl %ecx; popl %edi; popl %eax |
| 1214 | #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4) |
Glauber de Oliveira Costa | 658be9d | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1215 | #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4) |
| 1216 | #endif |
| 1217 | |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1218 | #define INTERRUPT_RETURN \ |
| 1219 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ |
| 1220 | jmp *%cs:pv_cpu_ops+PV_CPU_iret) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1221 | |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1222 | #define DISABLE_INTERRUPTS(clobbers) \ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1223 | PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1224 | PV_SAVE_REGS; \ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1225 | call *%cs:pv_irq_ops+PV_IRQ_irq_disable; \ |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1226 | PV_RESTORE_REGS;) \ |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1227 | |
Jeremy Fitzhardinge | d582203 | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1228 | #define ENABLE_INTERRUPTS(clobbers) \ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1229 | PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \ |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1230 | PV_SAVE_REGS; \ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1231 | call *%cs:pv_irq_ops+PV_IRQ_irq_enable; \ |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1232 | PV_RESTORE_REGS;) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1233 | |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 1234 | #define ENABLE_INTERRUPTS_SYSCALL_RET \ |
| 1235 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_syscall_ret),\ |
| 1236 | CLBR_NONE, \ |
| 1237 | jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_syscall_ret) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1238 | |
Glauber de Oliveira Costa | 2e47d3e | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1239 | |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1240 | #ifdef CONFIG_X86_32 |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1241 | #define GET_CR0_INTO_EAX \ |
Jeremy Fitzhardinge | 42c24fa | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1242 | push %ecx; push %edx; \ |
Jeremy Fitzhardinge | 93b1eab | 2007-10-16 11:51:29 -0700 | [diff] [blame] | 1243 | call *pv_cpu_ops+PV_CPU_read_cr0; \ |
Jeremy Fitzhardinge | 42c24fa | 2007-05-02 19:27:14 +0200 | [diff] [blame] | 1244 | pop %edx; pop %ecx |
Glauber de Oliveira Costa | 4a8c4c4 | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1245 | #else |
Glauber de Oliveira Costa | e801f86 | 2008-01-30 13:32:08 +0100 | [diff] [blame] | 1246 | #define SWAPGS \ |
| 1247 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ |
| 1248 | PV_SAVE_REGS; \ |
| 1249 | call *pv_cpu_ops+PV_CPU_swapgs; \ |
| 1250 | PV_RESTORE_REGS \ |
| 1251 | ) |
| 1252 | |
Glauber de Oliveira Costa | 4a8c4c4 | 2008-01-30 13:32:07 +0100 | [diff] [blame] | 1253 | #define GET_CR2_INTO_RCX \ |
| 1254 | call *pv_mmu_ops+PV_MMU_read_cr2; \ |
| 1255 | movq %rax, %rcx; \ |
| 1256 | xorq %rax, %rax; |
| 1257 | |
Glauber de Oliveira Costa | 6057fc8 | 2008-01-30 13:32:06 +0100 | [diff] [blame] | 1258 | #endif |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 1259 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 1260 | #endif /* __ASSEMBLY__ */ |
| 1261 | #endif /* CONFIG_PARAVIRT */ |
| 1262 | #endif /* __ASM_PARAVIRT_H */ |