Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef __ASM_SYSTEM_H |
| 2 | #define __ASM_SYSTEM_H |
| 3 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | #include <linux/kernel.h> |
| 5 | #include <asm/segment.h> |
Jeff Dike | a436ed9 | 2007-05-08 00:35:02 -0700 | [diff] [blame] | 6 | #include <asm/cmpxchg.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | |
| 8 | #ifdef __KERNEL__ |
| 9 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | #define __SAVE(reg,offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t" |
| 11 | #define __RESTORE(reg,offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t" |
| 12 | |
| 13 | /* frame pointer must be last for get_wchan */ |
Andi Kleen | 658fdbe | 2006-09-26 10:52:41 +0200 | [diff] [blame] | 14 | #define SAVE_CONTEXT "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t" |
| 15 | #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | |
| 17 | #define __EXTRA_CLOBBER \ |
| 18 | ,"rcx","rbx","rdx","r8","r9","r10","r11","r12","r13","r14","r15" |
| 19 | |
Andi Kleen | 658fdbe | 2006-09-26 10:52:41 +0200 | [diff] [blame] | 20 | /* Save restore flags to clear handle leaking NT */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #define switch_to(prev,next,last) \ |
| 22 | asm volatile(SAVE_CONTEXT \ |
| 23 | "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \ |
| 24 | "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \ |
| 25 | "call __switch_to\n\t" \ |
| 26 | ".globl thread_return\n" \ |
| 27 | "thread_return:\n\t" \ |
| 28 | "movq %%gs:%P[pda_pcurrent],%%rsi\n\t" \ |
| 29 | "movq %P[thread_info](%%rsi),%%r8\n\t" \ |
Gerd Hoffmann | d167a51 | 2006-06-26 13:56:16 +0200 | [diff] [blame] | 30 | LOCK_PREFIX "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | "movq %%rax,%%rdi\n\t" \ |
| 32 | "jc ret_from_fork\n\t" \ |
| 33 | RESTORE_CONTEXT \ |
| 34 | : "=a" (last) \ |
| 35 | : [next] "S" (next), [prev] "D" (prev), \ |
| 36 | [threadrsp] "i" (offsetof(struct task_struct, thread.rsp)), \ |
| 37 | [ti_flags] "i" (offsetof(struct thread_info, flags)),\ |
| 38 | [tif_fork] "i" (TIF_FORK), \ |
Roman Zippel | f7e4217 | 2007-05-09 02:35:17 -0700 | [diff] [blame] | 39 | [thread_info] "i" (offsetof(struct task_struct, stack)), \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 40 | [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent)) \ |
| 41 | : "memory", "cc" __EXTRA_CLOBBER) |
| 42 | |
| 43 | extern void load_gs_index(unsigned); |
| 44 | |
| 45 | /* |
| 46 | * Load a segment. Fall back on loading the zero |
| 47 | * segment if something goes wrong.. |
| 48 | */ |
| 49 | #define loadsegment(seg,value) \ |
| 50 | asm volatile("\n" \ |
| 51 | "1:\t" \ |
| 52 | "movl %k0,%%" #seg "\n" \ |
| 53 | "2:\n" \ |
| 54 | ".section .fixup,\"ax\"\n" \ |
| 55 | "3:\t" \ |
| 56 | "movl %1,%%" #seg "\n\t" \ |
| 57 | "jmp 2b\n" \ |
| 58 | ".previous\n" \ |
| 59 | ".section __ex_table,\"a\"\n\t" \ |
| 60 | ".align 8\n\t" \ |
| 61 | ".quad 1b,3b\n" \ |
| 62 | ".previous" \ |
| 63 | : :"r" (value), "r" (0)) |
| 64 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | /* |
| 66 | * Clear and set 'TS' bit respectively |
| 67 | */ |
| 68 | #define clts() __asm__ __volatile__ ("clts") |
| 69 | |
| 70 | static inline unsigned long read_cr0(void) |
| 71 | { |
| 72 | unsigned long cr0; |
| 73 | asm volatile("movq %%cr0,%0" : "=r" (cr0)); |
| 74 | return cr0; |
Glauber de Oliveira Costa | f51c945 | 2007-07-22 11:12:29 +0200 | [diff] [blame] | 75 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 76 | |
| 77 | static inline void write_cr0(unsigned long val) |
| 78 | { |
| 79 | asm volatile("movq %0,%%cr0" :: "r" (val)); |
Glauber de Oliveira Costa | f51c945 | 2007-07-22 11:12:29 +0200 | [diff] [blame] | 80 | } |
| 81 | |
| 82 | static inline unsigned long read_cr2(void) |
| 83 | { |
| 84 | unsigned long cr2; |
Kirill Korotaev | c1217a7 | 2007-10-17 18:04:33 +0200 | [diff] [blame] | 85 | asm volatile("movq %%cr2,%0" : "=r" (cr2)); |
Glauber de Oliveira Costa | f51c945 | 2007-07-22 11:12:29 +0200 | [diff] [blame] | 86 | return cr2; |
| 87 | } |
| 88 | |
| 89 | static inline void write_cr2(unsigned long val) |
| 90 | { |
| 91 | asm volatile("movq %0,%%cr2" :: "r" (val)); |
| 92 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | |
| 94 | static inline unsigned long read_cr3(void) |
| 95 | { |
| 96 | unsigned long cr3; |
Kirill Korotaev | c1217a7 | 2007-10-17 18:04:33 +0200 | [diff] [blame] | 97 | asm volatile("movq %%cr3,%0" : "=r" (cr3)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 98 | return cr3; |
Glauber de Oliveira Costa | f51c945 | 2007-07-22 11:12:29 +0200 | [diff] [blame] | 99 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | |
Glauber de Oliveira Costa | fbc16f2 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 101 | static inline void write_cr3(unsigned long val) |
| 102 | { |
| 103 | asm volatile("movq %0,%%cr3" :: "r" (val) : "memory"); |
| 104 | } |
| 105 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | static inline unsigned long read_cr4(void) |
| 107 | { |
| 108 | unsigned long cr4; |
Kirill Korotaev | c1217a7 | 2007-10-17 18:04:33 +0200 | [diff] [blame] | 109 | asm volatile("movq %%cr4,%0" : "=r" (cr4)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | return cr4; |
Glauber de Oliveira Costa | f51c945 | 2007-07-22 11:12:29 +0200 | [diff] [blame] | 111 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | |
| 113 | static inline void write_cr4(unsigned long val) |
| 114 | { |
Glauber de Oliveira Costa | fbc16f2 | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 115 | asm volatile("movq %0,%%cr4" :: "r" (val) : "memory"); |
Glauber de Oliveira Costa | f51c945 | 2007-07-22 11:12:29 +0200 | [diff] [blame] | 116 | } |
| 117 | |
| 118 | static inline unsigned long read_cr8(void) |
| 119 | { |
| 120 | unsigned long cr8; |
Kirill Korotaev | c1217a7 | 2007-10-17 18:04:33 +0200 | [diff] [blame] | 121 | asm volatile("movq %%cr8,%0" : "=r" (cr8)); |
Glauber de Oliveira Costa | f51c945 | 2007-07-22 11:12:29 +0200 | [diff] [blame] | 122 | return cr8; |
| 123 | } |
| 124 | |
| 125 | static inline void write_cr8(unsigned long val) |
| 126 | { |
| 127 | asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); |
| 128 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | |
| 130 | #define stts() write_cr0(8 | read_cr0()) |
| 131 | |
| 132 | #define wbinvd() \ |
Nick Piggin | 7113302 | 2007-07-21 04:37:17 -0700 | [diff] [blame] | 133 | __asm__ __volatile__ ("wbinvd": : :"memory") |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | |
| 135 | #endif /* __KERNEL__ */ |
| 136 | |
H. Peter Anvin | 6619a8f | 2007-10-17 18:04:37 +0200 | [diff] [blame] | 137 | static inline void clflush(volatile void *__p) |
| 138 | { |
| 139 | asm volatile("clflush %0" : "+m" (*(char __force *)__p)); |
| 140 | } |
| 141 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | #define nop() __asm__ __volatile__ ("nop") |
| 143 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 144 | #ifdef CONFIG_SMP |
| 145 | #define smp_mb() mb() |
Nick Piggin | b6c7347ff | 2007-10-13 03:07:38 +0200 | [diff] [blame] | 146 | #define smp_rmb() barrier() |
| 147 | #define smp_wmb() barrier() |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 148 | #define smp_read_barrier_depends() do {} while(0) |
| 149 | #else |
| 150 | #define smp_mb() barrier() |
| 151 | #define smp_rmb() barrier() |
| 152 | #define smp_wmb() barrier() |
| 153 | #define smp_read_barrier_depends() do {} while(0) |
| 154 | #endif |
| 155 | |
| 156 | |
| 157 | /* |
| 158 | * Force strict CPU ordering. |
| 159 | * And yes, this is required on UP too when we're talking |
| 160 | * to devices. |
| 161 | */ |
| 162 | #define mb() asm volatile("mfence":::"memory") |
| 163 | #define rmb() asm volatile("lfence":::"memory") |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 164 | #define wmb() asm volatile("sfence" ::: "memory") |
Nick Piggin | 4071c71 | 2007-10-13 03:06:55 +0200 | [diff] [blame] | 165 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | #define read_barrier_depends() do {} while(0) |
Takashi Iwai | 911b0ad | 2006-02-04 23:28:05 -0800 | [diff] [blame] | 167 | #define set_mb(var, value) do { (void) xchg(&var, value); } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 168 | |
| 169 | #define warn_if_not_ulong(x) do { unsigned long foo; (void) (&(x) == &foo); } while (0) |
| 170 | |
Ingo Molnar | 2601e64 | 2006-07-03 00:24:45 -0700 | [diff] [blame] | 171 | #include <linux/irqflags.h> |
Ravikiran G Thirumalai | 2ddb55f | 2006-01-17 07:03:47 +0100 | [diff] [blame] | 172 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | void cpu_idle_wait(void); |
| 174 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | extern unsigned long arch_align_stack(unsigned long sp); |
Gerd Hoffmann | d167a51 | 2006-06-26 13:56:16 +0200 | [diff] [blame] | 176 | extern void free_init_pages(char *what, unsigned long begin, unsigned long end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | |
| 178 | #endif |