blob: 80860afffbdbbede4092bb18a6946e5b0a2831f9 [file] [log] [blame]
Vegard Nossum77ef50a2008-06-18 17:08:48 +02001#ifndef ASM_X86__PDA_H
2#define ASM_X86__PDA_H
Linus Torvalds1da177e2005-04-16 15:20:36 -07003
4#ifndef __ASSEMBLY__
5#include <linux/stddef.h>
6#include <linux/types.h>
7#include <linux/cache.h>
Jan Beulichb556b352006-01-11 22:43:00 +01008#include <asm/page.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +010010/* Per processor datastructure. %gs points to it while the kernel runs */
Linus Torvalds1da177e2005-04-16 15:20:36 -070011struct x8664_pda {
Arjan van de Ven29a9af62006-09-26 10:52:38 +020012 struct task_struct *pcurrent; /* 0 Current process */
13 unsigned long data_offset; /* 8 Per cpu data offset from linker
14 address */
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +010015 unsigned long kernelstack; /* 16 top of kernel stack for current */
16 unsigned long oldrsp; /* 24 user rsp for system call */
17 int irqcount; /* 32 Irq nesting counter. Starts -1 */
18 unsigned int cpunumber; /* 36 Logical CPU number */
Arjan van de Ven0a425402006-09-26 10:52:38 +020019#ifdef CONFIG_CC_STACKPROTECTOR
20 unsigned long stack_canary; /* 40 stack canary value */
21 /* gcc-ABI: this canary MUST be at
22 offset 40!!! */
23#endif
24 char *irqstackptr;
Mike Travis3461b0a2008-05-12 21:21:13 +020025 short nodenumber; /* number of current node (32k max) */
26 short in_bootmem; /* pda lives in bootmem */
Linus Torvalds1da177e2005-04-16 15:20:36 -070027 unsigned int __softirq_pending;
28 unsigned int __nmi_count; /* number of NMI on this CPUs */
Andi Kleena15da492006-09-26 10:52:40 +020029 short mmu_state;
30 short isidle;
Arjan van de Vendf920042006-03-25 16:31:01 +010031 struct mm_struct *active_mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -070032 unsigned apic_timer_irqs;
Thomas Gleixner4e77ae32007-10-12 23:04:07 +020033 unsigned irq0_irqs;
Joe Korty38e760a2007-10-17 18:04:40 +020034 unsigned irq_resched_count;
35 unsigned irq_call_count;
36 unsigned irq_tlb_count;
37 unsigned irq_thermal_count;
38 unsigned irq_threshold_count;
39 unsigned irq_spurious_count;
Andi Kleenb9169112005-09-12 18:49:24 +020040} ____cacheline_aligned_in_smp;
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
Mike Travis3461b0a2008-05-12 21:21:13 +020042extern struct x8664_pda **_cpu_pda;
Thomas Gleixner40fec502008-01-30 13:30:18 +010043extern void pda_init(int);
Ravikiran G Thirumalaidf79efd2006-01-11 22:45:39 +010044
Ravikiran G Thirumalai365ba912006-01-11 22:45:42 +010045#define cpu_pda(i) (_cpu_pda[i])
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +010047/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 * There is no fast way to get the base address of the PDA, all the accesses
49 * have to mention %fs/%gs. So it needs to be done this Torvaldian way.
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +010050 */
Andi Kleenfd167e42006-09-26 10:52:40 +020051extern void __bad_pda_field(void) __attribute__((noreturn));
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
Andi Kleenc1a9d412006-09-26 10:52:40 +020053/*
54 * proxy_pda doesn't actually exist, but tell gcc it is accessed for
55 * all PDA accesses so it gets read/write dependencies right.
56 */
Andi Kleen53ee11a2006-09-26 10:52:38 +020057extern struct x8664_pda _proxy_pda;
58
Linus Torvalds1da177e2005-04-16 15:20:36 -070059#define pda_offset(field) offsetof(struct x8664_pda, field)
60
Joe Perches46e1abc2008-03-23 01:03:05 -070061#define pda_to_op(op, field, val) \
62do { \
63 typedef typeof(_proxy_pda.field) T__; \
64 if (0) { T__ tmp__; tmp__ = (val); } /* type checking */ \
65 switch (sizeof(_proxy_pda.field)) { \
66 case 2: \
67 asm(op "w %1,%%gs:%c2" : \
68 "+m" (_proxy_pda.field) : \
69 "ri" ((T__)val), \
70 "i"(pda_offset(field))); \
71 break; \
72 case 4: \
73 asm(op "l %1,%%gs:%c2" : \
74 "+m" (_proxy_pda.field) : \
75 "ri" ((T__)val), \
76 "i" (pda_offset(field))); \
77 break; \
78 case 8: \
79 asm(op "q %1,%%gs:%c2": \
80 "+m" (_proxy_pda.field) : \
81 "ri" ((T__)val), \
82 "i"(pda_offset(field))); \
83 break; \
84 default: \
85 __bad_pda_field(); \
86 } \
87} while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Joe Perches46e1abc2008-03-23 01:03:05 -070089#define pda_from_op(op, field) \
90({ \
Andi Kleenc1a9d412006-09-26 10:52:40 +020091 typeof(_proxy_pda.field) ret__; \
92 switch (sizeof(_proxy_pda.field)) { \
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +010093 case 2: \
94 asm(op "w %%gs:%c1,%0" : \
Andi Kleenc1a9d412006-09-26 10:52:40 +020095 "=r" (ret__) : \
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +010096 "i" (pda_offset(field)), \
97 "m" (_proxy_pda.field)); \
Joe Perches46e1abc2008-03-23 01:03:05 -070098 break; \
Andi Kleenc1a9d412006-09-26 10:52:40 +020099 case 4: \
100 asm(op "l %%gs:%c1,%0": \
101 "=r" (ret__): \
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +0100102 "i" (pda_offset(field)), \
103 "m" (_proxy_pda.field)); \
Joe Perches46e1abc2008-03-23 01:03:05 -0700104 break; \
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +0100105 case 8: \
Andi Kleenc1a9d412006-09-26 10:52:40 +0200106 asm(op "q %%gs:%c1,%0": \
107 "=r" (ret__) : \
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +0100108 "i" (pda_offset(field)), \
109 "m" (_proxy_pda.field)); \
Joe Perches46e1abc2008-03-23 01:03:05 -0700110 break; \
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +0100111 default: \
Andi Kleenc1a9d412006-09-26 10:52:40 +0200112 __bad_pda_field(); \
Joe Perches46e1abc2008-03-23 01:03:05 -0700113 } \
114 ret__; \
115})
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116
Cyrill Gorcunovfe758fb2008-01-30 13:31:25 +0100117#define read_pda(field) pda_from_op("mov", field)
118#define write_pda(field, val) pda_to_op("mov", field, val)
119#define add_pda(field, val) pda_to_op("add", field, val)
120#define sub_pda(field, val) pda_to_op("sub", field, val)
121#define or_pda(field, val) pda_to_op("or", field, val)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122
Andi Kleen94468682006-11-14 16:57:46 +0100123/* This is not atomic against other CPUs -- CPU preemption needs to be off */
Joe Perches46e1abc2008-03-23 01:03:05 -0700124#define test_and_clear_bit_pda(bit, field) \
125({ \
126 int old__; \
127 asm volatile("btr %2,%%gs:%c3\n\tsbbl %0,%0" \
128 : "=r" (old__), "+m" (_proxy_pda.field) \
129 : "dIr" (bit), "i" (pda_offset(field)) : "memory");\
130 old__; \
Andi Kleen94468682006-11-14 16:57:46 +0100131})
132
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133#endif
134
135#define PDA_STACKOFFSET (5*8)
136
Vegard Nossum77ef50a2008-06-18 17:08:48 +0200137#endif /* ASM_X86__PDA_H */