Tejun Heo | 62fde54 | 2014-06-17 19:12:34 -0400 | [diff] [blame] | 1 | /* |
| 2 | * linux/percpu-defs.h - basic definitions for percpu areas |
| 3 | * |
| 4 | * DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER. |
| 5 | * |
| 6 | * This file is separate from linux/percpu.h to avoid cyclic inclusion |
| 7 | * dependency from arch header files. Only to be included from |
| 8 | * asm/percpu.h. |
| 9 | * |
| 10 | * This file includes macros necessary to declare percpu sections and |
| 11 | * variables, and definitions of percpu accessors and operations. It |
| 12 | * should provide enough percpu features to arch header files even when |
| 13 | * they can only include asm/percpu.h to avoid cyclic inclusion dependency. |
| 14 | */ |
| 15 | |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 16 | #ifndef _LINUX_PERCPU_DEFS_H |
| 17 | #define _LINUX_PERCPU_DEFS_H |
| 18 | |
Tejun Heo | 62fde54 | 2014-06-17 19:12:34 -0400 | [diff] [blame] | 19 | #ifdef CONFIG_SMP |
| 20 | |
| 21 | #ifdef MODULE |
| 22 | #define PER_CPU_SHARED_ALIGNED_SECTION "" |
| 23 | #define PER_CPU_ALIGNED_SECTION "" |
| 24 | #else |
| 25 | #define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned" |
| 26 | #define PER_CPU_ALIGNED_SECTION "..shared_aligned" |
| 27 | #endif |
| 28 | #define PER_CPU_FIRST_SECTION "..first" |
| 29 | |
| 30 | #else |
| 31 | |
| 32 | #define PER_CPU_SHARED_ALIGNED_SECTION "" |
| 33 | #define PER_CPU_ALIGNED_SECTION "..shared_aligned" |
| 34 | #define PER_CPU_FIRST_SECTION "" |
| 35 | |
| 36 | #endif |
| 37 | |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 38 | /* |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 39 | * Base implementations of per-CPU variable declarations and definitions, where |
| 40 | * the section in which the variable is to be placed is provided by the |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 41 | * 'sec' argument. This may be used to affect the parameters governing the |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 42 | * variable's storage. |
| 43 | * |
| 44 | * NOTE! The sections for the DECLARE and for the DEFINE must match, lest |
| 45 | * linkage errors occur due the compiler generating the wrong code to access |
| 46 | * that section. |
| 47 | */ |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 48 | #define __PCPU_ATTRS(sec) \ |
Rusty Russell | e0fdb0e | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 49 | __percpu __attribute__((section(PER_CPU_BASE_SECTION sec))) \ |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 50 | PER_CPU_ATTRIBUTES |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 51 | |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 52 | #define __PCPU_DUMMY_ATTRS \ |
| 53 | __attribute__((section(".discard"), unused)) |
| 54 | |
| 55 | /* |
Tejun Heo | 545695f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 56 | * Macro which verifies @ptr is a percpu pointer without evaluating |
| 57 | * @ptr. This is to be used in percpu accessors to verify that the |
| 58 | * input parameter is a percpu pointer. |
Christoph Lameter | abec1a8 | 2013-08-23 19:01:58 +0000 | [diff] [blame] | 59 | * |
| 60 | * + 0 is required in order to convert the pointer type from a |
| 61 | * potential array type to a pointer to a single item of the array. |
Tejun Heo | 545695f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 62 | */ |
| 63 | #define __verify_pcpu_ptr(ptr) do { \ |
Christoph Lameter | abec1a8 | 2013-08-23 19:01:58 +0000 | [diff] [blame] | 64 | const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \ |
Tejun Heo | 545695f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 65 | (void)__vpp_verify; \ |
| 66 | } while (0) |
| 67 | |
| 68 | /* |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 69 | * s390 and alpha modules require percpu variables to be defined as |
| 70 | * weak to force the compiler to generate GOT based external |
| 71 | * references for them. This is necessary because percpu sections |
| 72 | * will be located outside of the usually addressable area. |
| 73 | * |
| 74 | * This definition puts the following two extra restrictions when |
| 75 | * defining percpu variables. |
| 76 | * |
| 77 | * 1. The symbol must be globally unique, even the static ones. |
| 78 | * 2. Static percpu variables cannot be defined inside a function. |
| 79 | * |
| 80 | * Archs which need weak percpu definitions should define |
| 81 | * ARCH_NEEDS_WEAK_PER_CPU in asm/percpu.h when necessary. |
| 82 | * |
| 83 | * To ensure that the generic code observes the above two |
| 84 | * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak |
| 85 | * definition is used for all cases. |
| 86 | */ |
| 87 | #if defined(ARCH_NEEDS_WEAK_PER_CPU) || defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU) |
| 88 | /* |
| 89 | * __pcpu_scope_* dummy variable is used to enforce scope. It |
| 90 | * receives the static modifier when it's used in front of |
| 91 | * DEFINE_PER_CPU() and will trigger build failure if |
| 92 | * DECLARE_PER_CPU() is used for the same variable. |
| 93 | * |
| 94 | * __pcpu_unique_* dummy variable is used to enforce symbol uniqueness |
| 95 | * such that hidden weak symbol collision, which will cause unrelated |
| 96 | * variables to share the same address, can be detected during build. |
| 97 | */ |
| 98 | #define DECLARE_PER_CPU_SECTION(type, name, sec) \ |
| 99 | extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \ |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 100 | extern __PCPU_ATTRS(sec) __typeof__(type) name |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 101 | |
| 102 | #define DEFINE_PER_CPU_SECTION(type, name, sec) \ |
| 103 | __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \ |
Tejun Heo | 0f5e481 | 2009-10-29 22:34:12 +0900 | [diff] [blame] | 104 | extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 105 | __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ |
Tejun Heo | b1a0fbf | 2013-12-04 10:12:40 -0500 | [diff] [blame] | 106 | extern __PCPU_ATTRS(sec) __typeof__(type) name; \ |
Tejun Heo | c43768c | 2009-07-04 07:13:18 +0900 | [diff] [blame] | 107 | __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak \ |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 108 | __typeof__(type) name |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 109 | #else |
| 110 | /* |
| 111 | * Normal declaration and definition macros. |
| 112 | */ |
| 113 | #define DECLARE_PER_CPU_SECTION(type, name, sec) \ |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 114 | extern __PCPU_ATTRS(sec) __typeof__(type) name |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 115 | |
| 116 | #define DEFINE_PER_CPU_SECTION(type, name, sec) \ |
Tejun Heo | c43768c | 2009-07-04 07:13:18 +0900 | [diff] [blame] | 117 | __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES \ |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 118 | __typeof__(type) name |
Tejun Heo | 7c756e6 | 2009-06-24 15:13:50 +0900 | [diff] [blame] | 119 | #endif |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 120 | |
| 121 | /* |
| 122 | * Variant on the per-CPU variable declaration/definition theme used for |
| 123 | * ordinary per-CPU variables. |
| 124 | */ |
| 125 | #define DECLARE_PER_CPU(type, name) \ |
| 126 | DECLARE_PER_CPU_SECTION(type, name, "") |
| 127 | |
| 128 | #define DEFINE_PER_CPU(type, name) \ |
| 129 | DEFINE_PER_CPU_SECTION(type, name, "") |
| 130 | |
| 131 | /* |
| 132 | * Declaration/definition used for per-CPU variables that must come first in |
| 133 | * the set of variables. |
| 134 | */ |
| 135 | #define DECLARE_PER_CPU_FIRST(type, name) \ |
| 136 | DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION) |
| 137 | |
| 138 | #define DEFINE_PER_CPU_FIRST(type, name) \ |
| 139 | DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION) |
| 140 | |
| 141 | /* |
| 142 | * Declaration/definition used for per-CPU variables that must be cacheline |
| 143 | * aligned under SMP conditions so that, whilst a particular instance of the |
| 144 | * data corresponds to a particular CPU, inefficiencies due to direct access by |
| 145 | * other CPUs are reduced by preventing the data from unnecessarily spanning |
| 146 | * cachelines. |
| 147 | * |
| 148 | * An example of this would be statistical data, where each CPU's set of data |
| 149 | * is updated by that CPU alone, but the data from across all CPUs is collated |
| 150 | * by a CPU processing a read from a proc file. |
| 151 | */ |
| 152 | #define DECLARE_PER_CPU_SHARED_ALIGNED(type, name) \ |
| 153 | DECLARE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \ |
| 154 | ____cacheline_aligned_in_smp |
| 155 | |
| 156 | #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ |
| 157 | DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \ |
| 158 | ____cacheline_aligned_in_smp |
| 159 | |
Jeremy Fitzhardinge | 53f8245 | 2009-09-03 14:31:44 -0700 | [diff] [blame] | 160 | #define DECLARE_PER_CPU_ALIGNED(type, name) \ |
| 161 | DECLARE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION) \ |
| 162 | ____cacheline_aligned |
| 163 | |
| 164 | #define DEFINE_PER_CPU_ALIGNED(type, name) \ |
| 165 | DEFINE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION) \ |
| 166 | ____cacheline_aligned |
| 167 | |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 168 | /* |
| 169 | * Declaration/definition used for per-CPU variables that must be page aligned. |
| 170 | */ |
Tejun Heo | 3e352aa | 2009-08-03 14:10:11 +0900 | [diff] [blame] | 171 | #define DECLARE_PER_CPU_PAGE_ALIGNED(type, name) \ |
Denys Vlasenko | 3d9a854 | 2010-02-20 01:03:43 +0100 | [diff] [blame] | 172 | DECLARE_PER_CPU_SECTION(type, name, "..page_aligned") \ |
Tejun Heo | 3e352aa | 2009-08-03 14:10:11 +0900 | [diff] [blame] | 173 | __aligned(PAGE_SIZE) |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 174 | |
| 175 | #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ |
Denys Vlasenko | 3d9a854 | 2010-02-20 01:03:43 +0100 | [diff] [blame] | 176 | DEFINE_PER_CPU_SECTION(type, name, "..page_aligned") \ |
Tejun Heo | 3e352aa | 2009-08-03 14:10:11 +0900 | [diff] [blame] | 177 | __aligned(PAGE_SIZE) |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 178 | |
| 179 | /* |
Shaohua Li | c957ef2 | 2010-10-20 11:07:02 +0800 | [diff] [blame] | 180 | * Declaration/definition used for per-CPU variables that must be read mostly. |
| 181 | */ |
| 182 | #define DECLARE_PER_CPU_READ_MOSTLY(type, name) \ |
| 183 | DECLARE_PER_CPU_SECTION(type, name, "..readmostly") |
| 184 | |
| 185 | #define DEFINE_PER_CPU_READ_MOSTLY(type, name) \ |
| 186 | DEFINE_PER_CPU_SECTION(type, name, "..readmostly") |
| 187 | |
| 188 | /* |
Tejun Heo | 545695f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 189 | * Intermodule exports for per-CPU variables. sparse forgets about |
| 190 | * address space across EXPORT_SYMBOL(), change EXPORT_SYMBOL() to |
| 191 | * noop if __CHECKER__. |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 192 | */ |
Tejun Heo | 545695f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 193 | #ifndef __CHECKER__ |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 194 | #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var) |
| 195 | #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var) |
Tejun Heo | 545695f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 196 | #else |
| 197 | #define EXPORT_PER_CPU_SYMBOL(var) |
| 198 | #define EXPORT_PER_CPU_SYMBOL_GPL(var) |
| 199 | #endif |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 200 | |
Tejun Heo | 62fde54 | 2014-06-17 19:12:34 -0400 | [diff] [blame] | 201 | /* |
| 202 | * Accessors and operations. |
| 203 | */ |
| 204 | #ifndef __ASSEMBLY__ |
| 205 | |
| 206 | #ifdef CONFIG_SMP |
| 207 | |
| 208 | /* |
| 209 | * Add an offset to a pointer but keep the pointer as-is. Use RELOC_HIDE() |
| 210 | * to prevent the compiler from making incorrect assumptions about the |
| 211 | * pointer value. The weird cast keeps both GCC and sparse happy. |
| 212 | */ |
| 213 | #define SHIFT_PERCPU_PTR(__p, __offset) ({ \ |
| 214 | __verify_pcpu_ptr((__p)); \ |
| 215 | RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \ |
| 216 | }) |
| 217 | |
| 218 | /* |
| 219 | * A percpu variable may point to a discarded regions. The following are |
| 220 | * established ways to produce a usable pointer from the percpu variable |
| 221 | * offset. |
| 222 | */ |
| 223 | #define per_cpu(var, cpu) \ |
| 224 | (*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu))) |
| 225 | |
| 226 | #define raw_cpu_ptr(ptr) arch_raw_cpu_ptr(ptr) |
| 227 | |
| 228 | #ifdef CONFIG_DEBUG_PREEMPT |
| 229 | #define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) |
| 230 | #else |
| 231 | #define this_cpu_ptr(ptr) raw_cpu_ptr(ptr) |
| 232 | #endif |
| 233 | |
| 234 | #define __get_cpu_var(var) (*this_cpu_ptr(&(var))) |
| 235 | #define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var))) |
| 236 | |
| 237 | #else /* CONFIG_SMP */ |
| 238 | |
| 239 | #define VERIFY_PERCPU_PTR(__p) ({ \ |
| 240 | __verify_pcpu_ptr((__p)); \ |
| 241 | (typeof(*(__p)) __kernel __force *)(__p); \ |
| 242 | }) |
| 243 | |
| 244 | #define per_cpu(var, cpu) (*((void)(cpu), VERIFY_PERCPU_PTR(&(var)))) |
| 245 | #define __get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var))) |
| 246 | #define __raw_get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var))) |
| 247 | #define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0) |
| 248 | #define raw_cpu_ptr(ptr) this_cpu_ptr(ptr) |
| 249 | |
| 250 | #endif /* CONFIG_SMP */ |
| 251 | |
| 252 | /* keep until we have removed all uses of __this_cpu_ptr */ |
| 253 | #define __this_cpu_ptr(ptr) raw_cpu_ptr(ptr) |
| 254 | |
Tejun Heo | 9defda1 | 2014-06-17 19:12:34 -0400 | [diff] [blame^] | 255 | /* |
| 256 | * Must be an lvalue. Since @var must be a simple identifier, |
| 257 | * we force a syntax error here if it isn't. |
| 258 | */ |
| 259 | #define get_cpu_var(var) (*({ \ |
| 260 | preempt_disable(); \ |
| 261 | this_cpu_ptr(&var); })) |
| 262 | |
| 263 | /* |
| 264 | * The weird & is necessary because sparse considers (void)(var) to be |
| 265 | * a direct dereference of percpu variable (var). |
| 266 | */ |
| 267 | #define put_cpu_var(var) do { \ |
| 268 | (void)&(var); \ |
| 269 | preempt_enable(); \ |
| 270 | } while (0) |
| 271 | |
| 272 | #define get_cpu_ptr(var) ({ \ |
| 273 | preempt_disable(); \ |
| 274 | this_cpu_ptr(var); }) |
| 275 | |
| 276 | #define put_cpu_ptr(var) do { \ |
| 277 | (void)(var); \ |
| 278 | preempt_enable(); \ |
| 279 | } while (0) |
| 280 | |
| 281 | #ifdef CONFIG_SMP |
| 282 | #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) |
| 283 | #else |
| 284 | #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); }) |
| 285 | #endif |
| 286 | |
Tejun Heo | 62fde54 | 2014-06-17 19:12:34 -0400 | [diff] [blame] | 287 | #endif /* __ASSEMBLY__ */ |
David Howells | 5028eaa | 2009-04-21 23:00:29 +0100 | [diff] [blame] | 288 | #endif /* _LINUX_PERCPU_DEFS_H */ |