blob: cf5efbcf716c8cecf74d4d315e2619f6fdcfa1f4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __LINUX_PERCPU_H
2#define __LINUX_PERCPU_H
Martin Peschke7ff6f082006-09-25 23:31:21 -07003
Robert P. J. Day0a3021f2007-07-15 23:39:57 -07004#include <linux/preempt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/slab.h> /* For kmalloc() */
6#include <linux/smp.h>
Martin Peschke7ff6f082006-09-25 23:31:21 -07007#include <linux/cpumask.h>
Tejun Heo6a242902009-03-06 14:33:58 +09008#include <linux/pfn.h>
Martin Peschke7ff6f082006-09-25 23:31:21 -07009
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <asm/percpu.h>
11
Tejun Heo6a242902009-03-06 14:33:58 +090012/* enough to cover all DEFINE_PER_CPUs in modules */
Jeremy Fitzhardingeb00742d2007-05-02 19:27:11 +020013#ifdef CONFIG_MODULES
Tejun Heo6a242902009-03-06 14:33:58 +090014#define PERCPU_MODULE_RESERVE (8 << 10)
Jeremy Fitzhardingeb00742d2007-05-02 19:27:11 +020015#else
Tejun Heo6a242902009-03-06 14:33:58 +090016#define PERCPU_MODULE_RESERVE 0
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#endif
18
Tejun Heo6a242902009-03-06 14:33:58 +090019#ifndef PERCPU_ENOUGH_ROOM
Jeremy Fitzhardingeb00742d2007-05-02 19:27:11 +020020#define PERCPU_ENOUGH_ROOM \
Tejun Heo6a242902009-03-06 14:33:58 +090021 (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
22 PERCPU_MODULE_RESERVE)
23#endif
Jeremy Fitzhardingeb00742d2007-05-02 19:27:11 +020024
Jan Blunck632bbfe2006-09-25 23:30:53 -070025/*
26 * Must be an lvalue. Since @var must be a simple identifier,
27 * we force a syntax error here if it isn't.
28 */
29#define get_cpu_var(var) (*({ \
Jan Bluncka666ecf2006-10-06 00:43:58 -070030 extern int simple_identifier_##var(void); \
Jan Blunck632bbfe2006-09-25 23:30:53 -070031 preempt_disable(); \
32 &__get_cpu_var(var); }))
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#define put_cpu_var(var) preempt_enable()
34
35#ifdef CONFIG_SMP
36
Tejun Heo8d408b42009-02-24 11:57:21 +090037/* minimum unit size, also is the maximum supported allocation size */
Tejun Heo6a242902009-03-06 14:33:58 +090038#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10)
Tejun Heo8d408b42009-02-24 11:57:21 +090039
40/*
41 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
Tejun Heo6b19b0c2009-03-06 14:33:59 +090042 * back on the first chunk for dynamic percpu allocation if arch is
43 * manually allocating and mapping it for faster access (as a part of
44 * large page mapping for example).
Tejun Heo8d408b42009-02-24 11:57:21 +090045 *
Tejun Heo6b19b0c2009-03-06 14:33:59 +090046 * The following values give between one and two pages of free space
47 * after typical minimal boot (2-way SMP, single disk and NIC) with
48 * both defconfig and a distro config on x86_64 and 32. More
49 * intelligent way to determine this would be nice.
Tejun Heo8d408b42009-02-24 11:57:21 +090050 */
Tejun Heo6b19b0c2009-03-06 14:33:59 +090051#if BITS_PER_LONG > 32
52#define PERCPU_DYNAMIC_RESERVE (20 << 10)
53#else
54#define PERCPU_DYNAMIC_RESERVE (12 << 10)
55#endif
Tejun Heo8d408b42009-02-24 11:57:21 +090056
Tejun Heofbf59bc2009-02-20 16:29:08 +090057extern void *pcpu_base_addr;
Tejun Heofb435d52009-08-14 15:00:51 +090058extern const unsigned long *pcpu_unit_offsets;
Tejun Heofbf59bc2009-02-20 16:29:08 +090059
Tejun Heofd1e8a12009-08-14 15:00:51 +090060struct pcpu_group_info {
61 int nr_units; /* aligned # of units */
62 unsigned long base_offset; /* base address offset */
63 unsigned int *cpu_map; /* unit->cpu map, empty
64 * entries contain NR_CPUS */
65};
66
67struct pcpu_alloc_info {
68 size_t static_size;
69 size_t reserved_size;
70 size_t dyn_size;
71 size_t unit_size;
72 size_t atom_size;
73 size_t alloc_size;
74 size_t __ai_size; /* internal, don't use */
75 int nr_groups; /* 0 if grouping unnecessary */
76 struct pcpu_group_info groups[];
77};
78
Tejun Heof58dc012009-08-14 15:00:50 +090079enum pcpu_fc {
80 PCPU_FC_AUTO,
81 PCPU_FC_EMBED,
82 PCPU_FC_PAGE,
Tejun Heof58dc012009-08-14 15:00:50 +090083
84 PCPU_FC_NR,
85};
86extern const char *pcpu_fc_names[PCPU_FC_NR];
87
88extern enum pcpu_fc pcpu_chosen_fc;
89
Tejun Heo3cbc8562009-08-14 15:00:50 +090090typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size,
91 size_t align);
Tejun Heod4b95f82009-07-04 08:10:59 +090092typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size);
93typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);
Tejun Heoa530b792009-07-04 08:11:00 +090094typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to);
Tejun Heofbf59bc2009-02-20 16:29:08 +090095
Tejun Heofd1e8a12009-08-14 15:00:51 +090096extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
97 int nr_units);
98extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai);
Tejun Heo033e48f2009-08-14 15:00:51 +090099
Tejun Heofd1e8a12009-08-14 15:00:51 +0900100extern struct pcpu_alloc_info * __init pcpu_build_alloc_info(
101 size_t reserved_size, ssize_t dyn_size,
102 size_t atom_size,
103 pcpu_fc_cpu_distance_fn_t cpu_distance_fn);
104
Tejun Heofb435d52009-08-14 15:00:51 +0900105extern int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
106 void *base_addr);
Tejun Heo8d408b42009-02-24 11:57:21 +0900107
Tejun Heo08fc4582009-08-14 15:00:49 +0900108#ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
Tejun Heoc8826dd2009-08-14 15:00:52 +0900109extern int __init pcpu_embed_first_chunk(size_t reserved_size, ssize_t dyn_size,
110 size_t atom_size,
111 pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
112 pcpu_fc_alloc_fn_t alloc_fn,
113 pcpu_fc_free_fn_t free_fn);
Tejun Heo08fc4582009-08-14 15:00:49 +0900114#endif
Tejun Heo66c3a752009-03-10 16:27:48 +0900115
Tejun Heo08fc4582009-08-14 15:00:49 +0900116#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
Tejun Heofb435d52009-08-14 15:00:51 +0900117extern int __init pcpu_page_first_chunk(size_t reserved_size,
Tejun Heod4b95f82009-07-04 08:10:59 +0900118 pcpu_fc_alloc_fn_t alloc_fn,
119 pcpu_fc_free_fn_t free_fn,
120 pcpu_fc_populate_pte_fn_t populate_pte_fn);
Tejun Heo08fc4582009-08-14 15:00:49 +0900121#endif
Tejun Heod4b95f82009-07-04 08:10:59 +0900122
Tejun Heofbf59bc2009-02-20 16:29:08 +0900123/*
124 * Use this to get to a cpu's version of the per-cpu object
125 * dynamically allocated. Non-atomic access to the current CPU's
126 * version should probably be combined with get_cpu()/put_cpu().
127 */
128#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
129
Tejun Heoedcb4632009-03-06 14:33:59 +0900130extern void *__alloc_reserved_percpu(size_t size, size_t align);
Tejun Heof2a82052009-02-20 16:29:08 +0900131extern void *__alloc_percpu(size_t size, size_t align);
132extern void free_percpu(void *__pdata);
Vivek Goyal3b034b02009-11-24 15:50:03 +0900133extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
Tejun Heof2a82052009-02-20 16:29:08 +0900134
Tejun Heoe74e3962009-03-30 19:07:44 +0900135#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
136extern void __init setup_per_cpu_areas(void);
137#endif
138
Tejun Heof2a82052009-02-20 16:29:08 +0900139#else /* CONFIG_SMP */
140
141#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
142
143static inline void *__alloc_percpu(size_t size, size_t align)
144{
145 /*
146 * Can't easily make larger alignment work with kmalloc. WARN
147 * on it. Larger alignment should only be used for module
148 * percpu sections on SMP for which this path isn't used.
149 */
Tejun Heoe3176032009-02-26 10:54:17 +0900150 WARN_ON_ONCE(align > SMP_CACHE_BYTES);
Ingo Molnard2b02612009-02-25 14:36:45 +0100151 return kzalloc(size, GFP_KERNEL);
Tejun Heof2a82052009-02-20 16:29:08 +0900152}
153
154static inline void free_percpu(void *p)
155{
156 kfree(p);
157}
158
Tejun Heoee0a6ef2009-12-02 08:36:58 +0900159static inline phys_addr_t per_cpu_ptr_to_phys(void *addr)
160{
161 return __pa(addr);
162}
163
Tejun Heoe74e3962009-03-30 19:07:44 +0900164static inline void __init setup_per_cpu_areas(void) { }
165
Tejun Heoa76761b2009-07-15 23:35:14 +0900166static inline void *pcpu_lpage_remapped(void *kaddr)
167{
168 return NULL;
169}
170
Tejun Heof2a82052009-02-20 16:29:08 +0900171#endif /* CONFIG_SMP */
172
Tejun Heo64ef2912009-10-29 22:34:12 +0900173#define alloc_percpu(type) \
174 (typeof(type) *)__alloc_percpu(sizeof(type), __alignof__(type))
Tejun Heof2a82052009-02-20 16:29:08 +0900175
Tejun Heo066123a2009-04-10 12:02:40 -0700176/*
177 * Optional methods for optimized non-lvalue per-cpu variable access.
178 *
179 * @var can be a percpu variable or a field of it and its size should
180 * equal char, int or long. percpu_read() evaluates to a lvalue and
181 * all others to void.
182 *
183 * These operations are guaranteed to be atomic w.r.t. preemption.
184 * The generic versions use plain get/put_cpu_var(). Archs are
185 * encouraged to implement single-instruction alternatives which don't
186 * require preemption protection.
187 */
188#ifndef percpu_read
189# define percpu_read(var) \
190 ({ \
191 typeof(per_cpu_var(var)) __tmp_var__; \
192 __tmp_var__ = get_cpu_var(var); \
193 put_cpu_var(var); \
194 __tmp_var__; \
195 })
196#endif
197
198#define __percpu_generic_to_op(var, val, op) \
199do { \
200 get_cpu_var(var) op val; \
201 put_cpu_var(var); \
202} while (0)
203
204#ifndef percpu_write
205# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
206#endif
207
208#ifndef percpu_add
209# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
210#endif
211
212#ifndef percpu_sub
213# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
214#endif
215
216#ifndef percpu_and
217# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
218#endif
219
220#ifndef percpu_or
221# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
222#endif
223
224#ifndef percpu_xor
225# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
226#endif
227
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900228/*
229 * Branching function to split up a function into a set of functions that
230 * are called for different scalar sizes of the objects handled.
231 */
232
233extern void __bad_size_call_parameter(void);
234
Tejun Heo0f5e4812009-10-29 22:34:12 +0900235#define __pcpu_size_call_return(stem, variable) \
236({ typeof(variable) pscr_ret__; \
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900237 switch(sizeof(variable)) { \
Tejun Heo0f5e4812009-10-29 22:34:12 +0900238 case 1: pscr_ret__ = stem##1(variable);break; \
239 case 2: pscr_ret__ = stem##2(variable);break; \
240 case 4: pscr_ret__ = stem##4(variable);break; \
241 case 8: pscr_ret__ = stem##8(variable);break; \
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900242 default: \
243 __bad_size_call_parameter();break; \
244 } \
Tejun Heo0f5e4812009-10-29 22:34:12 +0900245 pscr_ret__; \
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900246})
247
Tejun Heo0f5e4812009-10-29 22:34:12 +0900248#define __pcpu_size_call(stem, variable, ...) \
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900249do { \
250 switch(sizeof(variable)) { \
251 case 1: stem##1(variable, __VA_ARGS__);break; \
252 case 2: stem##2(variable, __VA_ARGS__);break; \
253 case 4: stem##4(variable, __VA_ARGS__);break; \
254 case 8: stem##8(variable, __VA_ARGS__);break; \
255 default: \
256 __bad_size_call_parameter();break; \
257 } \
258} while (0)
259
260/*
261 * Optimized manipulation for memory allocated through the per cpu
262 * allocator or for addresses of per cpu variables (can be determined
263 * using per_cpu_var(xx).
264 *
265 * These operation guarantee exclusivity of access for other operations
266 * on the *same* processor. The assumption is that per cpu data is only
267 * accessed by a single processor instance (the current one).
268 *
269 * The first group is used for accesses that must be done in a
270 * preemption safe way since we know that the context is not preempt
271 * safe. Interrupts may occur. If the interrupt modifies the variable
272 * too then RMW actions will not be reliable.
273 *
274 * The arch code can provide optimized functions in two ways:
275 *
276 * 1. Override the function completely. F.e. define this_cpu_add().
277 * The arch must then ensure that the various scalar format passed
278 * are handled correctly.
279 *
280 * 2. Provide functions for certain scalar sizes. F.e. provide
281 * this_cpu_add_2() to provide per cpu atomic operations for 2 byte
282 * sized RMW actions. If arch code does not provide operations for
283 * a scalar size then the fallback in the generic code will be
284 * used.
285 */
286
287#define _this_cpu_generic_read(pcp) \
288({ typeof(pcp) ret__; \
289 preempt_disable(); \
290 ret__ = *this_cpu_ptr(&(pcp)); \
291 preempt_enable(); \
292 ret__; \
293})
294
295#ifndef this_cpu_read
296# ifndef this_cpu_read_1
297# define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
298# endif
299# ifndef this_cpu_read_2
300# define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
301# endif
302# ifndef this_cpu_read_4
303# define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
304# endif
305# ifndef this_cpu_read_8
306# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
307# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900308# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900309#endif
310
311#define _this_cpu_generic_to_op(pcp, val, op) \
312do { \
313 preempt_disable(); \
314 *__this_cpu_ptr(&pcp) op val; \
315 preempt_enable(); \
316} while (0)
317
318#ifndef this_cpu_write
319# ifndef this_cpu_write_1
320# define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
321# endif
322# ifndef this_cpu_write_2
323# define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
324# endif
325# ifndef this_cpu_write_4
326# define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
327# endif
328# ifndef this_cpu_write_8
329# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
330# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900331# define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900332#endif
333
334#ifndef this_cpu_add
335# ifndef this_cpu_add_1
336# define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
337# endif
338# ifndef this_cpu_add_2
339# define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
340# endif
341# ifndef this_cpu_add_4
342# define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
343# endif
344# ifndef this_cpu_add_8
345# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
346# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900347# define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900348#endif
349
350#ifndef this_cpu_sub
351# define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(val))
352#endif
353
354#ifndef this_cpu_inc
355# define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
356#endif
357
358#ifndef this_cpu_dec
359# define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
360#endif
361
362#ifndef this_cpu_and
363# ifndef this_cpu_and_1
364# define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
365# endif
366# ifndef this_cpu_and_2
367# define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
368# endif
369# ifndef this_cpu_and_4
370# define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
371# endif
372# ifndef this_cpu_and_8
373# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
374# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900375# define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900376#endif
377
378#ifndef this_cpu_or
379# ifndef this_cpu_or_1
380# define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
381# endif
382# ifndef this_cpu_or_2
383# define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
384# endif
385# ifndef this_cpu_or_4
386# define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
387# endif
388# ifndef this_cpu_or_8
389# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
390# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900391# define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900392#endif
393
394#ifndef this_cpu_xor
395# ifndef this_cpu_xor_1
396# define this_cpu_xor_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
397# endif
398# ifndef this_cpu_xor_2
399# define this_cpu_xor_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
400# endif
401# ifndef this_cpu_xor_4
402# define this_cpu_xor_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
403# endif
404# ifndef this_cpu_xor_8
405# define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
406# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900407# define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900408#endif
409
410/*
411 * Generic percpu operations that do not require preemption handling.
412 * Either we do not care about races or the caller has the
413 * responsibility of handling preemptions issues. Arch code can still
414 * override these instructions since the arch per cpu code may be more
415 * efficient and may actually get race freeness for free (that is the
416 * case for x86 for example).
417 *
418 * If there is no other protection through preempt disable and/or
419 * disabling interupts then one of these RMW operations can show unexpected
420 * behavior because the execution thread was rescheduled on another processor
421 * or an interrupt occurred and the same percpu variable was modified from
422 * the interrupt context.
423 */
424#ifndef __this_cpu_read
425# ifndef __this_cpu_read_1
426# define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp)))
427# endif
428# ifndef __this_cpu_read_2
429# define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp)))
430# endif
431# ifndef __this_cpu_read_4
432# define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp)))
433# endif
434# ifndef __this_cpu_read_8
435# define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp)))
436# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900437# define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900438#endif
439
440#define __this_cpu_generic_to_op(pcp, val, op) \
441do { \
442 *__this_cpu_ptr(&(pcp)) op val; \
443} while (0)
444
445#ifndef __this_cpu_write
446# ifndef __this_cpu_write_1
447# define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
448# endif
449# ifndef __this_cpu_write_2
450# define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
451# endif
452# ifndef __this_cpu_write_4
453# define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
454# endif
455# ifndef __this_cpu_write_8
456# define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
457# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900458# define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900459#endif
460
461#ifndef __this_cpu_add
462# ifndef __this_cpu_add_1
463# define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
464# endif
465# ifndef __this_cpu_add_2
466# define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
467# endif
468# ifndef __this_cpu_add_4
469# define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
470# endif
471# ifndef __this_cpu_add_8
472# define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
473# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900474# define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900475#endif
476
477#ifndef __this_cpu_sub
478# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(val))
479#endif
480
481#ifndef __this_cpu_inc
482# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
483#endif
484
485#ifndef __this_cpu_dec
486# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
487#endif
488
489#ifndef __this_cpu_and
490# ifndef __this_cpu_and_1
491# define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
492# endif
493# ifndef __this_cpu_and_2
494# define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
495# endif
496# ifndef __this_cpu_and_4
497# define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
498# endif
499# ifndef __this_cpu_and_8
500# define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
501# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900502# define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900503#endif
504
505#ifndef __this_cpu_or
506# ifndef __this_cpu_or_1
507# define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
508# endif
509# ifndef __this_cpu_or_2
510# define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
511# endif
512# ifndef __this_cpu_or_4
513# define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
514# endif
515# ifndef __this_cpu_or_8
516# define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
517# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900518# define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900519#endif
520
521#ifndef __this_cpu_xor
522# ifndef __this_cpu_xor_1
523# define __this_cpu_xor_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
524# endif
525# ifndef __this_cpu_xor_2
526# define __this_cpu_xor_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
527# endif
528# ifndef __this_cpu_xor_4
529# define __this_cpu_xor_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
530# endif
531# ifndef __this_cpu_xor_8
532# define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
533# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900534# define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900535#endif
536
537/*
538 * IRQ safe versions of the per cpu RMW operations. Note that these operations
539 * are *not* safe against modification of the same variable from another
540 * processors (which one gets when using regular atomic operations)
541 . They are guaranteed to be atomic vs. local interrupts and
542 * preemption only.
543 */
544#define irqsafe_cpu_generic_to_op(pcp, val, op) \
545do { \
546 unsigned long flags; \
547 local_irq_save(flags); \
548 *__this_cpu_ptr(&(pcp)) op val; \
549 local_irq_restore(flags); \
550} while (0)
551
552#ifndef irqsafe_cpu_add
553# ifndef irqsafe_cpu_add_1
554# define irqsafe_cpu_add_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
555# endif
556# ifndef irqsafe_cpu_add_2
557# define irqsafe_cpu_add_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
558# endif
559# ifndef irqsafe_cpu_add_4
560# define irqsafe_cpu_add_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
561# endif
562# ifndef irqsafe_cpu_add_8
563# define irqsafe_cpu_add_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
564# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900565# define irqsafe_cpu_add(pcp, val) __pcpu_size_call(irqsafe_cpu_add_, (pcp), (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900566#endif
567
568#ifndef irqsafe_cpu_sub
569# define irqsafe_cpu_sub(pcp, val) irqsafe_cpu_add((pcp), -(val))
570#endif
571
572#ifndef irqsafe_cpu_inc
573# define irqsafe_cpu_inc(pcp) irqsafe_cpu_add((pcp), 1)
574#endif
575
576#ifndef irqsafe_cpu_dec
577# define irqsafe_cpu_dec(pcp) irqsafe_cpu_sub((pcp), 1)
578#endif
579
580#ifndef irqsafe_cpu_and
581# ifndef irqsafe_cpu_and_1
582# define irqsafe_cpu_and_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
583# endif
584# ifndef irqsafe_cpu_and_2
585# define irqsafe_cpu_and_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
586# endif
587# ifndef irqsafe_cpu_and_4
588# define irqsafe_cpu_and_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
589# endif
590# ifndef irqsafe_cpu_and_8
591# define irqsafe_cpu_and_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
592# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900593# define irqsafe_cpu_and(pcp, val) __pcpu_size_call(irqsafe_cpu_and_, (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900594#endif
595
596#ifndef irqsafe_cpu_or
597# ifndef irqsafe_cpu_or_1
598# define irqsafe_cpu_or_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
599# endif
600# ifndef irqsafe_cpu_or_2
601# define irqsafe_cpu_or_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
602# endif
603# ifndef irqsafe_cpu_or_4
604# define irqsafe_cpu_or_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
605# endif
606# ifndef irqsafe_cpu_or_8
607# define irqsafe_cpu_or_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
608# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900609# define irqsafe_cpu_or(pcp, val) __pcpu_size_call(irqsafe_cpu_or_, (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900610#endif
611
612#ifndef irqsafe_cpu_xor
613# ifndef irqsafe_cpu_xor_1
614# define irqsafe_cpu_xor_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
615# endif
616# ifndef irqsafe_cpu_xor_2
617# define irqsafe_cpu_xor_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
618# endif
619# ifndef irqsafe_cpu_xor_4
620# define irqsafe_cpu_xor_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
621# endif
622# ifndef irqsafe_cpu_xor_8
623# define irqsafe_cpu_xor_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
624# endif
Tejun Heo0f5e4812009-10-29 22:34:12 +0900625# define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val))
Christoph Lameter7340a0b2009-10-03 19:48:22 +0900626#endif
627
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628#endif /* __LINUX_PERCPU_H */