Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef __LINUX_PERCPU_H |
| 2 | #define __LINUX_PERCPU_H |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 3 | |
Robert P. J. Day | 0a3021f | 2007-07-15 23:39:57 -0700 | [diff] [blame] | 4 | #include <linux/preempt.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | #include <linux/slab.h> /* For kmalloc() */ |
| 6 | #include <linux/smp.h> |
| 7 | #include <linux/string.h> /* For memset() */ |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 8 | #include <linux/cpumask.h> |
| 9 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | #include <asm/percpu.h> |
| 11 | |
| 12 | /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */ |
| 13 | #ifndef PERCPU_ENOUGH_ROOM |
Jeremy Fitzhardinge | b00742d | 2007-05-02 19:27:11 +0200 | [diff] [blame] | 14 | #ifdef CONFIG_MODULES |
| 15 | #define PERCPU_MODULE_RESERVE 8192 |
| 16 | #else |
| 17 | #define PERCPU_MODULE_RESERVE 0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | #endif |
| 19 | |
Jeremy Fitzhardinge | b00742d | 2007-05-02 19:27:11 +0200 | [diff] [blame] | 20 | #define PERCPU_ENOUGH_ROOM \ |
| 21 | (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE) |
| 22 | #endif /* PERCPU_ENOUGH_ROOM */ |
| 23 | |
Jan Blunck | 632bbfe | 2006-09-25 23:30:53 -0700 | [diff] [blame] | 24 | /* |
| 25 | * Must be an lvalue. Since @var must be a simple identifier, |
| 26 | * we force a syntax error here if it isn't. |
| 27 | */ |
| 28 | #define get_cpu_var(var) (*({ \ |
Jan Blunck | a666ecf | 2006-10-06 00:43:58 -0700 | [diff] [blame] | 29 | extern int simple_identifier_##var(void); \ |
Jan Blunck | 632bbfe | 2006-09-25 23:30:53 -0700 | [diff] [blame] | 30 | preempt_disable(); \ |
| 31 | &__get_cpu_var(var); })) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | #define put_cpu_var(var) preempt_enable() |
| 33 | |
| 34 | #ifdef CONFIG_SMP |
| 35 | |
| 36 | struct percpu_data { |
| 37 | void *ptrs[NR_CPUS]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | }; |
| 39 | |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 40 | #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | /* |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 42 | * Use this to get to a cpu's version of the per-cpu object dynamically |
| 43 | * allocated. Non-atomic access to the current CPU's version should |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 44 | * probably be combined with get_cpu()/put_cpu(). |
| 45 | */ |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 46 | #define percpu_ptr(ptr, cpu) \ |
| 47 | ({ \ |
| 48 | struct percpu_data *__p = __percpu_disguise(ptr); \ |
| 49 | (__typeof__(ptr))__p->ptrs[(cpu)]; \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 50 | }) |
| 51 | |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 52 | extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu); |
| 53 | extern void percpu_depopulate(void *__pdata, int cpu); |
| 54 | extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, |
| 55 | cpumask_t *mask); |
| 56 | extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask); |
| 57 | extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask); |
| 58 | extern void percpu_free(void *__pdata); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 59 | |
| 60 | #else /* CONFIG_SMP */ |
| 61 | |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 62 | #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 64 | static inline void percpu_depopulate(void *__pdata, int cpu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 66 | } |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 67 | |
| 68 | static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask) |
| 69 | { |
| 70 | } |
| 71 | |
| 72 | static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, |
| 73 | int cpu) |
| 74 | { |
| 75 | return percpu_ptr(__pdata, cpu); |
| 76 | } |
| 77 | |
| 78 | static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, |
| 79 | cpumask_t *mask) |
| 80 | { |
| 81 | return 0; |
| 82 | } |
| 83 | |
Al Viro | 0891a8d | 2006-09-29 01:58:34 -0700 | [diff] [blame] | 84 | static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask) |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 85 | { |
| 86 | return kzalloc(size, gfp); |
| 87 | } |
| 88 | |
| 89 | static inline void percpu_free(void *__pdata) |
| 90 | { |
| 91 | kfree(__pdata); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 92 | } |
| 93 | |
| 94 | #endif /* CONFIG_SMP */ |
| 95 | |
Martin Peschke | 7ff6f08 | 2006-09-25 23:31:21 -0700 | [diff] [blame] | 96 | #define percpu_populate_mask(__pdata, size, gfp, mask) \ |
| 97 | __percpu_populate_mask((__pdata), (size), (gfp), &(mask)) |
| 98 | #define percpu_depopulate_mask(__pdata, mask) \ |
| 99 | __percpu_depopulate_mask((__pdata), &(mask)) |
| 100 | #define percpu_alloc_mask(size, gfp, mask) \ |
| 101 | __percpu_alloc_mask((size), (gfp), &(mask)) |
| 102 | |
| 103 | #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map) |
| 104 | |
| 105 | /* (legacy) interface for use without CPU hotplug handling */ |
| 106 | |
| 107 | #define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \ |
| 108 | cpu_possible_map) |
| 109 | #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type)) |
| 110 | #define free_percpu(ptr) percpu_free((ptr)) |
| 111 | #define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | |
| 113 | #endif /* __LINUX_PERCPU_H */ |