Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef _LINUX_PERCPU_COUNTER_H |
| 2 | #define _LINUX_PERCPU_COUNTER_H |
| 3 | /* |
| 4 | * A simple "approximate counter" for use in ext2 and ext3 superblocks. |
| 5 | * |
| 6 | * WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4. |
| 7 | */ |
| 8 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | #include <linux/spinlock.h> |
| 10 | #include <linux/smp.h> |
Andrew Morton | c67ad91 | 2007-07-15 23:39:51 -0700 | [diff] [blame] | 11 | #include <linux/list.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <linux/threads.h> |
| 13 | #include <linux/percpu.h> |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 14 | #include <linux/types.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | |
| 16 | #ifdef CONFIG_SMP |
| 17 | |
| 18 | struct percpu_counter { |
| 19 | spinlock_t lock; |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 20 | s64 count; |
Andrew Morton | c67ad91 | 2007-07-15 23:39:51 -0700 | [diff] [blame] | 21 | #ifdef CONFIG_HOTPLUG_CPU |
| 22 | struct list_head list; /* All percpu_counters are on a list */ |
| 23 | #endif |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 24 | s32 *counters; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | }; |
| 26 | |
Eric Dumazet | 179f7eb | 2009-01-06 14:41:04 -0800 | [diff] [blame^] | 27 | extern int percpu_counter_batch; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | |
Peter Zijlstra | 833f407 | 2007-10-16 23:25:45 -0700 | [diff] [blame] | 29 | int percpu_counter_init(struct percpu_counter *fbc, s64 amount); |
Peter Zijlstra | dc62a30 | 2007-10-16 23:25:46 -0700 | [diff] [blame] | 30 | int percpu_counter_init_irq(struct percpu_counter *fbc, s64 amount); |
Andrew Morton | c67ad91 | 2007-07-15 23:39:51 -0700 | [diff] [blame] | 31 | void percpu_counter_destroy(struct percpu_counter *fbc); |
Peter Zijlstra | 3a587f4 | 2007-10-16 23:25:44 -0700 | [diff] [blame] | 32 | void percpu_counter_set(struct percpu_counter *fbc, s64 amount); |
Peter Zijlstra | 20e8976 | 2007-10-16 23:25:43 -0700 | [diff] [blame] | 33 | void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch); |
Andrew Morton | 02d2116 | 2008-12-09 13:14:14 -0800 | [diff] [blame] | 34 | s64 __percpu_counter_sum(struct percpu_counter *fbc); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | |
Peter Zijlstra | 20e8976 | 2007-10-16 23:25:43 -0700 | [diff] [blame] | 36 | static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount) |
Peter Zijlstra | 252e0ba | 2007-10-16 23:25:43 -0700 | [diff] [blame] | 37 | { |
Eric Dumazet | 179f7eb | 2009-01-06 14:41:04 -0800 | [diff] [blame^] | 38 | __percpu_counter_add(fbc, amount, percpu_counter_batch); |
Peter Zijlstra | 252e0ba | 2007-10-16 23:25:43 -0700 | [diff] [blame] | 39 | } |
| 40 | |
Peter Zijlstra | bf1d89c | 2007-10-16 23:25:45 -0700 | [diff] [blame] | 41 | static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) |
| 42 | { |
Andrew Morton | 02d2116 | 2008-12-09 13:14:14 -0800 | [diff] [blame] | 43 | s64 ret = __percpu_counter_sum(fbc); |
Peter Zijlstra | bf1d89c | 2007-10-16 23:25:45 -0700 | [diff] [blame] | 44 | return ret < 0 ? 0 : ret; |
| 45 | } |
| 46 | |
| 47 | static inline s64 percpu_counter_sum(struct percpu_counter *fbc) |
| 48 | { |
Andrew Morton | 02d2116 | 2008-12-09 13:14:14 -0800 | [diff] [blame] | 49 | return __percpu_counter_sum(fbc); |
Peter Zijlstra | bf1d89c | 2007-10-16 23:25:45 -0700 | [diff] [blame] | 50 | } |
| 51 | |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 52 | static inline s64 percpu_counter_read(struct percpu_counter *fbc) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | { |
| 54 | return fbc->count; |
| 55 | } |
| 56 | |
| 57 | /* |
| 58 | * It is possible for the percpu_counter_read() to return a small negative |
| 59 | * number for some counter which should never be negative. |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 60 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 61 | */ |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 62 | static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | { |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 64 | s64 ret = fbc->count; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | |
| 66 | barrier(); /* Prevent reloads of fbc->count */ |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 67 | if (ret >= 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 68 | return ret; |
| 69 | return 1; |
| 70 | } |
| 71 | |
| 72 | #else |
| 73 | |
| 74 | struct percpu_counter { |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 75 | s64 count; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 76 | }; |
| 77 | |
Peter Zijlstra | 833f407 | 2007-10-16 23:25:45 -0700 | [diff] [blame] | 78 | static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 79 | { |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 80 | fbc->count = amount; |
Peter Zijlstra | 833f407 | 2007-10-16 23:25:45 -0700 | [diff] [blame] | 81 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | } |
| 83 | |
Peter Zijlstra | dc62a30 | 2007-10-16 23:25:46 -0700 | [diff] [blame] | 84 | #define percpu_counter_init_irq percpu_counter_init |
| 85 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 86 | static inline void percpu_counter_destroy(struct percpu_counter *fbc) |
| 87 | { |
| 88 | } |
| 89 | |
Peter Zijlstra | 3a587f4 | 2007-10-16 23:25:44 -0700 | [diff] [blame] | 90 | static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount) |
| 91 | { |
| 92 | fbc->count = amount; |
| 93 | } |
| 94 | |
Peter Zijlstra | 252e0ba | 2007-10-16 23:25:43 -0700 | [diff] [blame] | 95 | #define __percpu_counter_add(fbc, amount, batch) \ |
| 96 | percpu_counter_add(fbc, amount) |
| 97 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 98 | static inline void |
Peter Zijlstra | 20e8976 | 2007-10-16 23:25:43 -0700 | [diff] [blame] | 99 | percpu_counter_add(struct percpu_counter *fbc, s64 amount) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | { |
| 101 | preempt_disable(); |
| 102 | fbc->count += amount; |
| 103 | preempt_enable(); |
| 104 | } |
| 105 | |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 106 | static inline s64 percpu_counter_read(struct percpu_counter *fbc) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 107 | { |
| 108 | return fbc->count; |
| 109 | } |
| 110 | |
Mingming Cao | 0216bfc | 2006-06-23 02:05:41 -0700 | [diff] [blame] | 111 | static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | { |
| 113 | return fbc->count; |
| 114 | } |
| 115 | |
Peter Zijlstra | 52d9f3b | 2007-10-16 23:25:44 -0700 | [diff] [blame] | 116 | static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) |
Andrew Morton | e2bab3d | 2006-03-07 21:55:31 -0800 | [diff] [blame] | 117 | { |
| 118 | return percpu_counter_read_positive(fbc); |
| 119 | } |
| 120 | |
Peter Zijlstra | bf1d89c | 2007-10-16 23:25:45 -0700 | [diff] [blame] | 121 | static inline s64 percpu_counter_sum(struct percpu_counter *fbc) |
| 122 | { |
| 123 | return percpu_counter_read(fbc); |
| 124 | } |
| 125 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | #endif /* CONFIG_SMP */ |
| 127 | |
| 128 | static inline void percpu_counter_inc(struct percpu_counter *fbc) |
| 129 | { |
Peter Zijlstra | aa0dff2 | 2007-10-16 23:25:42 -0700 | [diff] [blame] | 130 | percpu_counter_add(fbc, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | } |
| 132 | |
| 133 | static inline void percpu_counter_dec(struct percpu_counter *fbc) |
| 134 | { |
Peter Zijlstra | aa0dff2 | 2007-10-16 23:25:42 -0700 | [diff] [blame] | 135 | percpu_counter_add(fbc, -1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | } |
| 137 | |
Peter Zijlstra | 3cb4f9f | 2007-10-16 23:25:42 -0700 | [diff] [blame] | 138 | static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount) |
| 139 | { |
| 140 | percpu_counter_add(fbc, -amount); |
| 141 | } |
| 142 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 143 | #endif /* _LINUX_PERCPU_COUNTER_H */ |