blob: a7684a513994725335ac078d457c6a7a02d78455 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_PERCPU_COUNTER_H
2#define _LINUX_PERCPU_COUNTER_H
3/*
4 * A simple "approximate counter" for use in ext2 and ext3 superblocks.
5 *
6 * WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4.
7 */
8
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/spinlock.h>
10#include <linux/smp.h>
Andrew Mortonc67ad912007-07-15 23:39:51 -070011#include <linux/list.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/threads.h>
13#include <linux/percpu.h>
Mingming Cao0216bfc2006-06-23 02:05:41 -070014#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015
16#ifdef CONFIG_SMP
17
18struct percpu_counter {
19 spinlock_t lock;
Mingming Cao0216bfc2006-06-23 02:05:41 -070020 s64 count;
Andrew Mortonc67ad912007-07-15 23:39:51 -070021#ifdef CONFIG_HOTPLUG_CPU
22 struct list_head list; /* All percpu_counters are on a list */
23#endif
Mingming Cao0216bfc2006-06-23 02:05:41 -070024 s32 *counters;
Linus Torvalds1da177e2005-04-16 15:20:36 -070025};
26
Eric Dumazet179f7eb2009-01-06 14:41:04 -080027extern int percpu_counter_batch;
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Peter Zijlstraea319512008-12-26 15:08:55 +010029int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
30 struct lock_class_key *key);
31
32#define percpu_counter_init(fbc, value) \
33 ({ \
34 static struct lock_class_key __key; \
35 \
36 __percpu_counter_init(fbc, value, &__key); \
37 })
38
Andrew Mortonc67ad912007-07-15 23:39:51 -070039void percpu_counter_destroy(struct percpu_counter *fbc);
Peter Zijlstra3a587f42007-10-16 23:25:44 -070040void percpu_counter_set(struct percpu_counter *fbc, s64 amount);
Peter Zijlstra20e89762007-10-16 23:25:43 -070041void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch);
Andrew Morton02d21162008-12-09 13:14:14 -080042s64 __percpu_counter_sum(struct percpu_counter *fbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
Peter Zijlstra20e89762007-10-16 23:25:43 -070044static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount)
Peter Zijlstra252e0ba2007-10-16 23:25:43 -070045{
Eric Dumazet179f7eb2009-01-06 14:41:04 -080046 __percpu_counter_add(fbc, amount, percpu_counter_batch);
Peter Zijlstra252e0ba2007-10-16 23:25:43 -070047}
48
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070049static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
50{
Andrew Morton02d21162008-12-09 13:14:14 -080051 s64 ret = __percpu_counter_sum(fbc);
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070052 return ret < 0 ? 0 : ret;
53}
54
55static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
56{
Andrew Morton02d21162008-12-09 13:14:14 -080057 return __percpu_counter_sum(fbc);
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070058}
59
Mingming Cao0216bfc2006-06-23 02:05:41 -070060static inline s64 percpu_counter_read(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -070061{
62 return fbc->count;
63}
64
65/*
66 * It is possible for the percpu_counter_read() to return a small negative
67 * number for some counter which should never be negative.
Mingming Cao0216bfc2006-06-23 02:05:41 -070068 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 */
Mingming Cao0216bfc2006-06-23 02:05:41 -070070static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -070071{
Mingming Cao0216bfc2006-06-23 02:05:41 -070072 s64 ret = fbc->count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
74 barrier(); /* Prevent reloads of fbc->count */
Mingming Cao0216bfc2006-06-23 02:05:41 -070075 if (ret >= 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070076 return ret;
77 return 1;
78}
79
80#else
81
82struct percpu_counter {
Mingming Cao0216bfc2006-06-23 02:05:41 -070083 s64 count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070084};
85
Peter Zijlstra833f4072007-10-16 23:25:45 -070086static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
Linus Torvalds1da177e2005-04-16 15:20:36 -070087{
Mingming Cao0216bfc2006-06-23 02:05:41 -070088 fbc->count = amount;
Peter Zijlstra833f4072007-10-16 23:25:45 -070089 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070090}
91
92static inline void percpu_counter_destroy(struct percpu_counter *fbc)
93{
94}
95
Peter Zijlstra3a587f42007-10-16 23:25:44 -070096static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
97{
98 fbc->count = amount;
99}
100
Peter Zijlstra252e0ba2007-10-16 23:25:43 -0700101#define __percpu_counter_add(fbc, amount, batch) \
102 percpu_counter_add(fbc, amount)
103
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104static inline void
Peter Zijlstra20e89762007-10-16 23:25:43 -0700105percpu_counter_add(struct percpu_counter *fbc, s64 amount)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106{
107 preempt_disable();
108 fbc->count += amount;
109 preempt_enable();
110}
111
Mingming Cao0216bfc2006-06-23 02:05:41 -0700112static inline s64 percpu_counter_read(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113{
114 return fbc->count;
115}
116
Mingming Cao0216bfc2006-06-23 02:05:41 -0700117static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118{
119 return fbc->count;
120}
121
Peter Zijlstra52d9f3b2007-10-16 23:25:44 -0700122static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
Andrew Mortone2bab3d2006-03-07 21:55:31 -0800123{
124 return percpu_counter_read_positive(fbc);
125}
126
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -0700127static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
128{
129 return percpu_counter_read(fbc);
130}
131
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132#endif /* CONFIG_SMP */
133
134static inline void percpu_counter_inc(struct percpu_counter *fbc)
135{
Peter Zijlstraaa0dff22007-10-16 23:25:42 -0700136 percpu_counter_add(fbc, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137}
138
139static inline void percpu_counter_dec(struct percpu_counter *fbc)
140{
Peter Zijlstraaa0dff22007-10-16 23:25:42 -0700141 percpu_counter_add(fbc, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142}
143
Peter Zijlstra3cb4f9f2007-10-16 23:25:42 -0700144static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount)
145{
146 percpu_counter_add(fbc, -amount);
147}
148
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149#endif /* _LINUX_PERCPU_COUNTER_H */