blob: d5dd4657c8d64133eaa2e31ca745770b7b81af4b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_PERCPU_COUNTER_H
2#define _LINUX_PERCPU_COUNTER_H
3/*
4 * A simple "approximate counter" for use in ext2 and ext3 superblocks.
5 *
6 * WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4.
7 */
8
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/spinlock.h>
10#include <linux/smp.h>
Andrew Mortonc67ad912007-07-15 23:39:51 -070011#include <linux/list.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/threads.h>
13#include <linux/percpu.h>
Mingming Cao0216bfc2006-06-23 02:05:41 -070014#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015
16#ifdef CONFIG_SMP
17
18struct percpu_counter {
Thomas Gleixnerf032a452009-07-25 16:21:48 +020019 raw_spinlock_t lock;
Mingming Cao0216bfc2006-06-23 02:05:41 -070020 s64 count;
Andrew Mortonc67ad912007-07-15 23:39:51 -070021#ifdef CONFIG_HOTPLUG_CPU
22 struct list_head list; /* All percpu_counters are on a list */
23#endif
Tejun Heo43cf38e2010-02-02 14:38:57 +090024 s32 __percpu *counters;
Linus Torvalds1da177e2005-04-16 15:20:36 -070025};
26
Eric Dumazet179f7eb2009-01-06 14:41:04 -080027extern int percpu_counter_batch;
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Peter Zijlstraea319512008-12-26 15:08:55 +010029int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
30 struct lock_class_key *key);
31
32#define percpu_counter_init(fbc, value) \
33 ({ \
34 static struct lock_class_key __key; \
35 \
36 __percpu_counter_init(fbc, value, &__key); \
37 })
38
Andrew Mortonc67ad912007-07-15 23:39:51 -070039void percpu_counter_destroy(struct percpu_counter *fbc);
Peter Zijlstra3a587f42007-10-16 23:25:44 -070040void percpu_counter_set(struct percpu_counter *fbc, s64 amount);
Peter Zijlstra20e89762007-10-16 23:25:43 -070041void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch);
Andrew Morton02d21162008-12-09 13:14:14 -080042s64 __percpu_counter_sum(struct percpu_counter *fbc);
Tim Chen27f5e0f2010-08-09 17:19:04 -070043int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
Peter Zijlstra20e89762007-10-16 23:25:43 -070045static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount)
Peter Zijlstra252e0ba2007-10-16 23:25:43 -070046{
Eric Dumazet179f7eb2009-01-06 14:41:04 -080047 __percpu_counter_add(fbc, amount, percpu_counter_batch);
Peter Zijlstra252e0ba2007-10-16 23:25:43 -070048}
49
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070050static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
51{
Andrew Morton02d21162008-12-09 13:14:14 -080052 s64 ret = __percpu_counter_sum(fbc);
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070053 return ret < 0 ? 0 : ret;
54}
55
56static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
57{
Andrew Morton02d21162008-12-09 13:14:14 -080058 return __percpu_counter_sum(fbc);
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070059}
60
Mingming Cao0216bfc2006-06-23 02:05:41 -070061static inline s64 percpu_counter_read(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -070062{
63 return fbc->count;
64}
65
66/*
67 * It is possible for the percpu_counter_read() to return a small negative
68 * number for some counter which should never be negative.
Mingming Cao0216bfc2006-06-23 02:05:41 -070069 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 */
Mingming Cao0216bfc2006-06-23 02:05:41 -070071static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072{
Mingming Cao0216bfc2006-06-23 02:05:41 -070073 s64 ret = fbc->count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
75 barrier(); /* Prevent reloads of fbc->count */
Mingming Cao0216bfc2006-06-23 02:05:41 -070076 if (ret >= 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070077 return ret;
Shaohua Lic84598b2011-05-24 17:13:35 -070078 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070079}
80
Theodore Ts'o7f93cff2010-10-27 21:30:13 -040081static inline int percpu_counter_initialized(struct percpu_counter *fbc)
82{
83 return (fbc->counters != NULL);
84}
85
Jesper Dangaard Brouer7fa4cf92013-02-04 23:14:12 +010086#else /* !CONFIG_SMP */
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
88struct percpu_counter {
Mingming Cao0216bfc2006-06-23 02:05:41 -070089 s64 count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070090};
91
Peter Zijlstra833f4072007-10-16 23:25:45 -070092static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
Linus Torvalds1da177e2005-04-16 15:20:36 -070093{
Mingming Cao0216bfc2006-06-23 02:05:41 -070094 fbc->count = amount;
Peter Zijlstra833f4072007-10-16 23:25:45 -070095 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070096}
97
98static inline void percpu_counter_destroy(struct percpu_counter *fbc)
99{
100}
101
Peter Zijlstra3a587f42007-10-16 23:25:44 -0700102static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
103{
104 fbc->count = amount;
105}
106
Tim Chen27f5e0f2010-08-09 17:19:04 -0700107static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
108{
109 if (fbc->count > rhs)
110 return 1;
111 else if (fbc->count < rhs)
112 return -1;
113 else
114 return 0;
115}
116
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117static inline void
Peter Zijlstra20e89762007-10-16 23:25:43 -0700118percpu_counter_add(struct percpu_counter *fbc, s64 amount)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119{
120 preempt_disable();
121 fbc->count += amount;
122 preempt_enable();
123}
124
Anton Blanchard0c9cf2e2010-02-02 14:46:10 -0800125static inline void
126__percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
127{
128 percpu_counter_add(fbc, amount);
129}
130
Mingming Cao0216bfc2006-06-23 02:05:41 -0700131static inline s64 percpu_counter_read(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132{
133 return fbc->count;
134}
135
Shaohua Lic84598b2011-05-24 17:13:35 -0700136/*
137 * percpu_counter is intended to track positive numbers. In the UP case the
138 * number should never be negative.
139 */
Mingming Cao0216bfc2006-06-23 02:05:41 -0700140static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141{
142 return fbc->count;
143}
144
Peter Zijlstra52d9f3b2007-10-16 23:25:44 -0700145static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
Andrew Mortone2bab3d2006-03-07 21:55:31 -0800146{
147 return percpu_counter_read_positive(fbc);
148}
149
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -0700150static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
151{
152 return percpu_counter_read(fbc);
153}
154
Theodore Ts'o7f93cff2010-10-27 21:30:13 -0400155static inline int percpu_counter_initialized(struct percpu_counter *fbc)
156{
157 return 1;
158}
159
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160#endif /* CONFIG_SMP */
161
162static inline void percpu_counter_inc(struct percpu_counter *fbc)
163{
Peter Zijlstraaa0dff22007-10-16 23:25:42 -0700164 percpu_counter_add(fbc, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165}
166
167static inline void percpu_counter_dec(struct percpu_counter *fbc)
168{
Peter Zijlstraaa0dff22007-10-16 23:25:42 -0700169 percpu_counter_add(fbc, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170}
171
Peter Zijlstra3cb4f9f2007-10-16 23:25:42 -0700172static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount)
173{
174 percpu_counter_add(fbc, -amount);
175}
176
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177#endif /* _LINUX_PERCPU_COUNTER_H */