blob: 9007ccdfc1127cfe73db03e31dd82a843f8f4fa8 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_PERCPU_COUNTER_H
2#define _LINUX_PERCPU_COUNTER_H
3/*
4 * A simple "approximate counter" for use in ext2 and ext3 superblocks.
5 *
6 * WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4.
7 */
8
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/spinlock.h>
10#include <linux/smp.h>
Andrew Mortonc67ad912007-07-15 23:39:51 -070011#include <linux/list.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/threads.h>
13#include <linux/percpu.h>
Mingming Cao0216bfc2006-06-23 02:05:41 -070014#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015
16#ifdef CONFIG_SMP
17
18struct percpu_counter {
19 spinlock_t lock;
Mingming Cao0216bfc2006-06-23 02:05:41 -070020 s64 count;
Andrew Mortonc67ad912007-07-15 23:39:51 -070021#ifdef CONFIG_HOTPLUG_CPU
22 struct list_head list; /* All percpu_counters are on a list */
23#endif
Mingming Cao0216bfc2006-06-23 02:05:41 -070024 s32 *counters;
Linus Torvalds1da177e2005-04-16 15:20:36 -070025};
26
27#if NR_CPUS >= 16
28#define FBC_BATCH (NR_CPUS*2)
29#else
30#define FBC_BATCH (NR_CPUS*4)
31#endif
32
Peter Zijlstra833f4072007-10-16 23:25:45 -070033int percpu_counter_init(struct percpu_counter *fbc, s64 amount);
Peter Zijlstradc62a302007-10-16 23:25:46 -070034int percpu_counter_init_irq(struct percpu_counter *fbc, s64 amount);
Andrew Mortonc67ad912007-07-15 23:39:51 -070035void percpu_counter_destroy(struct percpu_counter *fbc);
Peter Zijlstra3a587f42007-10-16 23:25:44 -070036void percpu_counter_set(struct percpu_counter *fbc, s64 amount);
Peter Zijlstra20e89762007-10-16 23:25:43 -070037void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch);
Mingming Cao1f7c14c2008-10-09 12:50:59 -040038s64 __percpu_counter_sum(struct percpu_counter *fbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -070039
Peter Zijlstra20e89762007-10-16 23:25:43 -070040static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount)
Peter Zijlstra252e0ba2007-10-16 23:25:43 -070041{
42 __percpu_counter_add(fbc, amount, FBC_BATCH);
43}
44
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070045static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
46{
Mingming Cao1f7c14c2008-10-09 12:50:59 -040047 s64 ret = __percpu_counter_sum(fbc);
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070048 return ret < 0 ? 0 : ret;
49}
50
51static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
52{
Mingming Cao1f7c14c2008-10-09 12:50:59 -040053 return __percpu_counter_sum(fbc);
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -070054}
55
Mingming Cao0216bfc2006-06-23 02:05:41 -070056static inline s64 percpu_counter_read(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -070057{
58 return fbc->count;
59}
60
61/*
62 * It is possible for the percpu_counter_read() to return a small negative
63 * number for some counter which should never be negative.
Mingming Cao0216bfc2006-06-23 02:05:41 -070064 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070065 */
Mingming Cao0216bfc2006-06-23 02:05:41 -070066static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -070067{
Mingming Cao0216bfc2006-06-23 02:05:41 -070068 s64 ret = fbc->count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070069
70 barrier(); /* Prevent reloads of fbc->count */
Mingming Cao0216bfc2006-06-23 02:05:41 -070071 if (ret >= 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 return ret;
73 return 1;
74}
75
76#else
77
78struct percpu_counter {
Mingming Cao0216bfc2006-06-23 02:05:41 -070079 s64 count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070080};
81
Peter Zijlstra833f4072007-10-16 23:25:45 -070082static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
Linus Torvalds1da177e2005-04-16 15:20:36 -070083{
Mingming Cao0216bfc2006-06-23 02:05:41 -070084 fbc->count = amount;
Peter Zijlstra833f4072007-10-16 23:25:45 -070085 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070086}
87
Peter Zijlstradc62a302007-10-16 23:25:46 -070088#define percpu_counter_init_irq percpu_counter_init
89
Linus Torvalds1da177e2005-04-16 15:20:36 -070090static inline void percpu_counter_destroy(struct percpu_counter *fbc)
91{
92}
93
Peter Zijlstra3a587f42007-10-16 23:25:44 -070094static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
95{
96 fbc->count = amount;
97}
98
Peter Zijlstra252e0ba2007-10-16 23:25:43 -070099#define __percpu_counter_add(fbc, amount, batch) \
100 percpu_counter_add(fbc, amount)
101
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102static inline void
Peter Zijlstra20e89762007-10-16 23:25:43 -0700103percpu_counter_add(struct percpu_counter *fbc, s64 amount)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104{
105 preempt_disable();
106 fbc->count += amount;
107 preempt_enable();
108}
109
Mingming Cao0216bfc2006-06-23 02:05:41 -0700110static inline s64 percpu_counter_read(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111{
112 return fbc->count;
113}
114
Mingming Cao0216bfc2006-06-23 02:05:41 -0700115static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116{
117 return fbc->count;
118}
119
Peter Zijlstra52d9f3b2007-10-16 23:25:44 -0700120static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
Andrew Mortone2bab3d2006-03-07 21:55:31 -0800121{
122 return percpu_counter_read_positive(fbc);
123}
124
Peter Zijlstrabf1d89c2007-10-16 23:25:45 -0700125static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
126{
127 return percpu_counter_read(fbc);
128}
129
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130#endif /* CONFIG_SMP */
131
132static inline void percpu_counter_inc(struct percpu_counter *fbc)
133{
Peter Zijlstraaa0dff22007-10-16 23:25:42 -0700134 percpu_counter_add(fbc, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135}
136
137static inline void percpu_counter_dec(struct percpu_counter *fbc)
138{
Peter Zijlstraaa0dff22007-10-16 23:25:42 -0700139 percpu_counter_add(fbc, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140}
141
Peter Zijlstra3cb4f9f2007-10-16 23:25:42 -0700142static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount)
143{
144 percpu_counter_add(fbc, -amount);
145}
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147#endif /* _LINUX_PERCPU_COUNTER_H */