Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Percpu refcounts: |
| 3 | * (C) 2012 Google, Inc. |
| 4 | * Author: Kent Overstreet <koverstreet@google.com> |
| 5 | * |
| 6 | * This implements a refcount with similar semantics to atomic_t - atomic_inc(), |
| 7 | * atomic_dec_and_test() - but percpu. |
| 8 | * |
| 9 | * There's one important difference between percpu refs and normal atomic_t |
| 10 | * refcounts; you have to keep track of your initial refcount, and then when you |
| 11 | * start shutting down you call percpu_ref_kill() _before_ dropping the initial |
| 12 | * refcount. |
| 13 | * |
| 14 | * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less |
| 15 | * than an atomic_t - this is because of the way shutdown works, see |
| 16 | * percpu_ref_kill()/PCPU_COUNT_BIAS. |
| 17 | * |
| 18 | * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the |
| 19 | * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill() |
| 20 | * puts the ref back in single atomic_t mode, collecting the per cpu refs and |
| 21 | * issuing the appropriate barriers, and then marks the ref as shutting down so |
| 22 | * that percpu_ref_put() will check for the ref hitting 0. After it returns, |
| 23 | * it's safe to drop the initial ref. |
| 24 | * |
| 25 | * USAGE: |
| 26 | * |
| 27 | * See fs/aio.c for some example usage; it's used there for struct kioctx, which |
| 28 | * is created when userspaces calls io_setup(), and destroyed when userspace |
| 29 | * calls io_destroy() or the process exits. |
| 30 | * |
| 31 | * In the aio code, kill_ioctx() is called when we wish to destroy a kioctx; it |
| 32 | * calls percpu_ref_kill(), then hlist_del_rcu() and sychronize_rcu() to remove |
| 33 | * the kioctx from the proccess's list of kioctxs - after that, there can't be |
| 34 | * any new users of the kioctx (from lookup_ioctx()) and it's then safe to drop |
| 35 | * the initial ref with percpu_ref_put(). |
| 36 | * |
| 37 | * Code that does a two stage shutdown like this often needs some kind of |
| 38 | * explicit synchronization to ensure the initial refcount can only be dropped |
| 39 | * once - percpu_ref_kill() does this for you, it returns true once and false if |
| 40 | * someone else already called it. The aio code uses it this way, but it's not |
| 41 | * necessary if the code has some other mechanism to synchronize teardown. |
| 42 | * around. |
| 43 | */ |
| 44 | |
| 45 | #ifndef _LINUX_PERCPU_REFCOUNT_H |
| 46 | #define _LINUX_PERCPU_REFCOUNT_H |
| 47 | |
| 48 | #include <linux/atomic.h> |
| 49 | #include <linux/kernel.h> |
| 50 | #include <linux/percpu.h> |
| 51 | #include <linux/rcupdate.h> |
| 52 | |
| 53 | struct percpu_ref; |
Tejun Heo | ac89906 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 54 | typedef void (percpu_ref_func_t)(struct percpu_ref *); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 55 | |
| 56 | struct percpu_ref { |
| 57 | atomic_t count; |
| 58 | /* |
| 59 | * The low bit of the pointer indicates whether the ref is in percpu |
| 60 | * mode; if set, then get/put will manipulate the atomic_t (this is a |
| 61 | * hack because we need to keep the pointer around for |
| 62 | * percpu_ref_kill_rcu()) |
| 63 | */ |
| 64 | unsigned __percpu *pcpu_count; |
Tejun Heo | ac89906 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 65 | percpu_ref_func_t *release; |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 66 | struct rcu_head rcu; |
| 67 | }; |
| 68 | |
Tejun Heo | acac788 | 2013-06-12 20:52:01 -0700 | [diff] [blame^] | 69 | int __must_check percpu_ref_init(struct percpu_ref *ref, |
| 70 | percpu_ref_func_t *release); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 71 | void percpu_ref_kill(struct percpu_ref *ref); |
| 72 | |
| 73 | #define PCPU_STATUS_BITS 2 |
| 74 | #define PCPU_STATUS_MASK ((1 << PCPU_STATUS_BITS) - 1) |
| 75 | #define PCPU_REF_PTR 0 |
| 76 | #define PCPU_REF_DEAD 1 |
| 77 | |
| 78 | #define REF_STATUS(count) (((unsigned long) count) & PCPU_STATUS_MASK) |
| 79 | |
| 80 | /** |
| 81 | * percpu_ref_get - increment a percpu refcount |
Tejun Heo | ac89906 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 82 | * @ref: percpu_ref to get |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 83 | * |
| 84 | * Analagous to atomic_inc(). |
| 85 | */ |
| 86 | static inline void percpu_ref_get(struct percpu_ref *ref) |
| 87 | { |
| 88 | unsigned __percpu *pcpu_count; |
| 89 | |
Tejun Heo | 6a24474 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 90 | rcu_read_lock(); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 91 | |
| 92 | pcpu_count = ACCESS_ONCE(ref->pcpu_count); |
| 93 | |
| 94 | if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) |
| 95 | __this_cpu_inc(*pcpu_count); |
| 96 | else |
| 97 | atomic_inc(&ref->count); |
| 98 | |
Tejun Heo | 6a24474 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 99 | rcu_read_unlock(); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 100 | } |
| 101 | |
| 102 | /** |
| 103 | * percpu_ref_put - decrement a percpu refcount |
Tejun Heo | ac89906 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 104 | * @ref: percpu_ref to put |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 105 | * |
| 106 | * Decrement the refcount, and if 0, call the release function (which was passed |
| 107 | * to percpu_ref_init()) |
| 108 | */ |
| 109 | static inline void percpu_ref_put(struct percpu_ref *ref) |
| 110 | { |
| 111 | unsigned __percpu *pcpu_count; |
| 112 | |
Tejun Heo | 6a24474 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 113 | rcu_read_lock(); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 114 | |
| 115 | pcpu_count = ACCESS_ONCE(ref->pcpu_count); |
| 116 | |
| 117 | if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) |
| 118 | __this_cpu_dec(*pcpu_count); |
| 119 | else if (unlikely(atomic_dec_and_test(&ref->count))) |
| 120 | ref->release(ref); |
| 121 | |
Tejun Heo | 6a24474 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 122 | rcu_read_unlock(); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 123 | } |
| 124 | |
| 125 | #endif |