blob: c4bfcb8836cdd961761d08ec275616ac80a464b6 [file] [log] [blame]
Linus Torvalds2f4f12e2013-09-02 11:58:20 -07001#include <linux/export.h>
2#include <linux/lockref.h>
3
Peter Zijlstra57f42572013-11-14 14:31:54 -08004#if USE_CMPXCHG_LOCKREF
Linus Torvaldsbc08b442013-09-02 12:12:15 -07005
6/*
7 * Note that the "cmpxchg()" reloads the "old" value for the
8 * failure case.
9 */
10#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
11 struct lockref old; \
12 BUILD_BUG_ON(sizeof(old) != 8); \
Davidlohr Bueso4d3199e2015-02-22 19:31:41 -080013 old.lock_count = READ_ONCE(lockref->lock_count); \
Linus Torvaldsbc08b442013-09-02 12:12:15 -070014 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
15 struct lockref new = old, prev = old; \
16 CODE \
Will Deacond2212b42013-09-26 17:27:00 +010017 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
18 old.lock_count, \
19 new.lock_count); \
Linus Torvaldsbc08b442013-09-02 12:12:15 -070020 if (likely(old.lock_count == prev.lock_count)) { \
21 SUCCESS; \
22 } \
Christian Borntraegerf2f09a42016-10-25 11:03:14 +020023 cpu_relax(); \
Linus Torvaldsbc08b442013-09-02 12:12:15 -070024 } \
25} while (0)
26
27#else
28
29#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
30
31#endif
32
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070033/**
34 * lockref_get - Increments reference count unconditionally
Linus Torvalds44a0cf92013-09-07 15:30:29 -070035 * @lockref: pointer to lockref structure
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070036 *
37 * This operation is only valid if you already hold a reference
38 * to the object, so you know the count cannot be zero.
39 */
40void lockref_get(struct lockref *lockref)
41{
Linus Torvaldsbc08b442013-09-02 12:12:15 -070042 CMPXCHG_LOOP(
43 new.count++;
44 ,
45 return;
46 );
47
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070048 spin_lock(&lockref->lock);
49 lockref->count++;
50 spin_unlock(&lockref->lock);
51}
52EXPORT_SYMBOL(lockref_get);
53
54/**
Linus Torvalds360f5472015-01-09 15:19:03 -080055 * lockref_get_not_zero - Increments count unless the count is 0 or dead
Linus Torvalds44a0cf92013-09-07 15:30:29 -070056 * @lockref: pointer to lockref structure
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070057 * Return: 1 if count updated successfully or 0 if count was zero
58 */
59int lockref_get_not_zero(struct lockref *lockref)
60{
Linus Torvaldsbc08b442013-09-02 12:12:15 -070061 int retval;
62
63 CMPXCHG_LOOP(
64 new.count++;
Linus Torvalds360f5472015-01-09 15:19:03 -080065 if (old.count <= 0)
Linus Torvaldsbc08b442013-09-02 12:12:15 -070066 return 0;
67 ,
68 return 1;
69 );
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070070
71 spin_lock(&lockref->lock);
Linus Torvaldsbc08b442013-09-02 12:12:15 -070072 retval = 0;
Linus Torvalds360f5472015-01-09 15:19:03 -080073 if (lockref->count > 0) {
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070074 lockref->count++;
75 retval = 1;
76 }
77 spin_unlock(&lockref->lock);
78 return retval;
79}
80EXPORT_SYMBOL(lockref_get_not_zero);
81
82/**
Linus Torvalds360f5472015-01-09 15:19:03 -080083 * lockref_get_or_lock - Increments count unless the count is 0 or dead
Linus Torvalds44a0cf92013-09-07 15:30:29 -070084 * @lockref: pointer to lockref structure
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070085 * Return: 1 if count updated successfully or 0 if count was zero
86 * and we got the lock instead.
87 */
88int lockref_get_or_lock(struct lockref *lockref)
89{
Linus Torvaldsbc08b442013-09-02 12:12:15 -070090 CMPXCHG_LOOP(
91 new.count++;
Linus Torvalds360f5472015-01-09 15:19:03 -080092 if (old.count <= 0)
Linus Torvaldsbc08b442013-09-02 12:12:15 -070093 break;
94 ,
95 return 1;
96 );
97
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070098 spin_lock(&lockref->lock);
Linus Torvalds360f5472015-01-09 15:19:03 -080099 if (lockref->count <= 0)
Linus Torvalds2f4f12e2013-09-02 11:58:20 -0700100 return 0;
101 lockref->count++;
102 spin_unlock(&lockref->lock);
103 return 1;
104}
105EXPORT_SYMBOL(lockref_get_or_lock);
106
107/**
Linus Torvalds360f5472015-01-09 15:19:03 -0800108 * lockref_put_return - Decrement reference count if possible
109 * @lockref: pointer to lockref structure
110 *
111 * Decrement the reference count and return the new value.
112 * If the lockref was dead or locked, return an error.
113 */
114int lockref_put_return(struct lockref *lockref)
115{
116 CMPXCHG_LOOP(
117 new.count--;
118 if (old.count <= 0)
119 return -1;
120 ,
121 return new.count;
122 );
123 return -1;
124}
125EXPORT_SYMBOL(lockref_put_return);
126
127/**
Linus Torvalds2f4f12e2013-09-02 11:58:20 -0700128 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
Linus Torvalds44a0cf92013-09-07 15:30:29 -0700129 * @lockref: pointer to lockref structure
Linus Torvalds2f4f12e2013-09-02 11:58:20 -0700130 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
131 */
132int lockref_put_or_lock(struct lockref *lockref)
133{
Linus Torvaldsbc08b442013-09-02 12:12:15 -0700134 CMPXCHG_LOOP(
135 new.count--;
136 if (old.count <= 1)
137 break;
138 ,
139 return 1;
140 );
141
Linus Torvalds2f4f12e2013-09-02 11:58:20 -0700142 spin_lock(&lockref->lock);
143 if (lockref->count <= 1)
144 return 0;
145 lockref->count--;
146 spin_unlock(&lockref->lock);
147 return 1;
148}
149EXPORT_SYMBOL(lockref_put_or_lock);
Linus Torvaldse7d33bb2013-09-07 15:49:18 -0700150
151/**
152 * lockref_mark_dead - mark lockref dead
153 * @lockref: pointer to lockref structure
154 */
155void lockref_mark_dead(struct lockref *lockref)
156{
157 assert_spin_locked(&lockref->lock);
158 lockref->count = -128;
159}
Steven Whitehousee66cf162013-10-15 15:18:08 +0100160EXPORT_SYMBOL(lockref_mark_dead);
Linus Torvaldse7d33bb2013-09-07 15:49:18 -0700161
162/**
163 * lockref_get_not_dead - Increments count unless the ref is dead
164 * @lockref: pointer to lockref structure
165 * Return: 1 if count updated successfully or 0 if lockref was dead
166 */
167int lockref_get_not_dead(struct lockref *lockref)
168{
169 int retval;
170
171 CMPXCHG_LOOP(
172 new.count++;
Linus Torvalds360f5472015-01-09 15:19:03 -0800173 if (old.count < 0)
Linus Torvaldse7d33bb2013-09-07 15:49:18 -0700174 return 0;
175 ,
176 return 1;
177 );
178
179 spin_lock(&lockref->lock);
180 retval = 0;
Linus Torvalds360f5472015-01-09 15:19:03 -0800181 if (lockref->count >= 0) {
Linus Torvaldse7d33bb2013-09-07 15:49:18 -0700182 lockref->count++;
183 retval = 1;
184 }
185 spin_unlock(&lockref->lock);
186 return retval;
187}
188EXPORT_SYMBOL(lockref_get_not_dead);