blob: 7e0d6a58fc830d1fba2de91427047ee230f431ea [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* rwsem-spinlock.c: R/W semaphores: contention handling functions for
2 * generic spinlock implementation
3 *
4 * Copyright (c) 2001 David Howells (dhowells@redhat.com).
5 * - Derived partially from idea by Andrea Arcangeli <andrea@suse.de>
6 * - Derived also from comments by Linus
7 */
8#include <linux/rwsem.h>
9#include <linux/sched.h>
Paul Gortmaker8bc3bcc2011-11-16 21:29:17 -050010#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
12struct rwsem_waiter {
13 struct list_head list;
14 struct task_struct *task;
15 unsigned int flags;
16#define RWSEM_WAITING_FOR_READ 0x00000001
17#define RWSEM_WAITING_FOR_WRITE 0x00000002
18};
19
Amerigo Wang29671f22009-12-14 18:00:21 -080020int rwsem_is_locked(struct rw_semaphore *sem)
21{
22 int ret = 1;
23 unsigned long flags;
24
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +010025 if (raw_spin_trylock_irqsave(&sem->wait_lock, flags)) {
Amerigo Wang29671f22009-12-14 18:00:21 -080026 ret = (sem->activity != 0);
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +010027 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Amerigo Wang29671f22009-12-14 18:00:21 -080028 }
29 return ret;
30}
31EXPORT_SYMBOL(rwsem_is_locked);
32
Linus Torvalds1da177e2005-04-16 15:20:36 -070033/*
34 * initialise the semaphore
35 */
Ingo Molnar4ea21762006-07-03 00:24:53 -070036void __init_rwsem(struct rw_semaphore *sem, const char *name,
37 struct lock_class_key *key)
Linus Torvalds1da177e2005-04-16 15:20:36 -070038{
Ingo Molnar4ea21762006-07-03 00:24:53 -070039#ifdef CONFIG_DEBUG_LOCK_ALLOC
40 /*
41 * Make sure we are not reinitializing a held semaphore:
42 */
43 debug_check_no_locks_freed((void *)sem, sizeof(*sem));
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -040044 lockdep_init_map(&sem->dep_map, name, key, 0);
Ingo Molnar4ea21762006-07-03 00:24:53 -070045#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 sem->activity = 0;
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +010047 raw_spin_lock_init(&sem->wait_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 INIT_LIST_HEAD(&sem->wait_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070049}
Amerigo Wang118d52d2009-12-14 18:00:20 -080050EXPORT_SYMBOL(__init_rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
52/*
53 * handle the lock release when processes blocked on it that can now run
54 * - if we come here, then:
55 * - the 'active count' _reached_ zero
56 * - the 'waiting count' is non-zero
57 * - the spinlock must be held by the caller
58 * - woken process blocks are discarded from the list after having task zeroed
59 * - writers are only woken if wakewrite is non-zero
60 */
61static inline struct rw_semaphore *
62__rwsem_do_wake(struct rw_semaphore *sem, int wakewrite)
63{
64 struct rwsem_waiter *waiter;
65 struct task_struct *tsk;
66 int woken;
67
Linus Torvalds1da177e2005-04-16 15:20:36 -070068 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
69
70 if (!wakewrite) {
71 if (waiter->flags & RWSEM_WAITING_FOR_WRITE)
72 goto out;
73 goto dont_wake_writers;
74 }
75
76 /* if we are allowed to wake writers try to grant a single write lock
77 * if there's a writer at the front of the queue
78 * - we leave the 'waiting count' incremented to signify potential
79 * contention
80 */
81 if (waiter->flags & RWSEM_WAITING_FOR_WRITE) {
82 sem->activity = -1;
83 list_del(&waiter->list);
84 tsk = waiter->task;
85 /* Don't touch waiter after ->task has been NULLed */
akpm@osdl.orgd59dd462005-05-01 08:58:47 -070086 smp_mb();
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 waiter->task = NULL;
88 wake_up_process(tsk);
89 put_task_struct(tsk);
90 goto out;
91 }
92
93 /* grant an infinite number of read locks to the front of the queue */
94 dont_wake_writers:
95 woken = 0;
96 while (waiter->flags & RWSEM_WAITING_FOR_READ) {
97 struct list_head *next = waiter->list.next;
98
99 list_del(&waiter->list);
100 tsk = waiter->task;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700101 smp_mb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 waiter->task = NULL;
103 wake_up_process(tsk);
104 put_task_struct(tsk);
105 woken++;
106 if (list_empty(&sem->wait_list))
107 break;
108 waiter = list_entry(next, struct rwsem_waiter, list);
109 }
110
111 sem->activity += woken;
112
113 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 return sem;
115}
116
117/*
118 * wake a single writer
119 */
120static inline struct rw_semaphore *
121__rwsem_wake_one_writer(struct rw_semaphore *sem)
122{
123 struct rwsem_waiter *waiter;
124 struct task_struct *tsk;
125
126 sem->activity = -1;
127
128 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
129 list_del(&waiter->list);
130
131 tsk = waiter->task;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700132 smp_mb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133 waiter->task = NULL;
134 wake_up_process(tsk);
135 put_task_struct(tsk);
136 return sem;
137}
138
139/*
140 * get a read lock on the semaphore
141 */
Harvey Harrison9f741cb2008-02-08 04:19:55 -0800142void __sched __down_read(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143{
144 struct rwsem_waiter waiter;
145 struct task_struct *tsk;
Kevin Hilman3eac4ab2010-04-07 11:52:46 -0700146 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100148 raw_spin_lock_irqsave(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
150 if (sem->activity >= 0 && list_empty(&sem->wait_list)) {
151 /* granted */
152 sem->activity++;
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100153 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154 goto out;
155 }
156
157 tsk = current;
158 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
159
160 /* set up my own style of waitqueue */
161 waiter.task = tsk;
162 waiter.flags = RWSEM_WAITING_FOR_READ;
163 get_task_struct(tsk);
164
165 list_add_tail(&waiter.list, &sem->wait_list);
166
167 /* we don't need to touch the semaphore struct anymore */
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100168 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169
170 /* wait to be given the lock */
171 for (;;) {
172 if (!waiter.task)
173 break;
174 schedule();
175 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
176 }
177
178 tsk->state = TASK_RUNNING;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 out:
Ingo Molnarc4e05112006-07-03 00:24:29 -0700180 ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181}
182
183/*
184 * trylock for reading -- returns 1 if successful, 0 if contention
185 */
Harvey Harrison9f741cb2008-02-08 04:19:55 -0800186int __down_read_trylock(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187{
188 unsigned long flags;
189 int ret = 0;
190
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100192 raw_spin_lock_irqsave(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194 if (sem->activity >= 0 && list_empty(&sem->wait_list)) {
195 /* granted */
196 sem->activity++;
197 ret = 1;
198 }
199
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100200 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 return ret;
203}
204
205/*
206 * get a write lock on the semaphore
207 * - we increment the waiting count anyway to indicate an exclusive lock
208 */
Harvey Harrison9f741cb2008-02-08 04:19:55 -0800209void __sched __down_write_nested(struct rw_semaphore *sem, int subclass)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210{
211 struct rwsem_waiter waiter;
212 struct task_struct *tsk;
Kevin Hilman3eac4ab2010-04-07 11:52:46 -0700213 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100215 raw_spin_lock_irqsave(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217 if (sem->activity == 0 && list_empty(&sem->wait_list)) {
218 /* granted */
219 sem->activity = -1;
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100220 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 goto out;
222 }
223
224 tsk = current;
225 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
226
227 /* set up my own style of waitqueue */
228 waiter.task = tsk;
229 waiter.flags = RWSEM_WAITING_FOR_WRITE;
230 get_task_struct(tsk);
231
232 list_add_tail(&waiter.list, &sem->wait_list);
233
234 /* we don't need to touch the semaphore struct anymore */
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100235 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236
237 /* wait to be given the lock */
238 for (;;) {
239 if (!waiter.task)
240 break;
241 schedule();
242 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
243 }
244
245 tsk->state = TASK_RUNNING;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246 out:
Ingo Molnarc4e05112006-07-03 00:24:29 -0700247 ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248}
249
Harvey Harrison9f741cb2008-02-08 04:19:55 -0800250void __sched __down_write(struct rw_semaphore *sem)
Ingo Molnar4ea21762006-07-03 00:24:53 -0700251{
252 __down_write_nested(sem, 0);
253}
254
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255/*
256 * trylock for writing -- returns 1 if successful, 0 if contention
257 */
Harvey Harrison9f741cb2008-02-08 04:19:55 -0800258int __down_write_trylock(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259{
260 unsigned long flags;
261 int ret = 0;
262
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100263 raw_spin_lock_irqsave(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264
265 if (sem->activity == 0 && list_empty(&sem->wait_list)) {
266 /* granted */
267 sem->activity = -1;
268 ret = 1;
269 }
270
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100271 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 return ret;
274}
275
276/*
277 * release a read lock on the semaphore
278 */
Harvey Harrison9f741cb2008-02-08 04:19:55 -0800279void __up_read(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280{
281 unsigned long flags;
282
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100283 raw_spin_lock_irqsave(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
285 if (--sem->activity == 0 && !list_empty(&sem->wait_list))
286 sem = __rwsem_wake_one_writer(sem);
287
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100288 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289}
290
291/*
292 * release a write lock on the semaphore
293 */
Harvey Harrison9f741cb2008-02-08 04:19:55 -0800294void __up_write(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295{
296 unsigned long flags;
297
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100298 raw_spin_lock_irqsave(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
300 sem->activity = 0;
301 if (!list_empty(&sem->wait_list))
302 sem = __rwsem_do_wake(sem, 1);
303
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100304 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305}
306
307/*
308 * downgrade a write lock into a read lock
309 * - just wake up any readers at the front of the queue
310 */
Harvey Harrison9f741cb2008-02-08 04:19:55 -0800311void __downgrade_write(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312{
313 unsigned long flags;
314
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100315 raw_spin_lock_irqsave(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316
317 sem->activity = 1;
318 if (!list_empty(&sem->wait_list))
319 sem = __rwsem_do_wake(sem, 0);
320
Thomas Gleixnerddb6c9b2010-02-24 09:54:54 +0100321 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322}
323