blob: aa7c3052261f2c3e6cdb6dd0301b64603fbca67d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* rwsem.c: R/W semaphores: contention handling functions
2 *
3 * Written by David Howells (dhowells@redhat.com).
4 * Derived from arch/i386/kernel/semaphore.c
5 */
6#include <linux/rwsem.h>
7#include <linux/sched.h>
8#include <linux/init.h>
9#include <linux/module.h>
10
Ingo Molnar4ea21762006-07-03 00:24:53 -070011/*
12 * Initialize an rwsem:
13 */
14void __init_rwsem(struct rw_semaphore *sem, const char *name,
15 struct lock_class_key *key)
16{
17#ifdef CONFIG_DEBUG_LOCK_ALLOC
18 /*
19 * Make sure we are not reinitializing a held semaphore:
20 */
21 debug_check_no_locks_freed((void *)sem, sizeof(*sem));
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -040022 lockdep_init_map(&sem->dep_map, name, key, 0);
Ingo Molnar4ea21762006-07-03 00:24:53 -070023#endif
24 sem->count = RWSEM_UNLOCKED_VALUE;
25 spin_lock_init(&sem->wait_lock);
26 INIT_LIST_HEAD(&sem->wait_list);
27}
28
29EXPORT_SYMBOL(__init_rwsem);
30
Linus Torvalds1da177e2005-04-16 15:20:36 -070031struct rwsem_waiter {
32 struct list_head list;
33 struct task_struct *task;
34 unsigned int flags;
35#define RWSEM_WAITING_FOR_READ 0x00000001
36#define RWSEM_WAITING_FOR_WRITE 0x00000002
37};
38
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -070039/* Wake types for __rwsem_do_wake(). Note that RWSEM_WAKE_NO_ACTIVE and
40 * RWSEM_WAKE_READ_OWNED imply that the spinlock must have been kept held
41 * since the rwsem value was observed.
42 */
43#define RWSEM_WAKE_ANY 0 /* Wake whatever's at head of wait list */
44#define RWSEM_WAKE_NO_ACTIVE 1 /* rwsem was observed with no active thread */
45#define RWSEM_WAKE_READ_OWNED 2 /* rwsem was observed to be read owned */
46
Linus Torvalds1da177e2005-04-16 15:20:36 -070047/*
48 * handle the lock release when processes blocked on it that can now run
49 * - if we come here from up_xxxx(), then:
50 * - the 'active part' of count (&0x0000ffff) reached 0 (but may have changed)
51 * - the 'waiting part' of count (&0xffff0000) is -ve (and will still be so)
Michel Lespinasse345af7b2010-08-09 17:21:15 -070052 * - there must be someone on the queue
Linus Torvalds1da177e2005-04-16 15:20:36 -070053 * - the spinlock must be held by the caller
54 * - woken process blocks are discarded from the list after having task zeroed
55 * - writers are only woken if downgrading is false
56 */
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -070057static struct rw_semaphore *
58__rwsem_do_wake(struct rw_semaphore *sem, int wake_type)
Linus Torvalds1da177e2005-04-16 15:20:36 -070059{
60 struct rwsem_waiter *waiter;
61 struct task_struct *tsk;
62 struct list_head *next;
Michel Lespinassefd41b332010-08-09 17:21:18 -070063 signed long oldcount, woken, loop, adjustment;
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
Michel Lespinasse345af7b2010-08-09 17:21:15 -070065 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
66 if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE))
67 goto readers_only;
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -070069 if (wake_type == RWSEM_WAKE_READ_OWNED)
Michel Lespinasse424acaa2010-08-09 17:21:19 -070070 /* Another active reader was observed, so wakeup is not
71 * likely to succeed. Save the atomic op.
72 */
Michel Lespinasse345af7b2010-08-09 17:21:15 -070073 goto out;
74
75 /* There's a writer at the front of the queue - try to grant it the
76 * write lock. However, we only wake this writer if we can transition
77 * the active part of the count from 0 -> 1
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 */
Michel Lespinassefd41b332010-08-09 17:21:18 -070079 adjustment = RWSEM_ACTIVE_WRITE_BIAS;
80 if (waiter->list.next == &sem->wait_list)
81 adjustment -= RWSEM_WAITING_BIAS;
82
Michel Lespinasse345af7b2010-08-09 17:21:15 -070083 try_again_write:
Michel Lespinassefd41b332010-08-09 17:21:18 -070084 oldcount = rwsem_atomic_update(adjustment, sem) - adjustment;
Linus Torvalds1da177e2005-04-16 15:20:36 -070085 if (oldcount & RWSEM_ACTIVE_MASK)
Michel Lespinasse345af7b2010-08-09 17:21:15 -070086 /* Someone grabbed the sem already */
87 goto undo_write;
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
89 /* We must be careful not to touch 'waiter' after we set ->task = NULL.
90 * It is an allocated on the waiter's stack and may become invalid at
91 * any time after that point (due to a wakeup from another source).
92 */
93 list_del(&waiter->list);
94 tsk = waiter->task;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -070095 smp_mb();
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 waiter->task = NULL;
97 wake_up_process(tsk);
98 put_task_struct(tsk);
99 goto out;
100
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 readers_only:
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -0700102 /* If we come here from up_xxxx(), another thread might have reached
103 * rwsem_down_failed_common() before we acquired the spinlock and
104 * woken up a waiter, making it now active. We prefer to check for
105 * this first in order to not spend too much time with the spinlock
106 * held if we're not going to be able to wake up readers in the end.
107 *
108 * Note that we do not need to update the rwsem count: any writer
109 * trying to acquire rwsem will run rwsem_down_write_failed() due
110 * to the waiting threads and block trying to acquire the spinlock.
111 *
112 * We use a dummy atomic update in order to acquire the cache line
113 * exclusively since we expect to succeed and run the final rwsem
114 * count adjustment pretty soon.
115 */
116 if (wake_type == RWSEM_WAKE_ANY &&
Michel Lespinasse424acaa2010-08-09 17:21:19 -0700117 rwsem_atomic_update(0, sem) < RWSEM_WAITING_BIAS)
118 /* Someone grabbed the sem for write already */
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -0700119 goto out;
Michel Lespinasse345af7b2010-08-09 17:21:15 -0700120
Michel Lespinasse345af7b2010-08-09 17:21:15 -0700121 /* Grant an infinite number of read locks to the readers at the front
122 * of the queue. Note we increment the 'active part' of the count by
123 * the number of readers before waking any processes up.
124 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125 woken = 0;
126 do {
127 woken++;
128
129 if (waiter->list.next == &sem->wait_list)
130 break;
131
132 waiter = list_entry(waiter->list.next,
133 struct rwsem_waiter, list);
134
135 } while (waiter->flags & RWSEM_WAITING_FOR_READ);
136
Michel Lespinassefd41b332010-08-09 17:21:18 -0700137 adjustment = woken * RWSEM_ACTIVE_READ_BIAS;
138 if (waiter->flags & RWSEM_WAITING_FOR_READ)
139 /* hit end of list above */
140 adjustment -= RWSEM_WAITING_BIAS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141
Michel Lespinassefd41b332010-08-09 17:21:18 -0700142 rwsem_atomic_add(adjustment, sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143
144 next = sem->wait_list.next;
Michel Lespinassefd41b332010-08-09 17:21:18 -0700145 for (loop = woken; loop > 0; loop--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 waiter = list_entry(next, struct rwsem_waiter, list);
147 next = waiter->list.next;
148 tsk = waiter->task;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700149 smp_mb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 waiter->task = NULL;
151 wake_up_process(tsk);
152 put_task_struct(tsk);
153 }
154
155 sem->wait_list.next = next;
156 next->prev = &sem->wait_list;
157
158 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 return sem;
160
Michel Lespinasse91af7082010-05-12 11:38:45 +0100161 /* undo the change to the active count, but check for a transition
162 * 1->0 */
Michel Lespinasse345af7b2010-08-09 17:21:15 -0700163 undo_write:
Michel Lespinassefd41b332010-08-09 17:21:18 -0700164 if (rwsem_atomic_update(-adjustment, sem) & RWSEM_ACTIVE_MASK)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 goto out;
Michel Lespinasse345af7b2010-08-09 17:21:15 -0700166 goto try_again_write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167}
168
169/*
170 * wait for a lock to be granted
171 */
Livio Soaresc7af77b2007-12-18 15:21:13 +0100172static struct rw_semaphore __sched *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173rwsem_down_failed_common(struct rw_semaphore *sem,
Michel Lespinassea8618a02010-08-09 17:21:20 -0700174 unsigned int flags, signed long adjustment)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175{
Michel Lespinassea8618a02010-08-09 17:21:20 -0700176 struct rwsem_waiter waiter;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 struct task_struct *tsk = current;
178 signed long count;
179
180 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
181
182 /* set up my own style of waitqueue */
183 spin_lock_irq(&sem->wait_lock);
Michel Lespinassea8618a02010-08-09 17:21:20 -0700184 waiter.task = tsk;
185 waiter.flags = flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186 get_task_struct(tsk);
187
Michel Lespinassefd41b332010-08-09 17:21:18 -0700188 if (list_empty(&sem->wait_list))
189 adjustment += RWSEM_WAITING_BIAS;
Michel Lespinassea8618a02010-08-09 17:21:20 -0700190 list_add_tail(&waiter.list, &sem->wait_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -0700192 /* we're now waiting on the lock, but no longer actively locking */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 count = rwsem_atomic_update(adjustment, sem);
194
Michel Lespinasse424acaa2010-08-09 17:21:19 -0700195 /* If there are no active locks, wake the front queued process(es) up.
196 *
197 * Alternatively, if we're called from a failed down_write(), there
198 * were already threads queued before us and there are no active
199 * writers, the lock must be read owned; so we try to wake any read
200 * locks that were queued ahead of us. */
201 if (count == RWSEM_WAITING_BIAS)
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -0700202 sem = __rwsem_do_wake(sem, RWSEM_WAKE_NO_ACTIVE);
Michel Lespinasse424acaa2010-08-09 17:21:19 -0700203 else if (count > RWSEM_WAITING_BIAS &&
204 adjustment == -RWSEM_ACTIVE_WRITE_BIAS)
205 sem = __rwsem_do_wake(sem, RWSEM_WAKE_READ_OWNED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206
207 spin_unlock_irq(&sem->wait_lock);
208
209 /* wait to be given the lock */
210 for (;;) {
Michel Lespinassea8618a02010-08-09 17:21:20 -0700211 if (!waiter.task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 break;
213 schedule();
214 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
215 }
216
217 tsk->state = TASK_RUNNING;
218
219 return sem;
220}
221
222/*
223 * wait for the read lock to be granted
224 */
Thomas Gleixnerd1233752011-01-26 21:32:01 +0100225struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226{
Michel Lespinassea8618a02010-08-09 17:21:20 -0700227 return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_READ,
228 -RWSEM_ACTIVE_READ_BIAS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229}
230
231/*
232 * wait for the write lock to be granted
233 */
Thomas Gleixnerd1233752011-01-26 21:32:01 +0100234struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235{
Michel Lespinassea8618a02010-08-09 17:21:20 -0700236 return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_WRITE,
237 -RWSEM_ACTIVE_WRITE_BIAS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238}
239
240/*
241 * handle waking up a waiter on the semaphore
242 * - up_read/up_write has decremented the active part of count if we come here
243 */
Thomas Gleixnerd1233752011-01-26 21:32:01 +0100244struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245{
246 unsigned long flags;
247
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 spin_lock_irqsave(&sem->wait_lock, flags);
249
250 /* do nothing if list empty */
251 if (!list_empty(&sem->wait_list))
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -0700252 sem = __rwsem_do_wake(sem, RWSEM_WAKE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253
254 spin_unlock_irqrestore(&sem->wait_lock, flags);
255
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 return sem;
257}
258
259/*
260 * downgrade a write lock into a read lock
261 * - caller incremented waiting part of count and discovered it still negative
262 * - just wake up any readers at the front of the queue
263 */
Thomas Gleixnerd1233752011-01-26 21:32:01 +0100264struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265{
266 unsigned long flags;
267
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 spin_lock_irqsave(&sem->wait_lock, flags);
269
270 /* do nothing if list empty */
271 if (!list_empty(&sem->wait_list))
Michel Lespinasse70bdc6e2010-08-09 17:21:17 -0700272 sem = __rwsem_do_wake(sem, RWSEM_WAKE_READ_OWNED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273
274 spin_unlock_irqrestore(&sem->wait_lock, flags);
275
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 return sem;
277}
278
279EXPORT_SYMBOL(rwsem_down_read_failed);
280EXPORT_SYMBOL(rwsem_down_write_failed);
281EXPORT_SYMBOL(rwsem_wake);
282EXPORT_SYMBOL(rwsem_downgrade_wake);