Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* rwsem-spinlock.c: R/W semaphores: contention handling functions for |
| 2 | * generic spinlock implementation |
| 3 | * |
| 4 | * Copyright (c) 2001 David Howells (dhowells@redhat.com). |
| 5 | * - Derived partially from idea by Andrea Arcangeli <andrea@suse.de> |
| 6 | * - Derived also from comments by Linus |
| 7 | */ |
| 8 | #include <linux/rwsem.h> |
| 9 | #include <linux/sched.h> |
Paul Gortmaker | 8bc3bcc | 2011-11-16 21:29:17 -0500 | [diff] [blame] | 10 | #include <linux/export.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | |
Michel Lespinasse | e2d57f7 | 2013-05-07 06:45:49 -0700 | [diff] [blame] | 12 | enum rwsem_waiter_type { |
| 13 | RWSEM_WAITING_FOR_WRITE, |
| 14 | RWSEM_WAITING_FOR_READ |
| 15 | }; |
| 16 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | struct rwsem_waiter { |
| 18 | struct list_head list; |
| 19 | struct task_struct *task; |
Michel Lespinasse | e2d57f7 | 2013-05-07 06:45:49 -0700 | [diff] [blame] | 20 | enum rwsem_waiter_type type; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | }; |
| 22 | |
Amerigo Wang | 29671f2 | 2009-12-14 18:00:21 -0800 | [diff] [blame] | 23 | int rwsem_is_locked(struct rw_semaphore *sem) |
| 24 | { |
| 25 | int ret = 1; |
| 26 | unsigned long flags; |
| 27 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 28 | if (raw_spin_trylock_irqsave(&sem->wait_lock, flags)) { |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 29 | ret = (sem->count != 0); |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 30 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Amerigo Wang | 29671f2 | 2009-12-14 18:00:21 -0800 | [diff] [blame] | 31 | } |
| 32 | return ret; |
| 33 | } |
| 34 | EXPORT_SYMBOL(rwsem_is_locked); |
| 35 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 36 | /* |
| 37 | * initialise the semaphore |
| 38 | */ |
Ingo Molnar | 4ea2176 | 2006-07-03 00:24:53 -0700 | [diff] [blame] | 39 | void __init_rwsem(struct rw_semaphore *sem, const char *name, |
| 40 | struct lock_class_key *key) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | { |
Ingo Molnar | 4ea2176 | 2006-07-03 00:24:53 -0700 | [diff] [blame] | 42 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 43 | /* |
| 44 | * Make sure we are not reinitializing a held semaphore: |
| 45 | */ |
| 46 | debug_check_no_locks_freed((void *)sem, sizeof(*sem)); |
Peter Zijlstra | 4dfbb9d | 2006-10-11 01:45:14 -0400 | [diff] [blame] | 47 | lockdep_init_map(&sem->dep_map, name, key, 0); |
Ingo Molnar | 4ea2176 | 2006-07-03 00:24:53 -0700 | [diff] [blame] | 48 | #endif |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 49 | sem->count = 0; |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 50 | raw_spin_lock_init(&sem->wait_lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 51 | INIT_LIST_HEAD(&sem->wait_list); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 52 | } |
Amerigo Wang | 118d52d | 2009-12-14 18:00:20 -0800 | [diff] [blame] | 53 | EXPORT_SYMBOL(__init_rwsem); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 54 | |
| 55 | /* |
| 56 | * handle the lock release when processes blocked on it that can now run |
| 57 | * - if we come here, then: |
| 58 | * - the 'active count' _reached_ zero |
| 59 | * - the 'waiting count' is non-zero |
| 60 | * - the spinlock must be held by the caller |
| 61 | * - woken process blocks are discarded from the list after having task zeroed |
| 62 | * - writers are only woken if wakewrite is non-zero |
| 63 | */ |
| 64 | static inline struct rw_semaphore * |
| 65 | __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite) |
| 66 | { |
| 67 | struct rwsem_waiter *waiter; |
| 68 | struct task_struct *tsk; |
| 69 | int woken; |
| 70 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 71 | waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list); |
| 72 | |
Michel Lespinasse | e2d57f7 | 2013-05-07 06:45:49 -0700 | [diff] [blame] | 73 | if (waiter->type == RWSEM_WAITING_FOR_WRITE) { |
Michel Lespinasse | 8cf5322 | 2013-05-07 06:45:58 -0700 | [diff] [blame] | 74 | if (wakewrite) |
| 75 | /* Wake up a writer. Note that we do not grant it the |
| 76 | * lock - it will have to acquire it when it runs. */ |
| 77 | wake_up_process(waiter->task); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | goto out; |
| 79 | } |
| 80 | |
| 81 | /* grant an infinite number of read locks to the front of the queue */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | woken = 0; |
Michel Lespinasse | 8cf5322 | 2013-05-07 06:45:58 -0700 | [diff] [blame] | 83 | do { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | struct list_head *next = waiter->list.next; |
| 85 | |
| 86 | list_del(&waiter->list); |
| 87 | tsk = waiter->task; |
Davidlohr Bueso | 49e4b2b | 2015-01-30 01:14:24 -0800 | [diff] [blame] | 88 | /* |
| 89 | * Make sure we do not wakeup the next reader before |
| 90 | * setting the nil condition to grant the next reader; |
| 91 | * otherwise we could miss the wakeup on the other |
| 92 | * side and end up sleeping again. See the pairing |
| 93 | * in rwsem_down_read_failed(). |
| 94 | */ |
akpm@osdl.org | d59dd46 | 2005-05-01 08:58:47 -0700 | [diff] [blame] | 95 | smp_mb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | waiter->task = NULL; |
| 97 | wake_up_process(tsk); |
| 98 | put_task_struct(tsk); |
| 99 | woken++; |
Michel Lespinasse | 8cf5322 | 2013-05-07 06:45:58 -0700 | [diff] [blame] | 100 | if (next == &sem->wait_list) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | break; |
| 102 | waiter = list_entry(next, struct rwsem_waiter, list); |
Michel Lespinasse | 8cf5322 | 2013-05-07 06:45:58 -0700 | [diff] [blame] | 103 | } while (waiter->type != RWSEM_WAITING_FOR_WRITE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 105 | sem->count += woken; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | |
| 107 | out: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | return sem; |
| 109 | } |
| 110 | |
| 111 | /* |
| 112 | * wake a single writer |
| 113 | */ |
| 114 | static inline struct rw_semaphore * |
| 115 | __rwsem_wake_one_writer(struct rw_semaphore *sem) |
| 116 | { |
| 117 | struct rwsem_waiter *waiter; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | |
| 119 | waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list); |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 120 | wake_up_process(waiter->task); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | return sem; |
| 123 | } |
| 124 | |
| 125 | /* |
| 126 | * get a read lock on the semaphore |
| 127 | */ |
Harvey Harrison | 9f741cb | 2008-02-08 04:19:55 -0800 | [diff] [blame] | 128 | void __sched __down_read(struct rw_semaphore *sem) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | { |
| 130 | struct rwsem_waiter waiter; |
| 131 | struct task_struct *tsk; |
Kevin Hilman | 3eac4ab | 2010-04-07 11:52:46 -0700 | [diff] [blame] | 132 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 134 | raw_spin_lock_irqsave(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 135 | |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 136 | if (sem->count >= 0 && list_empty(&sem->wait_list)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 137 | /* granted */ |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 138 | sem->count++; |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 139 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 140 | goto out; |
| 141 | } |
| 142 | |
| 143 | tsk = current; |
| 144 | set_task_state(tsk, TASK_UNINTERRUPTIBLE); |
| 145 | |
| 146 | /* set up my own style of waitqueue */ |
| 147 | waiter.task = tsk; |
Michel Lespinasse | e2d57f7 | 2013-05-07 06:45:49 -0700 | [diff] [blame] | 148 | waiter.type = RWSEM_WAITING_FOR_READ; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 149 | get_task_struct(tsk); |
| 150 | |
| 151 | list_add_tail(&waiter.list, &sem->wait_list); |
| 152 | |
| 153 | /* we don't need to touch the semaphore struct anymore */ |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 154 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | |
| 156 | /* wait to be given the lock */ |
| 157 | for (;;) { |
| 158 | if (!waiter.task) |
| 159 | break; |
| 160 | schedule(); |
| 161 | set_task_state(tsk, TASK_UNINTERRUPTIBLE); |
| 162 | } |
| 163 | |
Davidlohr Bueso | 7310599 | 2015-01-25 23:36:04 -0800 | [diff] [blame] | 164 | __set_task_state(tsk, TASK_RUNNING); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | out: |
Ingo Molnar | c4e0511 | 2006-07-03 00:24:29 -0700 | [diff] [blame] | 166 | ; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 167 | } |
| 168 | |
| 169 | /* |
| 170 | * trylock for reading -- returns 1 if successful, 0 if contention |
| 171 | */ |
Harvey Harrison | 9f741cb | 2008-02-08 04:19:55 -0800 | [diff] [blame] | 172 | int __down_read_trylock(struct rw_semaphore *sem) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | { |
| 174 | unsigned long flags; |
| 175 | int ret = 0; |
| 176 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 178 | raw_spin_lock_irqsave(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 180 | if (sem->count >= 0 && list_empty(&sem->wait_list)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 181 | /* granted */ |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 182 | sem->count++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 183 | ret = 1; |
| 184 | } |
| 185 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 186 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 187 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | return ret; |
| 189 | } |
| 190 | |
| 191 | /* |
| 192 | * get a write lock on the semaphore |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | */ |
Michal Hocko | d479960 | 2016-04-07 17:12:26 +0200 | [diff] [blame] | 194 | int __sched __down_write_common(struct rw_semaphore *sem, int state) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 195 | { |
| 196 | struct rwsem_waiter waiter; |
| 197 | struct task_struct *tsk; |
Kevin Hilman | 3eac4ab | 2010-04-07 11:52:46 -0700 | [diff] [blame] | 198 | unsigned long flags; |
Michal Hocko | d479960 | 2016-04-07 17:12:26 +0200 | [diff] [blame] | 199 | int ret = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 201 | raw_spin_lock_irqsave(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 202 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 203 | /* set up my own style of waitqueue */ |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 204 | tsk = current; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | waiter.task = tsk; |
Michel Lespinasse | e2d57f7 | 2013-05-07 06:45:49 -0700 | [diff] [blame] | 206 | waiter.type = RWSEM_WAITING_FOR_WRITE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 207 | list_add_tail(&waiter.list, &sem->wait_list); |
| 208 | |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 209 | /* wait for someone to release the lock */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 210 | for (;;) { |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 211 | /* |
| 212 | * That is the key to support write lock stealing: allows the |
| 213 | * task already on CPU to get the lock soon rather than put |
| 214 | * itself into sleep and waiting for system woke it or someone |
| 215 | * else in the head of the wait list up. |
| 216 | */ |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 217 | if (sem->count == 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 218 | break; |
Niklas Cassel | c406096 | 2017-02-25 01:17:53 +0100 | [diff] [blame] | 219 | if (signal_pending_state(state, current)) |
| 220 | goto out_nolock; |
Michal Hocko | d479960 | 2016-04-07 17:12:26 +0200 | [diff] [blame] | 221 | set_task_state(tsk, state); |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 222 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
| 223 | schedule(); |
| 224 | raw_spin_lock_irqsave(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 225 | } |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 226 | /* got the lock */ |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 227 | sem->count = -1; |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 228 | list_del(&waiter.list); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 229 | |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 230 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Michal Hocko | d479960 | 2016-04-07 17:12:26 +0200 | [diff] [blame] | 231 | |
| 232 | return ret; |
Niklas Cassel | c406096 | 2017-02-25 01:17:53 +0100 | [diff] [blame] | 233 | |
| 234 | out_nolock: |
| 235 | list_del(&waiter.list); |
Kirill Tkhai | 5497d74 | 2017-06-16 16:44:34 +0300 | [diff] [blame] | 236 | if (!list_empty(&sem->wait_list) && sem->count >= 0) |
| 237 | __rwsem_do_wake(sem, 0); |
Niklas Cassel | c406096 | 2017-02-25 01:17:53 +0100 | [diff] [blame] | 238 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
| 239 | |
| 240 | return -EINTR; |
Michal Hocko | d479960 | 2016-04-07 17:12:26 +0200 | [diff] [blame] | 241 | } |
| 242 | |
| 243 | void __sched __down_write(struct rw_semaphore *sem) |
| 244 | { |
| 245 | __down_write_common(sem, TASK_UNINTERRUPTIBLE); |
| 246 | } |
| 247 | |
| 248 | int __sched __down_write_killable(struct rw_semaphore *sem) |
| 249 | { |
| 250 | return __down_write_common(sem, TASK_KILLABLE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 251 | } |
| 252 | |
| 253 | /* |
| 254 | * trylock for writing -- returns 1 if successful, 0 if contention |
| 255 | */ |
Harvey Harrison | 9f741cb | 2008-02-08 04:19:55 -0800 | [diff] [blame] | 256 | int __down_write_trylock(struct rw_semaphore *sem) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 257 | { |
| 258 | unsigned long flags; |
| 259 | int ret = 0; |
| 260 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 261 | raw_spin_lock_irqsave(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 262 | |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 263 | if (sem->count == 0) { |
Yuanhan Liu | 41ef8f8 | 2013-02-01 18:59:16 +0800 | [diff] [blame] | 264 | /* got the lock */ |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 265 | sem->count = -1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | ret = 1; |
| 267 | } |
| 268 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 269 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 270 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 271 | return ret; |
| 272 | } |
| 273 | |
| 274 | /* |
| 275 | * release a read lock on the semaphore |
| 276 | */ |
Harvey Harrison | 9f741cb | 2008-02-08 04:19:55 -0800 | [diff] [blame] | 277 | void __up_read(struct rw_semaphore *sem) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 278 | { |
| 279 | unsigned long flags; |
| 280 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 281 | raw_spin_lock_irqsave(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 282 | |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 283 | if (--sem->count == 0 && !list_empty(&sem->wait_list)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 284 | sem = __rwsem_wake_one_writer(sem); |
| 285 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 286 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 287 | } |
| 288 | |
| 289 | /* |
| 290 | * release a write lock on the semaphore |
| 291 | */ |
Harvey Harrison | 9f741cb | 2008-02-08 04:19:55 -0800 | [diff] [blame] | 292 | void __up_write(struct rw_semaphore *sem) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 293 | { |
| 294 | unsigned long flags; |
| 295 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 296 | raw_spin_lock_irqsave(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 297 | |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 298 | sem->count = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | if (!list_empty(&sem->wait_list)) |
| 300 | sem = __rwsem_do_wake(sem, 1); |
| 301 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 302 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 303 | } |
| 304 | |
| 305 | /* |
| 306 | * downgrade a write lock into a read lock |
| 307 | * - just wake up any readers at the front of the queue |
| 308 | */ |
Harvey Harrison | 9f741cb | 2008-02-08 04:19:55 -0800 | [diff] [blame] | 309 | void __downgrade_write(struct rw_semaphore *sem) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 310 | { |
| 311 | unsigned long flags; |
| 312 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 313 | raw_spin_lock_irqsave(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 314 | |
Peter Zijlstra | 13b9a96 | 2014-07-16 14:54:55 +0200 | [diff] [blame] | 315 | sem->count = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 316 | if (!list_empty(&sem->wait_list)) |
| 317 | sem = __rwsem_do_wake(sem, 0); |
| 318 | |
Thomas Gleixner | ddb6c9b | 2010-02-24 09:54:54 +0100 | [diff] [blame] | 319 | raw_spin_unlock_irqrestore(&sem->wait_lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 320 | } |
| 321 | |