Will Deacon | e172800 | 2014-03-14 17:47:04 +0000 | [diff] [blame] | 1 | #ifndef _ASM_GENERIC_RWSEM_H |
| 2 | #define _ASM_GENERIC_RWSEM_H |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 3 | |
| 4 | #ifndef _LINUX_RWSEM_H |
| 5 | #error "Please don't include <asm/rwsem.h> directly, use <linux/rwsem.h> instead." |
| 6 | #endif |
| 7 | |
| 8 | #ifdef __KERNEL__ |
| 9 | |
| 10 | /* |
Will Deacon | e172800 | 2014-03-14 17:47:04 +0000 | [diff] [blame] | 11 | * R/W semaphores originally for PPC using the stuff in lib/rwsem.c. |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 12 | * Adapted largely from include/asm-i386/rwsem.h |
| 13 | * by Paul Mackerras <paulus@samba.org>. |
| 14 | */ |
| 15 | |
| 16 | /* |
| 17 | * the semaphore definition |
| 18 | */ |
Will Deacon | e172800 | 2014-03-14 17:47:04 +0000 | [diff] [blame] | 19 | #ifdef CONFIG_64BIT |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 20 | # define RWSEM_ACTIVE_MASK 0xffffffffL |
| 21 | #else |
| 22 | # define RWSEM_ACTIVE_MASK 0x0000ffffL |
| 23 | #endif |
| 24 | |
| 25 | #define RWSEM_UNLOCKED_VALUE 0x00000000L |
| 26 | #define RWSEM_ACTIVE_BIAS 0x00000001L |
| 27 | #define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) |
| 28 | #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS |
| 29 | #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) |
| 30 | |
| 31 | /* |
| 32 | * lock for reading |
| 33 | */ |
| 34 | static inline void __down_read(struct rw_semaphore *sem) |
| 35 | { |
Davidlohr Bueso | 00eb4ba | 2015-09-30 13:03:15 -0700 | [diff] [blame] | 36 | if (unlikely(atomic_long_inc_return_acquire((atomic_long_t *)&sem->count) <= 0)) |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 37 | rwsem_down_read_failed(sem); |
| 38 | } |
| 39 | |
| 40 | static inline int __down_read_trylock(struct rw_semaphore *sem) |
| 41 | { |
| 42 | long tmp; |
| 43 | |
| 44 | while ((tmp = sem->count) >= 0) { |
Davidlohr Bueso | 00eb4ba | 2015-09-30 13:03:15 -0700 | [diff] [blame] | 45 | if (tmp == cmpxchg_acquire(&sem->count, tmp, |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 46 | tmp + RWSEM_ACTIVE_READ_BIAS)) { |
| 47 | return 1; |
| 48 | } |
| 49 | } |
| 50 | return 0; |
| 51 | } |
| 52 | |
| 53 | /* |
| 54 | * lock for writing |
| 55 | */ |
Michal Hocko | f8e04d8 | 2016-04-07 17:12:21 +0200 | [diff] [blame] | 56 | static inline void __down_write(struct rw_semaphore *sem) |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 57 | { |
| 58 | long tmp; |
| 59 | |
Davidlohr Bueso | 00eb4ba | 2015-09-30 13:03:15 -0700 | [diff] [blame] | 60 | tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS, |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 61 | (atomic_long_t *)&sem->count); |
| 62 | if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS)) |
| 63 | rwsem_down_write_failed(sem); |
| 64 | } |
| 65 | |
Michal Hocko | d479960 | 2016-04-07 17:12:26 +0200 | [diff] [blame] | 66 | static inline int __down_write_killable(struct rw_semaphore *sem) |
| 67 | { |
| 68 | long tmp; |
| 69 | |
| 70 | tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS, |
| 71 | (atomic_long_t *)&sem->count); |
| 72 | if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS)) |
| 73 | if (IS_ERR(rwsem_down_write_failed_killable(sem))) |
| 74 | return -EINTR; |
| 75 | return 0; |
| 76 | } |
| 77 | |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 78 | static inline int __down_write_trylock(struct rw_semaphore *sem) |
| 79 | { |
| 80 | long tmp; |
| 81 | |
Davidlohr Bueso | 00eb4ba | 2015-09-30 13:03:15 -0700 | [diff] [blame] | 82 | tmp = cmpxchg_acquire(&sem->count, RWSEM_UNLOCKED_VALUE, |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 83 | RWSEM_ACTIVE_WRITE_BIAS); |
| 84 | return tmp == RWSEM_UNLOCKED_VALUE; |
| 85 | } |
| 86 | |
| 87 | /* |
| 88 | * unlock after reading |
| 89 | */ |
| 90 | static inline void __up_read(struct rw_semaphore *sem) |
| 91 | { |
| 92 | long tmp; |
| 93 | |
Davidlohr Bueso | 00eb4ba | 2015-09-30 13:03:15 -0700 | [diff] [blame] | 94 | tmp = atomic_long_dec_return_release((atomic_long_t *)&sem->count); |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 95 | if (unlikely(tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0)) |
| 96 | rwsem_wake(sem); |
| 97 | } |
| 98 | |
| 99 | /* |
| 100 | * unlock after writing |
| 101 | */ |
| 102 | static inline void __up_write(struct rw_semaphore *sem) |
| 103 | { |
Davidlohr Bueso | 00eb4ba | 2015-09-30 13:03:15 -0700 | [diff] [blame] | 104 | if (unlikely(atomic_long_sub_return_release(RWSEM_ACTIVE_WRITE_BIAS, |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 105 | (atomic_long_t *)&sem->count) < 0)) |
| 106 | rwsem_wake(sem); |
| 107 | } |
| 108 | |
| 109 | /* |
| 110 | * implement atomic add functionality |
| 111 | */ |
| 112 | static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) |
| 113 | { |
| 114 | atomic_long_add(delta, (atomic_long_t *)&sem->count); |
| 115 | } |
| 116 | |
| 117 | /* |
| 118 | * downgrade write lock to read lock |
| 119 | */ |
| 120 | static inline void __downgrade_write(struct rw_semaphore *sem) |
| 121 | { |
| 122 | long tmp; |
| 123 | |
Davidlohr Bueso | 00eb4ba | 2015-09-30 13:03:15 -0700 | [diff] [blame] | 124 | /* |
| 125 | * When downgrading from exclusive to shared ownership, |
| 126 | * anything inside the write-locked region cannot leak |
| 127 | * into the read side. In contrast, anything in the |
| 128 | * read-locked region is ok to be re-ordered into the |
| 129 | * write side. As such, rely on RELEASE semantics. |
| 130 | */ |
| 131 | tmp = atomic_long_add_return_release(-RWSEM_WAITING_BIAS, |
Richard Kuo | dd472da | 2011-10-31 18:47:33 -0500 | [diff] [blame] | 132 | (atomic_long_t *)&sem->count); |
| 133 | if (tmp < 0) |
| 134 | rwsem_downgrade_wake(sem); |
| 135 | } |
| 136 | |
| 137 | /* |
| 138 | * implement exchange and add functionality |
| 139 | */ |
| 140 | static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) |
| 141 | { |
| 142 | return atomic_long_add_return(delta, (atomic_long_t *)&sem->count); |
| 143 | } |
| 144 | |
| 145 | #endif /* __KERNEL__ */ |
Will Deacon | e172800 | 2014-03-14 17:47:04 +0000 | [diff] [blame] | 146 | #endif /* _ASM_GENERIC_RWSEM_H */ |