blob: 020bd98710a13ce2e82ff5e0741fa89308172985 [file] [log] [blame]
Nicolas Pitre823d0f42006-01-09 15:59:18 -08001/*
2 * include/asm-arm/mutex.h
3 *
4 * ARM optimized mutex locking primitives
5 *
6 * Please look into asm-generic/mutex-xchg.h for a formal definition.
7 */
8#ifndef _ASM_MUTEX_H
9#define _ASM_MUTEX_H
10
11#if __LINUX_ARM_ARCH__ < 6
12/* On pre-ARMv6 hardware the swp based implementation is the most efficient. */
13# include <asm-generic/mutex-xchg.h>
14#else
15
16/*
17 * Attempting to lock a mutex on ARMv6+ can be done with a bastardized
18 * atomic decrement (it is not a reliable atomic decrement but it satisfies
19 * the defined semantics for our purpose, while being smaller and faster
20 * than a real atomic decrement or atomic swap. The idea is to attempt
21 * decrementing the lock value only once. If once decremented it isn't zero,
22 * or if its store-back fails due to a dispute on the exclusive store, we
23 * simply bail out immediately through the slow path where the lock will be
24 * reattempted until it succeeds.
25 */
Nicolas Pitre365bf8a2006-02-08 21:19:38 +000026static inline void
Harvey Harrisonedde08f2008-02-08 04:19:57 -080027__mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))
Nicolas Pitre365bf8a2006-02-08 21:19:38 +000028{
29 int __ex_flag, __res;
Nicolas Pitre823d0f42006-01-09 15:59:18 -080030
Nicolas Pitre365bf8a2006-02-08 21:19:38 +000031 __asm__ (
32
33 "ldrex %0, [%2] \n\t"
34 "sub %0, %0, #1 \n\t"
35 "strex %1, %0, [%2] "
36
37 : "=&r" (__res), "=&r" (__ex_flag)
38 : "r" (&(count)->counter)
39 : "cc","memory" );
40
41 __res |= __ex_flag;
42 if (unlikely(__res != 0))
43 fail_fn(count);
44}
45
46static inline int
Harvey Harrisonedde08f2008-02-08 04:19:57 -080047__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
Nicolas Pitre365bf8a2006-02-08 21:19:38 +000048{
49 int __ex_flag, __res;
50
51 __asm__ (
52
53 "ldrex %0, [%2] \n\t"
54 "sub %0, %0, #1 \n\t"
55 "strex %1, %0, [%2] "
56
57 : "=&r" (__res), "=&r" (__ex_flag)
58 : "r" (&(count)->counter)
59 : "cc","memory" );
60
61 __res |= __ex_flag;
62 if (unlikely(__res != 0))
63 __res = fail_fn(count);
64 return __res;
65}
Nicolas Pitre823d0f42006-01-09 15:59:18 -080066
67/*
68 * Same trick is used for the unlock fast path. However the original value,
69 * rather than the result, is used to test for success in order to have
70 * better generated assembly.
71 */
Nicolas Pitre365bf8a2006-02-08 21:19:38 +000072static inline void
Harvey Harrisonedde08f2008-02-08 04:19:57 -080073__mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *))
Nicolas Pitre365bf8a2006-02-08 21:19:38 +000074{
75 int __ex_flag, __res, __orig;
76
77 __asm__ (
78
79 "ldrex %0, [%3] \n\t"
80 "add %1, %0, #1 \n\t"
81 "strex %2, %1, [%3] "
82
83 : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
84 : "r" (&(count)->counter)
85 : "cc","memory" );
86
87 __orig |= __ex_flag;
88 if (unlikely(__orig != 0))
89 fail_fn(count);
90}
Nicolas Pitre823d0f42006-01-09 15:59:18 -080091
92/*
93 * If the unlock was done on a contended lock, or if the unlock simply fails
94 * then the mutex remains locked.
95 */
96#define __mutex_slowpath_needs_to_unlock() 1
97
98/*
99 * For __mutex_fastpath_trylock we use another construct which could be
100 * described as a "single value cmpxchg".
101 *
102 * This provides the needed trylock semantics like cmpxchg would, but it is
103 * lighter and less generic than a true cmpxchg implementation.
104 */
105static inline int
106__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
107{
108 int __ex_flag, __res, __orig;
109
110 __asm__ (
111
Nicolas Pitre365bf8a2006-02-08 21:19:38 +0000112 "1: ldrex %0, [%3] \n\t"
113 "subs %1, %0, #1 \n\t"
114 "strexeq %2, %1, [%3] \n\t"
115 "movlt %0, #0 \n\t"
116 "cmpeq %2, #0 \n\t"
117 "bgt 1b "
Nicolas Pitre823d0f42006-01-09 15:59:18 -0800118
119 : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
120 : "r" (&count->counter)
121 : "cc", "memory" );
122
123 return __orig;
124}
125
126#endif
127#endif