blob: 85c233d0a34003d551587c7c378bd813f64712fe [file] [log] [blame]
David S. Miller24f287e2007-10-15 16:41:44 -07001/* atomic.S: These things are too big to do inline.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 *
David S. Miller193d2aa2012-11-09 19:37:59 -08003 * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 */
5
David S. Miller8695c372012-05-11 20:33:22 -07006#include <linux/linkage.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <asm/asi.h>
David S. Miller24f287e2007-10-15 16:41:44 -07008#include <asm/backoff.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 .text
11
12 /* Two versions of the atomic routines, one that
13 * does not return a value and does not perform
14 * memory barriers, and a second which returns
15 * a value and does the barriers.
16 */
David S. Miller8695c372012-05-11 20:33:22 -070017ENTRY(atomic_add) /* %o0 = increment, %o1 = atomic_ptr */
David S. Miller24f287e2007-10-15 16:41:44 -070018 BACKOFF_SETUP(%o2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191: lduw [%o1], %g1
20 add %g1, %o0, %g7
21 cas [%o1], %g1, %g7
22 cmp %g1, %g7
David S. Miller0f581892010-08-18 22:53:26 -070023 bne,pn %icc, BACKOFF_LABEL(2f, 1b)
Linus Torvalds1da177e2005-04-16 15:20:36 -070024 nop
25 retl
26 nop
David S. Miller24f287e2007-10-15 16:41:44 -0700272: BACKOFF_SPIN(%o2, %o3, 1b)
David S. Miller8695c372012-05-11 20:33:22 -070028ENDPROC(atomic_add)
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
David S. Miller8695c372012-05-11 20:33:22 -070030ENTRY(atomic_sub) /* %o0 = decrement, %o1 = atomic_ptr */
David S. Miller24f287e2007-10-15 16:41:44 -070031 BACKOFF_SETUP(%o2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321: lduw [%o1], %g1
33 sub %g1, %o0, %g7
34 cas [%o1], %g1, %g7
35 cmp %g1, %g7
David S. Miller0f581892010-08-18 22:53:26 -070036 bne,pn %icc, BACKOFF_LABEL(2f, 1b)
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 nop
38 retl
39 nop
David S. Miller24f287e2007-10-15 16:41:44 -0700402: BACKOFF_SPIN(%o2, %o3, 1b)
David S. Miller8695c372012-05-11 20:33:22 -070041ENDPROC(atomic_sub)
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
David S. Miller8695c372012-05-11 20:33:22 -070043ENTRY(atomic_add_ret) /* %o0 = increment, %o1 = atomic_ptr */
David S. Miller24f287e2007-10-15 16:41:44 -070044 BACKOFF_SETUP(%o2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451: lduw [%o1], %g1
46 add %g1, %o0, %g7
47 cas [%o1], %g1, %g7
48 cmp %g1, %g7
David S. Miller0f581892010-08-18 22:53:26 -070049 bne,pn %icc, BACKOFF_LABEL(2f, 1b)
Mikulas Patocka6ec27472010-08-18 08:03:37 +000050 add %g1, %o0, %g1
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 retl
Mikulas Patocka6ec27472010-08-18 08:03:37 +000052 sra %g1, 0, %o0
David S. Miller24f287e2007-10-15 16:41:44 -0700532: BACKOFF_SPIN(%o2, %o3, 1b)
David S. Miller8695c372012-05-11 20:33:22 -070054ENDPROC(atomic_add_ret)
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
David S. Miller8695c372012-05-11 20:33:22 -070056ENTRY(atomic_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */
David S. Miller24f287e2007-10-15 16:41:44 -070057 BACKOFF_SETUP(%o2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581: lduw [%o1], %g1
59 sub %g1, %o0, %g7
60 cas [%o1], %g1, %g7
61 cmp %g1, %g7
David S. Miller0f581892010-08-18 22:53:26 -070062 bne,pn %icc, BACKOFF_LABEL(2f, 1b)
Mikulas Patocka6ec27472010-08-18 08:03:37 +000063 sub %g1, %o0, %g1
Linus Torvalds1da177e2005-04-16 15:20:36 -070064 retl
Mikulas Patocka6ec27472010-08-18 08:03:37 +000065 sra %g1, 0, %o0
David S. Miller24f287e2007-10-15 16:41:44 -0700662: BACKOFF_SPIN(%o2, %o3, 1b)
David S. Miller8695c372012-05-11 20:33:22 -070067ENDPROC(atomic_sub_ret)
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
David S. Miller8695c372012-05-11 20:33:22 -070069ENTRY(atomic64_add) /* %o0 = increment, %o1 = atomic_ptr */
David S. Miller24f287e2007-10-15 16:41:44 -070070 BACKOFF_SETUP(%o2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711: ldx [%o1], %g1
72 add %g1, %o0, %g7
73 casx [%o1], %g1, %g7
74 cmp %g1, %g7
David S. Miller0f581892010-08-18 22:53:26 -070075 bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
Linus Torvalds1da177e2005-04-16 15:20:36 -070076 nop
77 retl
78 nop
David S. Miller24f287e2007-10-15 16:41:44 -0700792: BACKOFF_SPIN(%o2, %o3, 1b)
David S. Miller8695c372012-05-11 20:33:22 -070080ENDPROC(atomic64_add)
Linus Torvalds1da177e2005-04-16 15:20:36 -070081
David S. Miller8695c372012-05-11 20:33:22 -070082ENTRY(atomic64_sub) /* %o0 = decrement, %o1 = atomic_ptr */
David S. Miller24f287e2007-10-15 16:41:44 -070083 BACKOFF_SETUP(%o2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841: ldx [%o1], %g1
85 sub %g1, %o0, %g7
86 casx [%o1], %g1, %g7
87 cmp %g1, %g7
David S. Miller0f581892010-08-18 22:53:26 -070088 bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 nop
90 retl
91 nop
David S. Miller24f287e2007-10-15 16:41:44 -0700922: BACKOFF_SPIN(%o2, %o3, 1b)
David S. Miller8695c372012-05-11 20:33:22 -070093ENDPROC(atomic64_sub)
Linus Torvalds1da177e2005-04-16 15:20:36 -070094
David S. Miller8695c372012-05-11 20:33:22 -070095ENTRY(atomic64_add_ret) /* %o0 = increment, %o1 = atomic_ptr */
David S. Miller24f287e2007-10-15 16:41:44 -070096 BACKOFF_SETUP(%o2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971: ldx [%o1], %g1
98 add %g1, %o0, %g7
99 casx [%o1], %g1, %g7
100 cmp %g1, %g7
David S. Miller0f581892010-08-18 22:53:26 -0700101 bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
David S. Millerb445e262005-06-27 15:42:04 -0700102 nop
Mikulas Patocka6ec27472010-08-18 08:03:37 +0000103 retl
104 add %g1, %o0, %o0
David S. Miller24f287e2007-10-15 16:41:44 -07001052: BACKOFF_SPIN(%o2, %o3, 1b)
David S. Miller8695c372012-05-11 20:33:22 -0700106ENDPROC(atomic64_add_ret)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107
David S. Miller8695c372012-05-11 20:33:22 -0700108ENTRY(atomic64_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */
David S. Miller24f287e2007-10-15 16:41:44 -0700109 BACKOFF_SETUP(%o2)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001101: ldx [%o1], %g1
111 sub %g1, %o0, %g7
112 casx [%o1], %g1, %g7
113 cmp %g1, %g7
David S. Miller0f581892010-08-18 22:53:26 -0700114 bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
David S. Millerb445e262005-06-27 15:42:04 -0700115 nop
Mikulas Patocka6ec27472010-08-18 08:03:37 +0000116 retl
117 sub %g1, %o0, %o0
David S. Miller24f287e2007-10-15 16:41:44 -07001182: BACKOFF_SPIN(%o2, %o3, 1b)
David S. Miller8695c372012-05-11 20:33:22 -0700119ENDPROC(atomic64_sub_ret)
David S. Miller193d2aa2012-11-09 19:37:59 -0800120
121ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
122 BACKOFF_SETUP(%o2)
1231: ldx [%o0], %g1
124 brlez,pn %g1, 3f
125 sub %g1, 1, %g7
126 casx [%o0], %g1, %g7
127 cmp %g1, %g7
128 bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
129 nop
1303: retl
131 sub %g1, 1, %o0
1322: BACKOFF_SPIN(%o2, %o3, 1b)
133ENDPROC(atomic64_dec_if_positive)