powerpc: Rename LWSYNC_ON_SMP to PPC_RELEASE_BARRIER, ISYNC_ON_SMP to PPC_ACQUIRE_BARRIER
For performance reasons we are about to change ISYNC_ON_SMP to sometimes be
lwsync. Now that the macro name doesn't make sense, change it and LWSYNC_ON_SMP
to better explain what the barriers are doing.
Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h
index 4012483..b8f152e 100644
--- a/arch/powerpc/include/asm/atomic.h
+++ b/arch/powerpc/include/asm/atomic.h
@@ -49,13 +49,13 @@
int t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%2 # atomic_add_return\n\
add %0,%1,%0\n"
PPC405_ERR77(0,%2)
" stwcx. %0,0,%2 \n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (a), "r" (&v->counter)
: "cc", "memory");
@@ -85,13 +85,13 @@
int t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%2 # atomic_sub_return\n\
subf %0,%1,%0\n"
PPC405_ERR77(0,%2)
" stwcx. %0,0,%2 \n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (a), "r" (&v->counter)
: "cc", "memory");
@@ -119,13 +119,13 @@
int t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%1 # atomic_inc_return\n\
addic %0,%0,1\n"
PPC405_ERR77(0,%1)
" stwcx. %0,0,%1 \n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (&v->counter)
: "cc", "xer", "memory");
@@ -163,13 +163,13 @@
int t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%1 # atomic_dec_return\n\
addic %0,%0,-1\n"
PPC405_ERR77(0,%1)
" stwcx. %0,0,%1\n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (&v->counter)
: "cc", "xer", "memory");
@@ -194,7 +194,7 @@
int t;
__asm__ __volatile__ (
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%1 # atomic_add_unless\n\
cmpw 0,%0,%3 \n\
beq- 2f \n\
@@ -202,7 +202,7 @@
PPC405_ERR77(0,%2)
" stwcx. %0,0,%1 \n\
bne- 1b \n"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
" subf %0,%2,%0 \n\
2:"
: "=&r" (t)
@@ -227,7 +227,7 @@
int t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%1 # atomic_dec_if_positive\n\
cmpwi %0,1\n\
addi %0,%0,-1\n\
@@ -235,7 +235,7 @@
PPC405_ERR77(0,%1)
" stwcx. %0,0,%1\n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
"\n\
2:" : "=&b" (t)
: "r" (&v->counter)
@@ -286,12 +286,12 @@
long t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: ldarx %0,0,%2 # atomic64_add_return\n\
add %0,%1,%0\n\
stdcx. %0,0,%2 \n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (a), "r" (&v->counter)
: "cc", "memory");
@@ -320,12 +320,12 @@
long t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: ldarx %0,0,%2 # atomic64_sub_return\n\
subf %0,%1,%0\n\
stdcx. %0,0,%2 \n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (a), "r" (&v->counter)
: "cc", "memory");
@@ -352,12 +352,12 @@
long t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: ldarx %0,0,%1 # atomic64_inc_return\n\
addic %0,%0,1\n\
stdcx. %0,0,%1 \n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (&v->counter)
: "cc", "xer", "memory");
@@ -394,12 +394,12 @@
long t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: ldarx %0,0,%1 # atomic64_dec_return\n\
addic %0,%0,-1\n\
stdcx. %0,0,%1\n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (&v->counter)
: "cc", "xer", "memory");
@@ -419,13 +419,13 @@
long t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\
addic. %0,%0,-1\n\
blt- 2f\n\
stdcx. %0,0,%1\n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
"\n\
2:" : "=&r" (t)
: "r" (&v->counter)
@@ -451,14 +451,14 @@
long t;
__asm__ __volatile__ (
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: ldarx %0,0,%1 # atomic_add_unless\n\
cmpd 0,%0,%3 \n\
beq- 2f \n\
add %0,%2,%0 \n"
" stdcx. %0,0,%1 \n\
bne- 1b \n"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
" subf %0,%2,%0 \n\
2:"
: "=&r" (t)
diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h
index 3c7c37b..30964ae 100644
--- a/arch/powerpc/include/asm/bitops.h
+++ b/arch/powerpc/include/asm/bitops.h
@@ -78,7 +78,7 @@
DEFINE_BITOP(set_bits, or, "", "")
DEFINE_BITOP(clear_bits, andc, "", "")
-DEFINE_BITOP(clear_bits_unlock, andc, LWSYNC_ON_SMP, "")
+DEFINE_BITOP(clear_bits_unlock, andc, PPC_RELEASE_BARRIER, "")
DEFINE_BITOP(change_bits, xor, "", "")
static __inline__ void set_bit(int nr, volatile unsigned long *addr)
@@ -124,10 +124,14 @@
return (old & mask); \
}
-DEFINE_TESTOP(test_and_set_bits, or, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)
-DEFINE_TESTOP(test_and_set_bits_lock, or, "", ISYNC_ON_SMP, 1)
-DEFINE_TESTOP(test_and_clear_bits, andc, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)
-DEFINE_TESTOP(test_and_change_bits, xor, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)
+DEFINE_TESTOP(test_and_set_bits, or, PPC_RELEASE_BARRIER,
+ PPC_ACQUIRE_BARRIER, 0)
+DEFINE_TESTOP(test_and_set_bits_lock, or, "",
+ PPC_ACQUIRE_BARRIER, 1)
+DEFINE_TESTOP(test_and_clear_bits, andc, PPC_RELEASE_BARRIER,
+ PPC_ACQUIRE_BARRIER, 0)
+DEFINE_TESTOP(test_and_change_bits, xor, PPC_RELEASE_BARRIER,
+ PPC_ACQUIRE_BARRIER, 0)
static __inline__ int test_and_set_bit(unsigned long nr,
volatile unsigned long *addr)
@@ -158,7 +162,7 @@
static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr)
{
- __asm__ __volatile__(LWSYNC_ON_SMP "" ::: "memory");
+ __asm__ __volatile__(PPC_RELEASE_BARRIER "" ::: "memory");
__clear_bit(nr, addr);
}
diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h
index 9696cc3..7c589ef 100644
--- a/arch/powerpc/include/asm/futex.h
+++ b/arch/powerpc/include/asm/futex.h
@@ -11,7 +11,7 @@
#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \
__asm__ __volatile ( \
- LWSYNC_ON_SMP \
+ PPC_RELEASE_BARRIER \
"1: lwarx %0,0,%2\n" \
insn \
PPC405_ERR77(0, %2) \
@@ -90,14 +90,14 @@
return -EFAULT;
__asm__ __volatile__ (
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%2 # futex_atomic_cmpxchg_inatomic\n\
cmpw 0,%0,%3\n\
bne- 3f\n"
PPC405_ERR77(0,%2)
"2: stwcx. %4,0,%2\n\
bne- 1b\n"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
"3: .section .fixup,\"ax\"\n\
4: li %0,%5\n\
b 3b\n\
diff --git a/arch/powerpc/include/asm/mutex.h b/arch/powerpc/include/asm/mutex.h
index dabc01c..5399f7e 100644
--- a/arch/powerpc/include/asm/mutex.h
+++ b/arch/powerpc/include/asm/mutex.h
@@ -15,7 +15,7 @@
PPC405_ERR77(0,%1)
" stwcx. %3,0,%1\n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
"\n\
2:"
: "=&r" (t)
@@ -35,7 +35,7 @@
PPC405_ERR77(0,%1)
" stwcx. %0,0,%1\n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (t)
: "r" (&v->counter)
: "cc", "memory");
@@ -48,7 +48,7 @@
int t;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%1 # mutex unlock\n\
addic %0,%0,1\n"
PPC405_ERR77(0,%1)
diff --git a/arch/powerpc/include/asm/spinlock.h b/arch/powerpc/include/asm/spinlock.h
index 1c35b59..f9611bd 100644
--- a/arch/powerpc/include/asm/spinlock.h
+++ b/arch/powerpc/include/asm/spinlock.h
@@ -65,9 +65,10 @@
cmpwi 0,%0,0\n\
bne- 2f\n\
stwcx. %1,0,%2\n\
- bne- 1b\n\
- isync\n\
-2:" : "=&r" (tmp)
+ bne- 1b\n"
+ PPC_ACQUIRE_BARRIER
+"2:"
+ : "=&r" (tmp)
: "r" (token), "r" (&lock->slock)
: "cr0", "memory");
@@ -145,7 +146,7 @@
{
SYNC_IO;
__asm__ __volatile__("# arch_spin_unlock\n\t"
- LWSYNC_ON_SMP: : :"memory");
+ PPC_RELEASE_BARRIER: : :"memory");
lock->slock = 0;
}
@@ -193,9 +194,9 @@
ble- 2f\n"
PPC405_ERR77(0,%1)
" stwcx. %0,0,%1\n\
- bne- 1b\n\
- isync\n\
-2:" : "=&r" (tmp)
+ bne- 1b\n"
+ PPC_ACQUIRE_BARRIER
+"2:" : "=&r" (tmp)
: "r" (&rw->lock)
: "cr0", "xer", "memory");
@@ -217,9 +218,9 @@
bne- 2f\n"
PPC405_ERR77(0,%1)
" stwcx. %1,0,%2\n\
- bne- 1b\n\
- isync\n\
-2:" : "=&r" (tmp)
+ bne- 1b\n"
+ PPC_ACQUIRE_BARRIER
+"2:" : "=&r" (tmp)
: "r" (token), "r" (&rw->lock)
: "cr0", "memory");
@@ -270,7 +271,7 @@
__asm__ __volatile__(
"# read_unlock\n\t"
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%1\n\
addic %0,%0,-1\n"
PPC405_ERR77(0,%1)
@@ -284,7 +285,7 @@
static inline void arch_write_unlock(arch_rwlock_t *rw)
{
__asm__ __volatile__("# write_unlock\n\t"
- LWSYNC_ON_SMP: : :"memory");
+ PPC_RELEASE_BARRIER: : :"memory");
rw->lock = 0;
}
diff --git a/arch/powerpc/include/asm/synch.h b/arch/powerpc/include/asm/synch.h
index 28f6ddb..5db1f0d 100644
--- a/arch/powerpc/include/asm/synch.h
+++ b/arch/powerpc/include/asm/synch.h
@@ -37,11 +37,11 @@
#endif
#ifdef CONFIG_SMP
-#define ISYNC_ON_SMP "\n\tisync\n"
-#define LWSYNC_ON_SMP stringify_in_c(LWSYNC) "\n"
+#define PPC_ACQUIRE_BARRIER "\n\tisync\n"
+#define PPC_RELEASE_BARRIER stringify_in_c(LWSYNC) "\n"
#else
-#define ISYNC_ON_SMP
-#define LWSYNC_ON_SMP
+#define PPC_ACQUIRE_BARRIER
+#define PPC_RELEASE_BARRIER
#endif
#endif /* __KERNEL__ */
diff --git a/arch/powerpc/include/asm/system.h b/arch/powerpc/include/asm/system.h
index bb8e006..aa59f5b 100644
--- a/arch/powerpc/include/asm/system.h
+++ b/arch/powerpc/include/asm/system.h
@@ -232,12 +232,12 @@
unsigned long prev;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%2 \n"
PPC405_ERR77(0,%2)
" stwcx. %3,0,%2 \n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
: "r" (p), "r" (val)
: "cc", "memory");
@@ -275,12 +275,12 @@
unsigned long prev;
__asm__ __volatile__(
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: ldarx %0,0,%2 \n"
PPC405_ERR77(0,%2)
" stdcx. %3,0,%2 \n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
: "r" (p), "r" (val)
: "cc", "memory");
@@ -366,14 +366,14 @@
unsigned int prev;
__asm__ __volatile__ (
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
cmpw 0,%0,%3\n\
bne- 2f\n"
PPC405_ERR77(0,%2)
" stwcx. %4,0,%2\n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
"\n\
2:"
: "=&r" (prev), "+m" (*p)
@@ -412,13 +412,13 @@
unsigned long prev;
__asm__ __volatile__ (
- LWSYNC_ON_SMP
+ PPC_RELEASE_BARRIER
"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
cmpd 0,%0,%3\n\
bne- 2f\n\
stdcx. %4,0,%2\n\
bne- 1b"
- ISYNC_ON_SMP
+ PPC_ACQUIRE_BARRIER
"\n\
2:"
: "=&r" (prev), "+m" (*p)