mutex: speed up generic mutex implementations
- atomic operations which both modify the variable and return something imply
full smp memory barriers before and after the memory operations involved
(failing atomic_cmpxchg, atomic_add_unless, etc don't imply a barrier because
they don't modify the target). See Documentation/atomic_ops.txt.
So remove extra barriers and branches.
- All architectures support atomic_cmpxchg. This has no relation to
__HAVE_ARCH_CMPXCHG. We can just take the atomic_cmpxchg path unconditionally
This reduces a simple single threaded fastpath lock+unlock test from 590 cycles
to 203 cycles on a ppc970 system.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
diff --git a/include/asm-generic/mutex-xchg.h b/include/asm-generic/mutex-xchg.h
index 7b9cd2c..580a6d3 100644
--- a/include/asm-generic/mutex-xchg.h
+++ b/include/asm-generic/mutex-xchg.h
@@ -27,8 +27,6 @@
{
if (unlikely(atomic_xchg(count, 0) != 1))
fail_fn(count);
- else
- smp_mb();
}
/**
@@ -46,10 +44,7 @@
{
if (unlikely(atomic_xchg(count, 0) != 1))
return fail_fn(count);
- else {
- smp_mb();
- return 0;
- }
+ return 0;
}
/**
@@ -67,7 +62,6 @@
static inline void
__mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *))
{
- smp_mb();
if (unlikely(atomic_xchg(count, 1) != 0))
fail_fn(count);
}
@@ -110,7 +104,6 @@
if (prev < 0)
prev = 0;
}
- smp_mb();
return prev;
}