Add atomic(9) implementations of atomic operations.
Add atomic(9) implementations of atomic operations. These are used on
FreeBSD for non-x86 architectures.
diff --git a/configure.ac b/configure.ac
index 3a7a245..f4c2506 100644
--- a/configure.ac
+++ b/configure.ac
@@ -981,6 +981,29 @@
fi
dnl ============================================================================
+dnl Check for atomic(9) operations as provided on FreeBSD.
+
+JE_COMPILABLE([atomic(9)], [
+#include <sys/types.h>
+#include <machine/atomic.h>
+#include <inttypes.h>
+], [
+ {
+ uint32_t x32 = 0;
+ volatile uint32_t *x32p = &x32;
+ atomic_fetchadd_32(x32p, 1);
+ }
+ {
+ unsigned long xlong = 0;
+ volatile unsigned long *xlongp = &xlong;
+ atomic_fetchadd_long(xlongp, 1);
+ }
+], [je_cv_atomic9])
+if test "x${je_cv_atomic9}" = "xyes" ; then
+ AC_DEFINE([JEMALLOC_ATOMIC9])
+fi
+
+dnl ============================================================================
dnl Check for atomic(3) operations as provided on Darwin.
JE_COMPILABLE([Darwin OSAtomic*()], [
@@ -1031,7 +1054,7 @@
fi
])
-if test "x${je_cv_osatomic}" != "xyes" ; then
+if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then
JE_SYNC_COMPARE_AND_SWAP_CHECK(32, 4)
JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8)
fi
diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h
index d8f6ca5..016c472 100644
--- a/include/jemalloc/internal/atomic.h
+++ b/include/jemalloc/internal/atomic.h
@@ -32,7 +32,8 @@
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
/******************************************************************************/
/* 64-bit operations. */
-#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
+#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
+# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
@@ -60,7 +61,7 @@
return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
}
-#elif (defined(__amd64__) || defined(__x86_64__))
+# elif (defined(__amd64__) || defined(__x86_64__))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
@@ -87,7 +88,29 @@
return (x);
}
-#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
+# elif (defined(JEMALLOC_ATOMIC9))
+JEMALLOC_INLINE uint64_t
+atomic_add_uint64(uint64_t *p, uint64_t x)
+{
+
+ /*
+ * atomic_fetchadd_64() doesn't exist, but we only ever use this
+ * function on LP64 systems, so atomic_fetchadd_long() will do.
+ */
+ assert(sizeof(uint64_t) == sizeof(unsigned long));
+
+ return (atomic_fetchadd_long(p, (unsigned long)x) + x);
+}
+
+JEMALLOC_INLINE uint64_t
+atomic_sub_uint64(uint64_t *p, uint64_t x)
+{
+
+ assert(sizeof(uint64_t) == sizeof(unsigned long));
+
+ return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
+}
+# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
@@ -101,8 +124,7 @@
return (__sync_sub_and_fetch(p, x));
}
-#else
-# if (LG_SIZEOF_PTR == 3)
+# else
# error "Missing implementation for 64-bit atomic operations"
# endif
#endif
@@ -164,6 +186,20 @@
return (x);
}
+#elif (defined(JEMALLOC_ATOMIC9))
+JEMALLOC_INLINE uint32_t
+atomic_add_uint32(uint32_t *p, uint32_t x)
+{
+
+ return (atomic_fetchadd_32(p, x) + x);
+}
+
+JEMALLOC_INLINE uint32_t
+atomic_sub_uint32(uint32_t *p, uint32_t x)
+{
+
+ return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
+}
#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x)
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 51d40fb..905653a 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -161,6 +161,10 @@
#endif
;
+#ifdef JEMALLOC_ATOMIC9
+#include <machine/atomic.h>
+#endif
+
#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
#include <libkern/OSAtomic.h>
#endif
diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in
index b6e5593..90baa35 100644
--- a/include/jemalloc/jemalloc_defs.h.in
+++ b/include/jemalloc/jemalloc_defs.h.in
@@ -47,6 +47,9 @@
*/
#undef CPU_SPINWAIT
+/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */
+#undef JEMALLOC_ATOMIC9
+
/*
* Defined if OSAtomic*() functions are available, as provided by Darwin, and
* documented in the atomic(3) manual page.