opal/atomic: always use C11 atomics if available

hjelmn · hjelmn · commit 6786d1d5d8b4 · 2018-07-17T09:17:11.000-06:00
This commit disables the use of both the builtin and hand-written
atomics if proper C11 atomic support is detected. This is the first
step towards requiring the availability of C11 atomics for the C
compiler used to build Open MPI.

Signed-off-by: Nathan Hjelm &lt;hjelmn@lanl.gov&gt;
diff --git a/opal/class/opal_object.c b/opal/class/opal_object.c
@@ -55,7 +55,7 @@ int opal_class_init_epoch = 1;
 /*
  * Local variables
  */
-static opal_atomic_lock_t class_lock = { { OPAL_ATOMIC_LOCK_UNLOCKED } };
+static opal_atomic_lock_t class_lock = OPAL_ATOMIC_LOCK_INIT;
 static void** classes = NULL;
 static int num_classes = 0;
 static int max_classes = 0;
diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h
@@ -58,6 +58,12 @@
 #include "opal/sys/architecture.h"
 #include "opal_stdatomic.h"
 
+#if OPAL_C_HAVE__ATOMIC
+
+#include "atomic_stdc.h"
+
+#else /* !OPAL_C_HAVE__ATOMIC */
+
 /* do some quick #define cleanup in cases where we are doing
    testing... */
 #ifdef OPAL_DISABLE_INLINE_ASM
@@ -643,6 +649,8 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, vo
  */
 #include "opal/sys/atomic_impl.h"
 
+#endif /* !OPAL_C_HAVE__ATOMIC */
+
 END_C_DECLS
 
 #endif /* OPAL_SYS_ATOMIC_H */
diff --git a/opal/include/opal/sys/atomic_stdc.h b/opal/include/opal/sys/atomic_stdc.h
@@ -0,0 +1,235 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/* This file provides shims between the opal atomics interface and the C11 atomics interface. It
+ * is intended as the first step in moving to using C11 atomics across the entire codebase. Once
+ * all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then
+ * this shim will go away and the codebase will be updated to use C11's atomic support
+ * directly.
+ * This shim contains some functions already present in atomic_impl.h because we do not include
+ * atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate
+ * definitions to be worthwhile. */
+
+#if !defined(OPAL_ATOMIC_STDC_H)
+#define OPAL_ATOMIC_STDC_H
+
+#include <stdatomic.h>
+#include <stdint.h>
+#include "opal/include/opal_stdint.h"
+
+#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
+
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
+#define OPAL_HAVE_ATOMIC_SWAP_32 1
+
+#define OPAL_HAVE_ATOMIC_MATH_32 1
+#define OPAL_HAVE_ATOMIC_ADD_32 1
+#define OPAL_HAVE_ATOMIC_AND_32 1
+#define OPAL_HAVE_ATOMIC_OR_32 1
+#define OPAL_HAVE_ATOMIC_XOR_32 1
+#define OPAL_HAVE_ATOMIC_SUB_32 1
+
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
+#define OPAL_HAVE_ATOMIC_SWAP_64 1
+
+#define OPAL_HAVE_ATOMIC_MATH_64 1
+#define OPAL_HAVE_ATOMIC_ADD_64 1
+#define OPAL_HAVE_ATOMIC_AND_64 1
+#define OPAL_HAVE_ATOMIC_OR_64 1
+#define OPAL_HAVE_ATOMIC_XOR_64 1
+#define OPAL_HAVE_ATOMIC_SUB_64 1
+
+#define OPAL_HAVE_ATOMIC_LLSC_32 0
+#define OPAL_HAVE_ATOMIC_LLSC_64 0
+#define OPAL_HAVE_ATOMIC_LLSC_PTR 0
+
+#define OPAL_HAVE_ATOMIC_MIN_32 1
+#define OPAL_HAVE_ATOMIC_MAX_32 1
+
+#define OPAL_HAVE_ATOMIC_MIN_64 1
+#define OPAL_HAVE_ATOMIC_MAX_64 1
+
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
+
+#define OPAL_HAVE_ATOMIC_SPINLOCKS 1
+
+static inline void opal_atomic_mb (void)
+{
+    atomic_thread_fence (memory_order_seq_cst);
+}
+
+static inline void opal_atomic_wmb (void)
+{
+    atomic_thread_fence (memory_order_release);
+}
+
+static inline void opal_atomic_rmb (void)
+{
+    atomic_thread_fence (memory_order_acquire);
+}
+
+#define opal_atomic_compare_exchange_strong_32 atomic_compare_exchange_strong
+#define opal_atomic_compare_exchange_strong_64 atomic_compare_exchange_strong
+#define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong
+#define opal_atomic_compare_exchange_strong_ptr atomic_compare_exchange_strong
+#define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
+
+#define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
+
+#define opal_atomic_compare_exchange_strong atomic_compare_exchange_strong
+#define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval)  atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval)  atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed)
+
+#define opal_atomic_swap_32 atomic_exchange
+#define opal_atomic_swap_64 atomic_exchange
+#define opal_atomic_swap_ptr atomic_exchange
+
+#define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator)      \
+    static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \
+    {                                                                   \
+        return atomic_fetch_ ## op (addr, value);      \
+    }                                                                   \
+                                                                        \
+    static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \
+    {                                                                   \
+        return atomic_fetch_ ## op (addr, value) operator value; \
+    }
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +)
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -)
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |)
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^)
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &)
+
+#define opal_atomic_add (void) atomic_fetch_add
+
+static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = *addr;
+    do {
+        if (old <= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
+
+    return old;
+}
+
+static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = *addr;
+    do {
+        if (old >= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
+
+    return old;
+}
+
+static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = *addr;
+    do {
+        if (old <= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
+
+    return old;
+}
+
+static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = *addr;
+    do {
+        if (old >= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
+
+    return old;
+}
+
+static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = opal_atomic_fetch_min_32 (addr, value);
+    return old <= value ? old : value;
+}
+
+static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = opal_atomic_fetch_max_32 (addr, value);
+    return old >= value ? old : value;
+}
+
+static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = opal_atomic_fetch_min_64 (addr, value);
+    return old <= value ? old : value;
+}
+
+static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = opal_atomic_fetch_max_64 (addr, value);
+    return old >= value ? old : value;
+}
+
+#define OPAL_ATOMIC_LOCK_UNLOCKED false
+#define OPAL_ATOMIC_LOCK_LOCKED true
+
+#define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT
+
+typedef atomic_flag opal_atomic_lock_t;
+
+/*
+ * Lock initialization function. It set the lock to UNLOCKED.
+ */
+static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value)
+{
+    atomic_flag_clear (lock);
+}
+
+
+static inline int opal_atomic_trylock (opal_atomic_lock_t *lock)
+{
+    return (int) atomic_flag_test_and_set (lock);
+}
+
+
+static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
+{
+    while (opal_atomic_trylock (lock)) {
+    }
+}
+
+
+static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
+{
+    atomic_flag_clear (lock);
+}
+
+#endif /* !defined(OPAL_ATOMIC_STDC_H) */
diff --git a/opal/include/opal_stdatomic.h b/opal/include/opal_stdatomic.h
@@ -14,6 +14,9 @@
 
 #include "opal_stdint.h"
 
+#if !OPAL_C_HAVE__ATOMIC
+
+typedef volatile int opal_atomic_int_t;
 typedef volatile long opal_atomic_long_t;
 
 typedef volatile int32_t opal_atomic_int32_t;
@@ -22,6 +25,7 @@ typedef volatile int64_t opal_atomic_int64_t;
 typedef volatile uint64_t opal_atomic_uint64_t;
 
 typedef volatile size_t opal_atomic_size_t;
+typedef volatile ssize_t opal_atomic_ssize_t;
 typedef volatile intptr_t opal_atomic_intptr_t;
 typedef volatile uintptr_t opal_atomic_uintptr_t;
 
@@ -31,5 +35,29 @@ typedef volatile opal_int128_t opal_atomic_int128_t;
 
 #endif
 
+#else /* OPAL_HAVE_C__ATOMIC */
+
+#include <stdatomic.h>
+
+typedef atomic_int opal_atomic_int_t;
+typedef atomic_long opal_atomic_long_t;
+
+typedef _Atomic int32_t opal_atomic_int32_t;
+typedef _Atomic uint32_t opal_atomic_uint32_t;
+typedef _Atomic int64_t opal_atomic_int64_t;
+typedef _Atomic uint64_t opal_atomic_uint64_t;
+
+typedef _Atomic size_t opal_atomic_size_t;
+typedef _Atomic ssize_t opal_atomic_ssize_t;
+typedef _Atomic intptr_t opal_atomic_intptr_t;
+typedef _Atomic uintptr_t opal_atomic_uintptr_t;
+
+#ifdef HAVE_OPAL_INT128_T
+
+typedef _Atomic opal_int128_t opal_atomic_int128_t;
+
+#endif
+
+#endif /* OPAL_HAVE_C__ATOMIC */
 
 #endif /* !defined(OPAL_STDATOMIC_H) */
diff --git a/opal/include/opal_stdint.h b/opal/include/opal_stdint.h
@@ -28,6 +28,8 @@
 #ifndef OPAL_STDINT_H
 #define OPAL_STDINT_H 1
 
+#include "opal_config.h"
+
 /*
  * Include what we can and define what is missing.
  */
diff --git a/opal/mca/btl/vader/btl_vader_component.c b/opal/mca/btl/vader/btl_vader_component.c
@@ -365,6 +365,7 @@ static int mca_btl_base_vader_modex_send (void)
     return rc;
 }
 
+#if OPAL_BTL_VADER_HAVE_XPMEM || OPAL_BTL_VADER_HAVE_CMA || OPAL_BTL_VADER_HAVE_KNEM
 static void mca_btl_vader_select_next_single_copy_mechanism (void)
 {
     for (int i = 0 ; single_copy_mechanisms[i].value != MCA_BTL_VADER_NONE ; ++i) {
@@ -374,10 +375,13 @@ static void mca_btl_vader_select_next_single_copy_mechanism (void)
         }
     }
 }
+#endif
 
 static void mca_btl_vader_check_single_copy (void)
 {
+#if OPAL_BTL_VADER_HAVE_XPMEM || OPAL_BTL_VADER_HAVE_CMA || OPAL_BTL_VADER_HAVE_KNEM
     int initial_mechanism = mca_btl_vader_component.single_copy_mechanism;
+#endif
 
 #if OPAL_BTL_VADER_HAVE_XPMEM
     if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
@@ -520,7 +524,6 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
     mca_btl_vader_check_single_copy ();
 
     if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
-        const char *base_dir = opal_process_info.proc_session_dir;
         char *sm_file;
 
         rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%x.%d", mca_btl_vader_component.backing_directory,
diff --git a/opal/threads/mutex_unix.h b/opal/threads/mutex_unix.h
@@ -76,14 +76,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
         .m_lock_debug = 0,                                              \
         .m_lock_file = NULL,                                            \
         .m_lock_line = 0,                                               \
-        .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } },     \
+        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
     }
 #else
 #define OPAL_MUTEX_STATIC_INIT                                          \
     {                                                                   \
         .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
         .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER,                    \
-        .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } },     \
+        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
     }
 #endif
 
@@ -97,14 +97,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
         .m_lock_debug = 0,                                              \
         .m_lock_file = NULL,                                            \
         .m_lock_line = 0,                                               \
-        .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } },     \
+        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
     }
 #else
 #define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                \
     {                                                                   \
         .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
         .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER,     \
-        .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } },     \
+        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
     }
 #endif
 
diff --git a/orte/runtime/orte_locks.c b/orte/runtime/orte_locks.c

Original file line number	Diff line number	Diff line change
`@@ -365,6 +365,7 @@ static int mca_btl_base_vader_modex_send (void)`
`365`	`365`	`return rc;`
`366`	`366`	`}`
`367`	`367`
	`368`	`+#if OPAL_BTL_VADER_HAVE_XPMEM \|\| OPAL_BTL_VADER_HAVE_CMA \|\| OPAL_BTL_VADER_HAVE_KNEM`
`368`	`369`	`static void mca_btl_vader_select_next_single_copy_mechanism (void)`
`369`	`370`	`{`
`370`	`371`	`for (int i = 0 ; single_copy_mechanisms[i].value != MCA_BTL_VADER_NONE ; ++i) {`
`@@ -374,10 +375,13 @@ static void mca_btl_vader_select_next_single_copy_mechanism (void)`
`374`	`375`	`}`
`375`	`376`	`}`
`376`	`377`	`}`
	`378`	`+#endif`
`377`	`379`
`378`	`380`	`static void mca_btl_vader_check_single_copy (void)`
`379`	`381`	`{`
	`382`	`+#if OPAL_BTL_VADER_HAVE_XPMEM \|\| OPAL_BTL_VADER_HAVE_CMA \|\| OPAL_BTL_VADER_HAVE_KNEM`
`380`	`383`	`int initial_mechanism = mca_btl_vader_component.single_copy_mechanism;`
	`384`	`+#endif`
`381`	`385`
`382`	`386`	`#if OPAL_BTL_VADER_HAVE_XPMEM`
`383`	`387`	`if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {`
`@@ -520,7 +524,6 @@ static mca_btl_base_module_t *mca_btl_vader_component_init (int num_btls,`
`520`	`524`	`mca_btl_vader_check_single_copy ();`
`521`	`525`
`522`	`526`	`if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {`
`523`		`- const char *base_dir = opal_process_info.proc_session_dir;`
`524`	`527`	`char *sm_file;`
`525`	`528`
`526`	`529`	`rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%x.%d", mca_btl_vader_component.backing_directory,`