osc/rdma : fence mechanism for btls do not support remote completion

Currently, fence is implemented by two steps:

First, waiting for RDMA operiatons to complete locally.
Second, call coll->barrier on the communicator.

This is correct only if selected BTL support remote completion.
Otherwise, it can happen that when coll->barrier() finished, the remote
side of the RDMA operation has not completed yet.

This patch implemented a different barrier mechanism, which is used
when any of the selected BTL does not support remote completion.

In which case, each process will post an atomic operation to every
peer to increase a counter on the peer through the selected BTL endpoint.
Though wait for its own counter to reach number of peers. This ensures
all previous RDMA operations have completed.

Signed-off-by: Wei Zhang <[email protected]>

Author: wzamazon

ompi/mca/osc/rdma/osc_rdma.h

@@ -262,6 +262,12 @@ struct ompi_osc_rdma_module_t {
      */
     bool btl_support_remote_completion;
 
+    /** ordering requirement on selected btls.
+     * If all btls support remote completion, btl_order is MCA_BTL_NO_ORDER;
+     * otherwise, it will be MCA_BTL_IN_ORDER_RDMA_ATOMICS;
+     */
+    int btl_order;
+
     /** array of peer state. Used when local leader is NOT used */
     uintptr_t *peer_state_array;
 

ompi/mca/osc/rdma/osc_rdma_accumulate.c

@@ -758,7 +758,7 @@ static inline int cas_rdma (ompi_osc_rdma_sync_t *sync, const void *source_addr,
 
     do {
         ret = btl->btl_put (btl, peer->data_endpoint, ptr, target_address,
-                            local_handle, target_handle, len, 0, MCA_BTL_NO_ORDER,
+                            local_handle, target_handle, len, 0, module->btl_order,
                             ompi_osc_rdma_cas_put_complete, (void *) &complete, NULL);
         if (OPAL_SUCCESS == ret || (OPAL_ERR_OUT_OF_RESOURCE != ret && OPAL_ERR_TEMP_OUT_OF_RESOURCE != ret)) {
             break;

ompi/mca/osc/rdma/osc_rdma_active_target.c

@@ -357,6 +357,7 @@ int ompi_osc_rdma_post_atomic (ompi_group_t *group, int mpi_assert, ompi_win_t *
     return ret;
 }
 
+
 int ompi_osc_rdma_start_atomic (ompi_group_t *group, int mpi_assert, ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
@@ -590,6 +591,82 @@ int ompi_osc_rdma_test_atomic (ompi_win_t *win, int *flag)
     return OMPI_SUCCESS;
 }
 
+/**
+ * This function implements a different barrier mechanism for Fence,
+ * when any of the selected btl does not support remote completion.
+ * This barrier is based on imposing the MCA_BTL_ORDER_RDMA_ATOMCS
+ * ordering requirement on seleted btls.
+ */
+static
+int ompi_osc_rdma_fence_barrier_by_ordered_channel (ompi_win_t *win)
+{
+    ompi_osc_rdma_module_t *module = GET_MODULE(win);
+    ompi_osc_rdma_state_t *state = module->state;
+    ompi_osc_rdma_sync_t *sync = &module->all_sync;
+    ompi_osc_rdma_peer_t **peers;
+    ompi_group_t *group;
+    int num_peers;
+    int ret;
+
+    assert(module->btl_order == MCA_BTL_IN_ORDER_RDMA_ATOMICS);
+    OPAL_THREAD_LOCK(&module->lock);
+
+    if (ompi_comm_size(module->comm) == 1) {
+        OPAL_THREAD_UNLOCK(&(module->lock));
+        return OMPI_SUCCESS;
+    }
+
+    ret = ompi_comm_group(module->comm, &group);
+    if (OMPI_SUCCESS != ret) {
+        OPAL_THREAD_UNLOCK(&(module->lock));
+	return ret;
+    }
+
+    num_peers = sync->num_peers;
+    assert(ompi_group_size(group) == num_peers);
+    peers = ompi_osc_rdma_get_peers(module, group);
+    if (NULL == peers) {
+        OPAL_THREAD_UNLOCK(&(module->lock));
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    module->state->num_fenced_peers = 0;
+    OPAL_THREAD_UNLOCK(&(module->lock));
+    ret = module->comm->c_coll->coll_barrier(module->comm, module->comm->c_coll->coll_barrier_module);
+    if (ret) {
+        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "barrier failed!");
+        return ret;
+    }
+
+    /* for each process in the group increment their number of fenced peers */
+    for (int i = 0 ; i < num_peers; ++i) {
+        ompi_osc_rdma_peer_t *peer = peers[i];
+        intptr_t target = (intptr_t) peer->state + offsetof (ompi_osc_rdma_state_t, num_fenced_peers);
+
+	/* the usage of peer local state requires selected btls to support remote completion,
+	 * if that is the case, this function will not have been called
+	 */
+        assert (!ompi_osc_rdma_peer_local_state (peer));
+        ret = ompi_osc_rdma_lock_btl_op (module, peer, target, MCA_BTL_ATOMIC_ADD, 1, true);
+        if (OMPI_SUCCESS != ret) {
+            return ret;
+        }
+    }
+
+    ompi_osc_rdma_release_peers (peers, num_peers);
+    ompi_group_free (&group);
+
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "increased fenced_peer counter of all peers");
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "waiting for all peers to increase my counter");
+    while (num_peers != state->num_fenced_peers) {
+        ompi_osc_rdma_progress (module);
+        opal_atomic_mb ();
+    }
+
+    OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "received fence message from all peers");
+    return OMPI_SUCCESS;
+}
+
 int ompi_osc_rdma_fence_atomic (int mpi_assert, ompi_win_t *win)
 {
     ompi_osc_rdma_module_t *module = GET_MODULE(win);
@@ -627,7 +704,18 @@ int ompi_osc_rdma_fence_atomic (int mpi_assert, ompi_win_t *win)
     ompi_osc_rdma_sync_rdma_complete (&module->all_sync);
 
     /* ensure all writes to my memory are complete (both local stores, and RMA operations) */
-    ret = module->comm->c_coll->coll_barrier(module->comm, module->comm->c_coll->coll_barrier_module);
+    if (module->btl_support_remote_completion) {
+        /* if all selected btls support remote completion, then all RMA operations have finished
+	 * on remote side. A barrier is enough to complete the fence.
+	 */
+        ret = module->comm->c_coll->coll_barrier(module->comm, module->comm->c_coll->coll_barrier_module);
+    } else {
+        /*
+	 * if any selected btl does not support remote completion, we will have to send a completion
+	 * message (through the same endpoint of data transfer) to every peer, then wait for a message from every peer.
+	 */
+        ret = ompi_osc_rdma_fence_barrier_by_ordered_channel(win);
+    }
 
     if (mpi_assert & MPI_MODE_NOSUCCEED) {
         /* as specified in MPI-3 p 438 3-5 the fence can end an epoch. it isn't explicitly

ompi/mca/osc/rdma/osc_rdma_comm.c

@@ -97,7 +97,7 @@ int ompi_osc_get_data_blocking (ompi_osc_rdma_module_t *module, uint8_t btl_inde
 
     do {
         ret = btl->btl_get (btl, endpoint, ptr, aligned_addr,
-                            local_handle, source_handle, aligned_len, 0, MCA_BTL_NO_ORDER,
+                            local_handle, source_handle, aligned_len, 0, module->btl_order,
                             ompi_osc_get_data_complete, (void *) &read_complete, NULL);
         if (!ompi_osc_rdma_oor (ret)) {
             break;
@@ -455,7 +455,7 @@ static int ompi_osc_rdma_put_real (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_pee
 
     do {
         ret = btl->btl_put (btl, peer->data_endpoint, ptr, target_address,
-                            local_handle, target_handle, size, 0, MCA_BTL_NO_ORDER,
+                            local_handle, target_handle, size, 0, module->btl_order,
                             cb, context, cbdata);
         if (OPAL_UNLIKELY(OMPI_SUCCESS == ret)) {
             return OMPI_SUCCESS;
@@ -705,7 +705,7 @@ static int ompi_osc_rdma_get_contig (ompi_osc_rdma_sync_t *sync, ompi_osc_rdma_p
     do {
         ret = btl->btl_get (btl, peer->data_endpoint, ptr,
                             aligned_source_base, local_handle, source_handle,
-                            aligned_len, 0, MCA_BTL_NO_ORDER, ompi_osc_rdma_get_complete,
+                            aligned_len, 0, module->btl_order, ompi_osc_rdma_get_complete,
                             request, frag);
         if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
             return OMPI_SUCCESS;

ompi/mca/osc/rdma/osc_rdma_component.c

@@ -926,7 +926,7 @@ static int allocate_module_state(ompi_osc_rdma_module_t *module, void **base, si
         return ret;
 
     if (!module->btl_support_remote_completion) {
-        /* if btl does not support remote_completion, the local leader optimization
+        /* if any selected btl does not support remote_completion, the local leader optimization
 	 * cannot be used, which means each endpoint will communicate with the peer
 	 * directly to update its state (instead of through the peer's local leader).
 	 * Therefore each endpoint need to have the state pointer of each peer.
@@ -936,6 +936,15 @@ static int allocate_module_state(ompi_osc_rdma_module_t *module, void **base, si
         if (OPAL_UNLIKELY(OMPI_SUCCESS !=ret)) {
             return ret;
         }
+
+	/* If any selected btl does not support remote completeion, fence cannot use
+	 * the MPI_barrier based implementation, but have to use an implementation that
+	 * is based on orderedd RDMA and ATOMICS operation, for that we have to impose
+	 * the ordering requirement for each RDMA and ATOMIC operation
+	 */
+	module->btl_order = MCA_BTL_IN_ORDER_RDMA_ATOMICS;
+    } else {
+        module->btl_order = MCA_BTL_NO_ORDER;
     }
 }
 

ompi/mca/osc/rdma/osc_rdma_lock.h

@@ -74,7 +74,7 @@ static inline int ompi_osc_rdma_btl_fop (ompi_osc_rdma_module_t *module, uint8_t
         if (NULL != pending_op->op_frag) {
             ret = selected_btl->btl_atomic_fop (selected_btl, endpoint, pending_op->op_buffer,
                                                 (intptr_t) address, pending_op->op_frag->handle, address_handle,
-                                                op, operand, flags, MCA_BTL_NO_ORDER, ompi_osc_rdma_atomic_complete,
+                                                op, operand, flags, module->btl_order, ompi_osc_rdma_atomic_complete,
                                                 (void *) pending_op, NULL);
         }
 
@@ -148,7 +148,7 @@ static inline int ompi_osc_rdma_btl_op (ompi_osc_rdma_module_t *module, uint8_t
     /* spin until the btl has accepted the operation */
     do {
         ret = selected_btl->btl_atomic_op (selected_btl, endpoint, (intptr_t) address, address_handle,
-                                           op, operand, flags, MCA_BTL_NO_ORDER, ompi_osc_rdma_atomic_complete,
+                                           op, operand, flags, module->btl_order, ompi_osc_rdma_atomic_complete,
                                            (void *) pending_op, NULL);
 
         if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {

ompi/mca/osc/rdma/osc_rdma_types.h

@@ -136,6 +136,12 @@ struct ompi_osc_rdma_state_t {
     osc_rdma_atomic_counter_t num_post_msgs;
     /** counter for number of complete messages received */
     osc_rdma_counter_t num_complete_msgs;
+    /** counter for number of fenced peers. This counter is used
+     * when any seleted btl does NOT support remote complete.
+     * In which case, a process will increase this counter on
+     * all its peers, then wait for all its peer to increase
+     * its counter. */
+    osc_rdma_counter_t num_fenced_peers;
     /** lock for the region state to ensure consistency */
     ompi_osc_rdma_lock_t regions_lock;
     /** displacement unit for this process */