btl/base_am_rdma: fix handling of btl_send returning 1

Active message RDMA uses btl_send to send the initial
request and RDMA response.

btl_send will return 0 when the descriptor has been
successfully queued for send, and will return 1 when
the desciptor has been successfully sent.

Currently, active message RDMA treats the return value
1 as an error, thus will either return the value
to caller, or retry the send.

This patch address the issue by correctly handling
the return value 1.

Signed-off-by: Wei Zhang <[email protected]>

Author: wzamazon

opal/mca/btl/base/btl_base_am_rdma.c

@@ -372,6 +372,7 @@ static inline int mca_btl_base_am_rdma_advance(mca_btl_base_module_t *btl,
                                                mca_btl_base_rdma_context_t *context,
                                                bool send_descriptor)
 {
+    int ret;
     const size_t remaining = context->total_size - context->sent;
 
     if (0 == remaining) {
@@ -409,7 +410,19 @@ static inline int mca_btl_base_am_rdma_advance(mca_btl_base_module_t *btl,
     }
 
     if (send_descriptor) {
-        return btl->btl_send(btl, endpoint, descriptor, mca_btl_base_rdma_tag(hdr->type));
+        /* btl_send return OPAL_SUCCESS when that the descriptor has been queued for send, and
+         * descriptor's callback will be called upon completion.
+         *
+         * btl_send return 1 when the descriptor has been sent. In this case, the descriptor's
+         * callback is called only if MCA_BTL_DES_SEND_ALWAYS_CALLBACK is set on the descriptor's flags.
+         *
+         * When creating the initiator request, we specified MCA_BTL_DES_SEND_ALWAYS_CALLBACK,
+         * therefore btl_send's returning 1 is the same as it returning OPAL_SUCCESS.
+         */
+        ret = btl->btl_send(btl, endpoint, descriptor, mca_btl_base_rdma_tag(hdr->type));
+        if (ret == 1)
+            ret = OPAL_SUCCESS;
+        return ret;
     }
 
     /* queue for later to avoid btl_send in callback */
@@ -614,7 +627,19 @@ static int mca_btl_base_am_rdma_respond(mca_btl_base_module_t *btl,
 
     send_descriptor->des_cbfunc = NULL;
 
+    /* btl_send return OPAL_SUCCESS when that the descriptor has been queued for send, and
+     * descriptor's callback will be called upon completion.
+     *
+     * btl_send return 1 when the descriptor has been sent. In this case, descriptor's
+     * callback is called only if MCA_BTL_DES_SEND_ALWAYS_CALLBACK is set on the descriptor.
+     *
+     * There is no callback for the response descriptor, therefore for us btl_send
+     * return 1 is the the same as it returning OPAL_SUCCESS.
+     */
     int ret = btl->btl_send(btl, endpoint, send_descriptor, mca_btl_base_rdma_resp_tag());
+    if (ret == 1)
+        ret = OPAL_SUCCESS;
+
     if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
         *descriptor = send_descriptor;
     }
@@ -779,6 +804,18 @@ static int mca_btl_base_am_rdma_progress(void)
 
     OPAL_THREAD_SCOPED_LOCK(&default_module.mutex, ACTION1);
 
+    /* Note in ACTION2, we treated btl_send's return value of opal_success
+     * and 1 the same. This is because:
+     *
+     * btl_send return OPAL_SUCCESS when that the descriptor has been queued for send, and
+     * descriptor's callback will be called upon completion.
+     *
+     * btl_send return 1 when the descriptor has been sent and descriptor's callback has been
+     * called if MCA_BTL_DES_SEND_ALWAYS_CALLBACK is set on the descriptor's flags.
+     *
+     * When creating the initiator descriptor, we specified MCA_BTL_DES_SEND_ALWAYS_CALLBACK,
+     * therefore for us, btl_send's returning 1 is the same as it returning OPAL_SUCCESS.
+     */
 #define ACTION2                                                         \
     mca_btl_base_am_rdma_queued_descriptor_t *descriptor, *next;        \
     OPAL_LIST_FOREACH_SAFE (descriptor, next,                           \
@@ -791,7 +828,7 @@ static int mca_btl_base_am_rdma_progress(void)
                                             descriptor->endpoint,       \
                                             descriptor->descriptor,     \
                                             mca_btl_base_rdma_tag(context->type)); \
-        if (OPAL_SUCCESS == ret) {                                      \
+        if (OPAL_SUCCESS == ret || 1 == ret) {                                      \
             opal_list_remove_item(&default_module.queued_initiator_descriptors, \
                                   &descriptor->super);                  \
         }                                                               \