btl/uct: make uct endpoints array a flexible array member

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

opal/mca/btl/uct/btl_uct_endpoint.c

@@ -17,23 +17,21 @@
 
 static void mca_btl_uct_endpoint_construct (mca_btl_uct_endpoint_t *endpoint)
 {
-    memset (endpoint->uct_eps, 0, sizeof (endpoint->uct_eps));
+    memset (endpoint->uct_eps, 0, sizeof (endpoint->uct_eps[0]) * mca_btl_uct_component.num_contexts_per_module);
     endpoint->conn_ep = NULL;
     OBJ_CONSTRUCT(&endpoint->ep_lock, opal_recursive_mutex_t);
 }
 
 static void mca_btl_uct_endpoint_destruct (mca_btl_uct_endpoint_t *endpoint)
 {
     for (int tl_index = 0 ; tl_index < 2 ; ++tl_index) {
-        for (int i = 0 ; i < MCA_BTL_UCT_MAX_WORKERS ; ++i) {
-            if (NULL != endpoint->uct_eps[tl_index][i].uct_ep) {
-                uct_ep_destroy (endpoint->uct_eps[tl_index][i].uct_ep);
+        for (int i = 0 ; i < mca_btl_uct_component.num_contexts_per_module ; ++i) {
+            if (NULL != endpoint->uct_eps[i][tl_index].uct_ep) {
+                uct_ep_destroy (endpoint->uct_eps[i][tl_index].uct_ep);
             }
         }
     }
 
-    memset (endpoint->uct_eps, 0, sizeof (endpoint->uct_eps));
-
     OBJ_DESTRUCT(&endpoint->ep_lock);
 }
 
@@ -43,12 +41,14 @@ OBJ_CLASS_INSTANCE(mca_btl_uct_endpoint_t, opal_object_t,
 
 mca_btl_base_endpoint_t *mca_btl_uct_endpoint_create (opal_proc_t *proc)
 {
-    mca_btl_uct_endpoint_t *endpoint = OBJ_NEW(mca_btl_uct_endpoint_t);
+    mca_btl_uct_endpoint_t *endpoint = calloc (1, sizeof (*endpoint) + sizeof (endpoint->uct_eps[0]) *
+                                               mca_btl_uct_component.num_contexts_per_module);
 
     if (OPAL_UNLIKELY(NULL == endpoint)) {
         return NULL;
     }
 
+    OBJ_CONSTRUCT(endpoint, mca_btl_uct_endpoint_t);
     endpoint->ep_proc = proc;
 
     return (mca_btl_base_endpoint_t *) endpoint;
@@ -295,7 +295,7 @@ static int mca_btl_uct_endpoint_connect_endpoint (mca_btl_uct_module_t *uct_btl,
 int mca_btl_uct_endpoint_connect (mca_btl_uct_module_t *uct_btl, mca_btl_uct_endpoint_t *endpoint, int context_id,
                                   void *ep_addr, int tl_index)
 {
-    mca_btl_uct_tl_endpoint_t *tl_endpoint = endpoint->uct_eps[tl_index] + context_id;
+    mca_btl_uct_tl_endpoint_t *tl_endpoint = endpoint->uct_eps[context_id] + tl_index;
     mca_btl_uct_device_context_t *tl_context = mca_btl_uct_module_get_rdma_context_specific (uct_btl, context_id);
     mca_btl_uct_tl_t *tl = (tl_index == uct_btl->rdma_tl->tl_index) ? uct_btl->rdma_tl : uct_btl->am_tl;
     uint8_t *rdma_tl_data = NULL, *conn_tl_data = NULL, *am_tl_data = NULL, *tl_data;

opal/mca/btl/uct/btl_uct_endpoint.h

@@ -37,8 +37,8 @@ static int mca_btl_uct_endpoint_test_am (mca_btl_uct_module_t *module, mca_btl_u
     int tl_index = module->am_tl->tl_index;
     int ep_index = context->context_id;
 
-    if (OPAL_LIKELY(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & endpoint->uct_eps[tl_index][ep_index].flags)) {
-        *ep_handle = endpoint->uct_eps[tl_index][ep_index].uct_ep;
+    if (OPAL_LIKELY(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & endpoint->uct_eps[ep_index][tl_index].flags)) {
+        *ep_handle = endpoint->uct_eps[ep_index][tl_index].uct_ep;
         return OPAL_SUCCESS;
     }
 
@@ -65,13 +65,13 @@ static inline int mca_btl_uct_endpoint_check (mca_btl_uct_module_t *module, mca_
     int ep_index = context->context_id;
     int rc;
 
-    if (OPAL_LIKELY(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & endpoint->uct_eps[tl_index][ep_index].flags)) {
-        *ep_handle = endpoint->uct_eps[tl_index][ep_index].uct_ep;
+    if (OPAL_LIKELY(MCA_BTL_UCT_ENDPOINT_FLAG_CONN_READY & endpoint->uct_eps[ep_index][tl_index].flags)) {
+        *ep_handle = endpoint->uct_eps[ep_index][tl_index].uct_ep;
         return OPAL_SUCCESS;
     }
 
     rc = mca_btl_uct_endpoint_connect (module, endpoint, ep_index, NULL, tl_index);
-    *ep_handle = endpoint->uct_eps[tl_index][ep_index].uct_ep;
+    *ep_handle = endpoint->uct_eps[ep_index][tl_index].uct_ep;
     BTL_VERBOSE(("mca_btl_uct_endpoint_connect returned %d", rc));
     return rc;
 }

opal/mca/btl/uct/btl_uct_rdma.c

@@ -243,8 +243,8 @@ int mca_btl_uct_flush (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endp
         do {
             uct_worker_progress (context->uct_worker);
 
-            if (NULL != endpoint && endpoint->uct_eps[tl_index][context->context_id].uct_ep) {
-                ucs_status = uct_ep_flush (endpoint->uct_eps[tl_index][context->context_id].uct_ep, 0, NULL);
+            if (NULL != endpoint && endpoint->uct_eps[context->context_id][tl_index].uct_ep) {
+                ucs_status = uct_ep_flush (endpoint->uct_eps[context->context_id][tl_index].uct_ep, 0, NULL);
             } else {
                 ucs_status = uct_iface_flush (context->uct_iface, 0, NULL);
             }

opal/mca/btl/uct/btl_uct_tl.c

@@ -165,7 +165,7 @@ static ucs_status_t mca_btl_uct_conn_req_cb (void *arg, void *data, size_t lengt
     mca_btl_uct_conn_req_t *req = (mca_btl_uct_conn_req_t *) ((uintptr_t) data + 8);
     struct opal_proc_t *remote_proc = opal_proc_for_name (req->proc_name);
     mca_btl_base_endpoint_t *endpoint = mca_btl_uct_get_ep (&module->super, remote_proc);
-    mca_btl_uct_tl_endpoint_t *tl_endpoint = endpoint->uct_eps[req->tl_index] + req->context_id;
+    mca_btl_uct_tl_endpoint_t *tl_endpoint = endpoint->uct_eps[req->context_id] + req->tl_index;
     int64_t type = *((int64_t *) data);
     int32_t ep_flags;
     int rc;

opal/mca/btl/uct/btl_uct_types.h

@@ -269,7 +269,7 @@ struct mca_btl_base_endpoint_t {
     mca_btl_uct_connection_ep_t *conn_ep;
 
     /** endpoints into UCT for this BTL endpoint */
-    mca_btl_uct_tl_endpoint_t uct_eps[2][MCA_BTL_UCT_MAX_WORKERS];
+    mca_btl_uct_tl_endpoint_t uct_eps[][2];
 };
 
 typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;