Merge pull request #10231 from awlauria/s11_v5.0.x

v5.0.x: SS11: initial ofi btl fixes

Author: awlauria

opal/mca/btl/ofi/btl_ofi.h

@@ -15,6 +15,8 @@
  * Copyright (c) 2018-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -124,6 +126,7 @@ struct mca_btl_ofi_module_t {
     bool initialized;
     bool use_virt_addr;
     bool is_scalable_ep;
+    bool use_fi_mr_bind;
 
     opal_atomic_int64_t outstanding_rdma;
     opal_atomic_int64_t outstanding_send;

opal/mca/btl/ofi/btl_ofi_component.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2018-2019 Intel, Inc.  All rights reserved.
  *
  * Copyright (c) 2018-2021 Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -44,7 +44,7 @@
 #define MCA_BTL_OFI_ONE_SIDED_REQUIRED_CAPS (FI_RMA | FI_ATOMIC)
 #define MCA_BTL_OFI_TWO_SIDED_REQUIRED_CAPS (FI_MSG)
 
-#define MCA_BTL_OFI_REQUESTED_MR_MODE (FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_VIRT_ADDR)
+#define MCA_BTL_OFI_REQUESTED_MR_MODE (FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_VIRT_ADDR | FI_MR_ENDPOINT)
 
 static char *ofi_progress_mode;
 static bool disable_sep;
@@ -106,7 +106,7 @@ static int validate_info(struct fi_info *info, uint64_t required_caps, char **in
     mr_mode = info->domain_attr->mr_mode;
 
     if (!(mr_mode == FI_MR_BASIC || mr_mode == FI_MR_SCALABLE
-          || (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY)) == 0)) {
+          || (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_ENDPOINT)) == 0)) {
         BTL_VERBOSE(("unsupported MR mode"));
         return OPAL_ERROR;
     }
@@ -562,12 +562,17 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     module->linux_device_name = linux_device_name;
     module->outstanding_rdma = 0;
     module->use_virt_addr = false;
+    module->use_fi_mr_bind = false;
 
     if (ofi_info->domain_attr->mr_mode == FI_MR_BASIC
         || ofi_info->domain_attr->mr_mode & FI_MR_VIRT_ADDR) {
         module->use_virt_addr = true;
     }
 
+    if (ofi_info->domain_attr->mr_mode & FI_MR_ENDPOINT) {
+        module->use_fi_mr_bind = true;
+    }
+
     /* create endpoint list */
     OBJ_CONSTRUCT(&module->endpoints, opal_list_t);
     OBJ_CONSTRUCT(&module->module_lock, opal_mutex_t);

opal/mca/btl/ofi/btl_ofi_module.c

@@ -16,6 +16,8 @@
  *
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -239,6 +241,18 @@ int mca_btl_ofi_reg_mem(void *reg_data, void *base, size_t size,
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
+    if (btl->use_fi_mr_bind) {
+        BTL_VERBOSE(("binding mr to endpoint"));
+        rc = fi_mr_bind(ur->ur_mr, &btl->ofi_endpoint->fid, 0ULL);
+        if (FI_SUCCESS != rc) {
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+        rc = fi_mr_enable(ur->ur_mr);
+        if (FI_SUCCESS != rc) {
+            return OPAL_ERR_OUT_OF_RESOURCE;
+        }
+    }
+
     ur->handle.rkey = fi_mr_key(ur->ur_mr);
     ur->handle.desc = fi_mr_desc(ur->ur_mr);