btl/usnic: update for mpool/rcache rewrite

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

opal/mca/btl/usnic/btl_usnic.h

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,8 @@
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
  * Copyright (c) 2011-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,16 +40,20 @@
 #include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/rcache/rcache.h"
+
+#include "btl_usnic_compat.h"
+
+#if RCACHE_VERSION < 30
 #include "opal/mca/mpool/grdma/mpool_grdma.h"
+#endif
 #else
 #include "ompi/mca/btl/btl.h"
 #include "ompi/mca/btl/base/btl_base_error.h"
 #include "ompi/mca/btl/base/base.h"
 #include "ompi/mca/mpool/grdma/mpool_grdma.h"
 #endif
 
-#include "btl_usnic_compat.h"
-
 BEGIN_C_DECLS
 
 /*
@@ -153,8 +160,16 @@ typedef struct opal_btl_usnic_component_t {
     /** list of usnic proc structures */
     opal_list_t usnic_procs;
 
+#if RCACHE_VERSION == 30
+    /** memory pool hints */
+    char* usnic_mpool_hints;
+
+    /** registration cache name */
+    char *usnic_rcache_name;
+#else
     /** name of memory pool */
     char* usnic_mpool_name;
+#endif
 
     char *if_include;
     char *if_exclude;

opal/mca/btl/usnic/btl_usnic_compat.h

@@ -1,5 +1,8 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -8,11 +11,13 @@
  */
 
 /* This header contains macros to help minimize usnic BTL differences
- * between v1.7/v1.8 and v1.9/v2.0. */
+ * between v1.7/v1.8, v1.9/v2.0, and v2.0/v2.1. */
 
 #ifndef BTL_USNIC_COMPAT_H
 #define BTL_USNIC_COMPAT_H
 
+#include "opal/mca/rcache/rcache.h"
+
 /************************************************************************/
 
 /* v2.0 and beyond */
@@ -358,4 +363,18 @@ opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
 
 #endif /* BTL_VERSION */
 
+#if defined(RCACHE_MAJOR_VERSION) && RCACHE_MAJOR_VERSION >= 3
+
+#define RCACHE_VERSION 30
+
+/* these structures got renamed with the mpool/rcache rewrite */
+#define mca_mpool_base_registration_t mca_rcache_base_registration_t
+#define mca_mpool_base_resources_t mca_rcache_base_resources_t
+
+#else
+
+#define RCACHE_VERSION 20
+
+#endif
+
 #endif /* BTL_USNIC_COMPAT_H */

opal/mca/btl/usnic/btl_usnic_mca.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,7 +13,7 @@
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
  * Copyright (c) 2008-2016 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2012-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
@@ -203,9 +204,19 @@ int opal_btl_usnic_component_register(void)
                   0, &stats_relative, 0, OPAL_INFO_LVL_4));
     mca_btl_usnic_component.stats_relative = (bool) stats_relative;
 
+#if RCACHE_VERSION == 30
+    CHECK(reg_string("mpool_hints", "Hints to use when selecting mpool",
+                     NULL, &mca_btl_usnic_component.usnic_mpool_hints, 0,
+                     OPAL_INFO_LVL_5));
+
+    CHECK(reg_string("rcache", "Name of the registration cache to be used",
+                     "grdma", &mca_btl_usnic_component.usnic_rcache_name, 0,
+                     OPAL_INFO_LVL_5));
+#else
     CHECK(reg_string("mpool", "Name of the memory pool to be used",
                      "grdma", &mca_btl_usnic_component.usnic_mpool_name, 0,
                      OPAL_INFO_LVL_5));
+#endif
 
     want_numa_device_assignment = OPAL_HAVE_HWLOC ? 1 : -1;
     CHECK(reg_int("want_numa_device_assignment",

opal/mca/btl/usnic/btl_usnic_module.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
  * Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014      Intel, Inc. All rights reserved
  * $COPYRIGHT$
@@ -44,6 +44,8 @@
 #include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/mpool/mpool.h"
+#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/rcache/rcache.h"
 #else
 #include "ompi/mca/btl/btl.h"
 #include "ompi/mca/btl/base/btl_base_error.h"
@@ -884,7 +886,11 @@ static int usnic_finalize(struct mca_btl_base_module_t* btl)
     OBJ_DESTRUCT(&module->chunk_segs);
     OBJ_DESTRUCT(&module->senders);
 
+#if RCACHE_VERSION == 30
+    mca_rcache_base_module_destroy(module->rcache);
+#else
     mca_mpool_base_module_destroy(module->super.btl_mpool);
+#endif
 
     if (NULL != module->av) {
         fi_close(&module->av->fid);
@@ -1736,9 +1742,9 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
                                     rd_num /* num erorments to alloc */,
                                     rd_num /* max elements to alloc */,
                                     rd_num /* num elements per alloc */,
-                                    module->super.btl_mpool /* mpool for reg */,
+                                    module->super.btl_mpool /* mpool for (1.x, 2.0: reg, 2.1+: allocation) */,
                                     0 /* mpool reg flags */,
-                                    NULL /* unused0 */,
+                                    module->rcache /* registration cache for 2.1+ */,
                                     NULL /* item_init */,
                                     NULL /* item_init_context */);
     channel->recv_segs.ctx = module; /* must come after
@@ -2054,11 +2060,28 @@ static int init_mpool(opal_btl_usnic_module_t *module)
     mpool_resources.sizeof_reg = sizeof(opal_btl_usnic_reg_t);
     mpool_resources.register_mem = usnic_reg_mr;
     mpool_resources.deregister_mem = usnic_dereg_mr;
+#if RCACHE_VERSION == 30
+    module->rcache =
+        mca_rcache_base_module_create (mca_btl_usnic_component.usnic_rcache_name,
+                                       &module->super, &mpool_resources);
+    if (NULL == module->rcache) {
+        opal_show_help("help-mpi-btl-usnic.txt",
+                       "internal error during init",
+                       true,
+                       opal_process_info.nodename,
+                       module->fabric_info->fabric_attr->name,
+                       "create rcache", __FILE__, __LINE__);
+        return OPAL_ERROR;
+    }
+    module->super.btl_mpool =
+        mca_mpool_base_module_lookup (mca_btl_usnic_component.usnic_mpool_hints);
+#else
     asprintf(&mpool_resources.pool_name, "%s",
              module->fabric_info->fabric_attr->name);
     module->super.btl_mpool =
         mca_mpool_base_module_create(mca_btl_usnic_component.usnic_mpool_name,
                                      &module->super, &mpool_resources);
+#endif
     if (NULL == module->super.btl_mpool) {
         opal_show_help("help-mpi-btl-usnic.txt",
                        "internal error during init",
@@ -2232,7 +2255,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
                              module->sd_num / 2,
                              module->super.btl_mpool,
                              0 /* mpool reg flags */,
-                             NULL /* unused0 */,
+                             module->rcache,
                              NULL /* item_init */,
                              NULL /* item_init_context */);
     assert(OPAL_SUCCESS == rc);
@@ -2287,7 +2310,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
                              module->sd_num / 2,
                              module->super.btl_mpool,
                              0 /* mpool reg flags */,
-                             NULL /* unused0 */,
+                             module->rcache,
                              NULL /* item_init */,
                              NULL /* item_init_context */);
     assert(OPAL_SUCCESS == rc);
@@ -2309,7 +2332,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
                              module->sd_num / 2,
                              module->super.btl_mpool,
                              0 /* mpool reg flags */,
-                             NULL /* unused0 */,
+                             module->rcache,
                              NULL /* item_init */,
                              NULL /* item_init_context */);
     assert(OPAL_SUCCESS == rc);
@@ -2366,7 +2389,11 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module)
     int ret;
     if (OPAL_SUCCESS != (ret = init_mpool(module)) ||
         OPAL_SUCCESS != (ret = init_channels(module))) {
+#if RCACHE_VERSION == 30
+        mca_rcache_base_module_destroy (module->rcache);
+#else
         mca_mpool_base_module_destroy(module->super.btl_mpool);
+#endif
         return ret;
     }
 

opal/mca/btl/usnic/btl_usnic_module.h

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,8 @@
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
  * Copyright (c) 2011-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -197,6 +200,9 @@ typedef struct opal_btl_usnic_module_t {
 
     /* Performance / debugging statistics */
     opal_btl_usnic_module_stats_t stats;
+
+    /** registration cache module (v2.1+) */
+    mca_rcache_base_module_t *rcache;
 } opal_btl_usnic_module_t;
 
 struct opal_btl_usnic_frag_t;