Skip to content

Commit fb833e8

Browse files
hjelmnhjelmn
committed
btl/usnic: update for mpool/rcache rewrite
Signed-off-by: Nathan Hjelm <[email protected]>
1 parent 6ded816 commit fb833e8

File tree

5 files changed

+88
-10
lines changed

5 files changed

+88
-10
lines changed

‎opal/mca/btl/usnic/btl_usnic.h‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -12,6 +13,8 @@
1213
* Copyright (c) 2006 Sandia National Laboratories. All rights
1314
* reserved.
1415
* Copyright (c) 2011-2016 Cisco Systems, Inc. All rights reserved.
16+
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
17+
* reserved.
1518
* $COPYRIGHT$
1619
*
1720
* Additional copyrights may follow
@@ -37,16 +40,20 @@
3740
#include "opal/mca/btl/btl.h"
3841
#include "opal/mca/btl/base/btl_base_error.h"
3942
#include "opal/mca/btl/base/base.h"
43+
#include "opal/mca/rcache/rcache.h"
44+
45+
#include "btl_usnic_compat.h"
46+
47+
#if RCACHE_VERSION < 30
4048
#include "opal/mca/mpool/grdma/mpool_grdma.h"
49+
#endif
4150
#else
4251
#include "ompi/mca/btl/btl.h"
4352
#include "ompi/mca/btl/base/btl_base_error.h"
4453
#include "ompi/mca/btl/base/base.h"
4554
#include "ompi/mca/mpool/grdma/mpool_grdma.h"
4655
#endif
4756

48-
#include "btl_usnic_compat.h"
49-
5057
BEGIN_C_DECLS
5158

5259
/*
@@ -153,8 +160,16 @@ typedef struct opal_btl_usnic_component_t {
153160
/** list of usnic proc structures */
154161
opal_list_t usnic_procs;
155162

163+
#if RCACHE_VERSION == 30
164+
/** memory pool hints */
165+
char* usnic_mpool_hints;
166+
167+
/** registration cache name */
168+
char *usnic_rcache_name;
169+
#else
156170
/** name of memory pool */
157171
char* usnic_mpool_name;
172+
#endif
158173

159174
char *if_include;
160175
char *if_exclude;

‎opal/mca/btl/usnic/btl_usnic_compat.h‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2013-2016 Cisco Systems, Inc. All rights reserved.
4+
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
5+
* reserved.
36
* $COPYRIGHT$
47
*
58
* Additional copyrights may follow
@@ -8,11 +11,13 @@
811
*/
912

1013
/* This header contains macros to help minimize usnic BTL differences
11-
* between v1.7/v1.8 and v1.9/v2.0. */
14+
* between v1.7/v1.8, v1.9/v2.0, and v2.0/v2.1. */
1215

1316
#ifndef BTL_USNIC_COMPAT_H
1417
#define BTL_USNIC_COMPAT_H
1518

19+
#include "opal/mca/rcache/rcache.h"
20+
1621
/************************************************************************/
1722

1823
/* v2.0 and beyond */
@@ -358,4 +363,18 @@ opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,
358363

359364
#endif /* BTL_VERSION */
360365

366+
#if defined(RCACHE_MAJOR_VERSION) && RCACHE_MAJOR_VERSION >= 3
367+
368+
#define RCACHE_VERSION 30
369+
370+
/* these structures got renamed with the mpool/rcache rewrite */
371+
#define mca_mpool_base_registration_t mca_rcache_base_registration_t
372+
#define mca_mpool_base_resources_t mca_rcache_base_resources_t
373+
374+
#else
375+
376+
#define RCACHE_VERSION 20
377+
378+
#endif
379+
361380
#endif /* BTL_USNIC_COMPAT_H */

‎opal/mca/btl/usnic/btl_usnic_mca.c‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -12,7 +13,7 @@
1213
* Copyright (c) 2006 Sandia National Laboratories. All rights
1314
* reserved.
1415
* Copyright (c) 2008-2016 Cisco Systems, Inc. All rights reserved.
15-
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights
16+
* Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights
1617
* reserved.
1718
* Copyright (c) 2015 Intel, Inc. All rights reserved.
1819
* $COPYRIGHT$
@@ -203,9 +204,19 @@ int opal_btl_usnic_component_register(void)
203204
0, &stats_relative, 0, OPAL_INFO_LVL_4));
204205
mca_btl_usnic_component.stats_relative = (bool) stats_relative;
205206

207+
#if RCACHE_VERSION == 30
208+
CHECK(reg_string("mpool_hints", "Hints to use when selecting mpool",
209+
NULL, &mca_btl_usnic_component.usnic_mpool_hints, 0,
210+
OPAL_INFO_LVL_5));
211+
212+
CHECK(reg_string("rcache", "Name of the registration cache to be used",
213+
"grdma", &mca_btl_usnic_component.usnic_rcache_name, 0,
214+
OPAL_INFO_LVL_5));
215+
#else
206216
CHECK(reg_string("mpool", "Name of the memory pool to be used",
207217
"grdma", &mca_btl_usnic_component.usnic_mpool_name, 0,
208218
OPAL_INFO_LVL_5));
219+
#endif
209220

210221
want_numa_device_assignment = OPAL_HAVE_HWLOC ? 1 : -1;
211222
CHECK(reg_int("want_numa_device_assignment",

‎opal/mca/btl/usnic/btl_usnic_module.c‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
* Copyright (c) 2006 Sandia National Laboratories. All rights
1414
* reserved.
1515
* Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved.
16-
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
16+
* Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
1717
* reserved.
1818
* Copyright (c) 2014 Intel, Inc. All rights reserved
1919
* $COPYRIGHT$
@@ -44,6 +44,8 @@
4444
#include "opal/mca/btl/base/btl_base_error.h"
4545
#include "opal/mca/mpool/base/base.h"
4646
#include "opal/mca/mpool/mpool.h"
47+
#include "opal/mca/rcache/base/base.h"
48+
#include "opal/mca/rcache/rcache.h"
4749
#else
4850
#include "ompi/mca/btl/btl.h"
4951
#include "ompi/mca/btl/base/btl_base_error.h"
@@ -884,7 +886,11 @@ static int usnic_finalize(struct mca_btl_base_module_t* btl)
884886
OBJ_DESTRUCT(&module->chunk_segs);
885887
OBJ_DESTRUCT(&module->senders);
886888

889+
#if RCACHE_VERSION == 30
890+
mca_rcache_base_module_destroy(module->rcache);
891+
#else
887892
mca_mpool_base_module_destroy(module->super.btl_mpool);
893+
#endif
888894

889895
if (NULL != module->av) {
890896
fi_close(&module->av->fid);
@@ -1736,9 +1742,9 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
17361742
rd_num /* num erorments to alloc */,
17371743
rd_num /* max elements to alloc */,
17381744
rd_num /* num elements per alloc */,
1739-
module->super.btl_mpool /* mpool for reg */,
1745+
module->super.btl_mpool /* mpool for (1.x, 2.0: reg, 2.1+: allocation) */,
17401746
0 /* mpool reg flags */,
1741-
NULL /* unused0 */,
1747+
module->rcache /* registration cache for 2.1+ */,
17421748
NULL /* item_init */,
17431749
NULL /* item_init_context */);
17441750
channel->recv_segs.ctx = module; /* must come after
@@ -2054,11 +2060,28 @@ static int init_mpool(opal_btl_usnic_module_t *module)
20542060
mpool_resources.sizeof_reg = sizeof(opal_btl_usnic_reg_t);
20552061
mpool_resources.register_mem = usnic_reg_mr;
20562062
mpool_resources.deregister_mem = usnic_dereg_mr;
2063+
#if RCACHE_VERSION == 30
2064+
module->rcache =
2065+
mca_rcache_base_module_create (mca_btl_usnic_component.usnic_rcache_name,
2066+
&module->super, &mpool_resources);
2067+
if (NULL == module->rcache) {
2068+
opal_show_help("help-mpi-btl-usnic.txt",
2069+
"internal error during init",
2070+
true,
2071+
opal_process_info.nodename,
2072+
module->fabric_info->fabric_attr->name,
2073+
"create rcache", __FILE__, __LINE__);
2074+
return OPAL_ERROR;
2075+
}
2076+
module->super.btl_mpool =
2077+
mca_mpool_base_module_lookup (mca_btl_usnic_component.usnic_mpool_hints);
2078+
#else
20572079
asprintf(&mpool_resources.pool_name, "%s",
20582080
module->fabric_info->fabric_attr->name);
20592081
module->super.btl_mpool =
20602082
mca_mpool_base_module_create(mca_btl_usnic_component.usnic_mpool_name,
20612083
&module->super, &mpool_resources);
2084+
#endif
20622085
if (NULL == module->super.btl_mpool) {
20632086
opal_show_help("help-mpi-btl-usnic.txt",
20642087
"internal error during init",
@@ -2232,7 +2255,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
22322255
module->sd_num / 2,
22332256
module->super.btl_mpool,
22342257
0 /* mpool reg flags */,
2235-
NULL /* unused0 */,
2258+
module->rcache,
22362259
NULL /* item_init */,
22372260
NULL /* item_init_context */);
22382261
assert(OPAL_SUCCESS == rc);
@@ -2287,7 +2310,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
22872310
module->sd_num / 2,
22882311
module->super.btl_mpool,
22892312
0 /* mpool reg flags */,
2290-
NULL /* unused0 */,
2313+
module->rcache,
22912314
NULL /* item_init */,
22922315
NULL /* item_init_context */);
22932316
assert(OPAL_SUCCESS == rc);
@@ -2309,7 +2332,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
23092332
module->sd_num / 2,
23102333
module->super.btl_mpool,
23112334
0 /* mpool reg flags */,
2312-
NULL /* unused0 */,
2335+
module->rcache,
23132336
NULL /* item_init */,
23142337
NULL /* item_init_context */);
23152338
assert(OPAL_SUCCESS == rc);
@@ -2366,7 +2389,11 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module)
23662389
int ret;
23672390
if (OPAL_SUCCESS != (ret = init_mpool(module)) ||
23682391
OPAL_SUCCESS != (ret = init_channels(module))) {
2392+
#if RCACHE_VERSION == 30
2393+
mca_rcache_base_module_destroy (module->rcache);
2394+
#else
23692395
mca_mpool_base_module_destroy(module->super.btl_mpool);
2396+
#endif
23702397
return ret;
23712398
}
23722399

‎opal/mca/btl/usnic/btl_usnic_module.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -12,6 +13,8 @@
1213
* Copyright (c) 2006 Sandia National Laboratories. All rights
1314
* reserved.
1415
* Copyright (c) 2011-2016 Cisco Systems, Inc. All rights reserved.
16+
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
17+
* reserved.
1518
* $COPYRIGHT$
1619
*
1720
* Additional copyrights may follow
@@ -197,6 +200,9 @@ typedef struct opal_btl_usnic_module_t {
197200

198201
/* Performance / debugging statistics */
199202
opal_btl_usnic_module_stats_t stats;
203+
204+
/** registration cache module (v2.1+) */
205+
mca_rcache_base_module_t *rcache;
200206
} opal_btl_usnic_module_t;
201207

202208
struct opal_btl_usnic_frag_t;

0 commit comments

Comments
 (0)