Skip to content

Commit da1cdd7

Browse files
author
rhc54
committed
Merge pull request #7 from miked-mellanox/topic/oshmem_add_two_new_mca_variables
Add-two-new-mca-variables, fix opal_help failure
2 parents da7c460 + 38ae556 commit da1cdd7

File tree

6 files changed

+121
-16
lines changed

6 files changed

+121
-16
lines changed

‎oshmem/mca/sshmem/base/help-oshmem-sshmem.txt

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# -*- text -*-
2+
#
3+
# Copyright (c) 2013 Mellanox Technologies, Inc.
4+
# All rights reserved.
5+
# Copyright (c) 2014 Cisco Systems, Inc. All rights reserved.
6+
# $COPYRIGHT$
7+
#
8+
# Additional copyrights may follow
9+
#
10+
# $HEADER$
11+
#
12+
# This is the US/English help file for Open SHMEM MCA error messages.
13+
#
14+
[create segment failure]
15+
The OpenSHMEM "(%s)" plugin in the "sshmem" framework failed to
16+
allocate a shared memory segment via the system call. This
17+
usually means that there are not enough resources available to memory subsystem on your server.
18+
19+
Your OpenSHMEM job will now abort.
20+
21+
Server: %s
22+
Requested shared
23+
memory segment size: %llu
24+
Specific error: %s (%d)
25+

‎oshmem/mca/sshmem/mmap/sshmem_mmap_module.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,8 +198,8 @@ segment_create(map_segment_t *ds_buf,
198198
if (MAP_FAILED == addr) {
199199
opal_show_help("help-oshmem-sshmem.txt",
200200
"create segment failure",
201-
"mmap",
202201
true,
202+
"mmap",
203203
orte_process_info.nodename, (unsigned long long) size,
204204
strerror(errno), errno);
205205
opal_show_help("help-oshmem-sshmem-mmap.txt",

‎oshmem/mca/sshmem/sysv/sshmem_sysv.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313

1414
#include "oshmem_config.h"
1515

16+
#include "opal/util/sys_limits.h"
17+
1618
#include "oshmem/mca/sshmem/sshmem.h"
1719

1820
BEGIN_C_DECLS
@@ -36,6 +38,8 @@ typedef struct mca_sshmem_sysv_module_t {
3638
} mca_sshmem_sysv_module_t;
3739
extern mca_sshmem_sysv_module_t mca_sshmem_sysv_module;
3840

41+
OSHMEM_MODULE_DECLSPEC extern size_t sshmem_sysv_gethugepagesize(void);
42+
3943
END_C_DECLS
4044

4145
#endif /* MCA_SSHMEM_SYSV_EXPORT_H */

‎oshmem/mca/sshmem/sysv/sshmem_sysv_component.c

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -104,31 +104,54 @@ sysv_runtime_query(mca_base_module_t **module,
104104
char *addr = NULL;
105105
struct shmid_ds tmp_buff;
106106
int flags;
107+
int ret;
108+
109+
ret = OSHMEM_SUCCESS;
107110

108111
*priority = 0;
109112
*module = NULL;
110113

111114
/* if we are here, then let the run-time test games begin */
112115

113116
#if defined (SHM_HUGETLB)
114-
mca_sshmem_sysv_component.use_hp = 1;
115-
flags = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR | SHM_HUGETLB;
116-
if (-1 == (shmid = shmget(IPC_PRIVATE, (size_t)(opal_getpagesize()), flags))) {
117-
mca_sshmem_sysv_component.use_hp = 0;
117+
if (mca_sshmem_sysv_component.use_hp != 0) {
118+
flags = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR | SHM_HUGETLB;
119+
if (-1 == (shmid = shmget(IPC_PRIVATE, sshmem_sysv_gethugepagesize(), flags))) {
120+
if (mca_sshmem_sysv_component.use_hp == 1) {
121+
mca_sshmem_sysv_component.use_hp = 0;
122+
ret = OSHMEM_ERR_NOT_AVAILABLE;
123+
goto out;
124+
}
125+
mca_sshmem_sysv_component.use_hp = 0;
126+
}
127+
else if ((void *)-1 == (addr = shmat(shmid, NULL, 0))) {
128+
shmctl(shmid, IPC_RMID, NULL);
129+
if (mca_sshmem_sysv_component.use_hp == 1) {
130+
mca_sshmem_sysv_component.use_hp = 0;
131+
ret = OSHMEM_ERR_NOT_AVAILABLE;
132+
goto out;
133+
}
134+
mca_sshmem_sysv_component.use_hp = 0;
135+
}
118136
}
119-
else if ((void *)-1 == (addr = shmat(shmid, NULL, 0))) {
120-
shmctl(shmid, IPC_RMID, NULL );
137+
#else
138+
if (mca_sshmem_sysv_component.use_hp == 1) {
121139
mca_sshmem_sysv_component.use_hp = 0;
140+
ret = OSHMEM_ERR_NOT_AVAILABLE;
141+
goto out;
122142
}
143+
mca_sshmem_sysv_component.use_hp = 0;
123144
#endif
124145

125146
if (0 == mca_sshmem_sysv_component.use_hp) {
126147
flags = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR;
127148
if (-1 == (shmid = shmget(IPC_PRIVATE, (size_t)(opal_getpagesize()), flags))) {
149+
ret = OSHMEM_ERR_NOT_AVAILABLE;
128150
goto out;
129151
}
130152
else if ((void *)-1 == (addr = shmat(shmid, NULL, 0))) {
131-
shmctl(shmid, IPC_RMID, NULL );
153+
shmctl(shmid, IPC_RMID, NULL);
154+
ret = OSHMEM_ERR_NOT_AVAILABLE;
132155
goto out;
133156
}
134157
}
@@ -153,7 +176,7 @@ sysv_runtime_query(mca_base_module_t **module,
153176
if ((char *)-1 != addr) {
154177
shmdt(addr);
155178
}
156-
return OSHMEM_SUCCESS;
179+
return ret;
157180
}
158181

159182
/* ////////////////////////////////////////////////////////////////////////// */
@@ -171,7 +194,14 @@ sysv_register(void)
171194
MCA_BASE_VAR_SCOPE_ALL_EQ,
172195
&mca_sshmem_sysv_component.priority);
173196

174-
mca_sshmem_sysv_component.use_hp = 0;
197+
mca_sshmem_sysv_component.use_hp = -1;
198+
mca_base_component_var_register (&mca_sshmem_sysv_component.super.base_version,
199+
"use_hp", "Huge pages usage "
200+
"[0 - off, 1 - on, -1 - auto] (default: -1)", MCA_BASE_VAR_TYPE_INT,
201+
NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
202+
OPAL_INFO_LVL_4,
203+
MCA_BASE_VAR_SCOPE_ALL_EQ,
204+
&mca_sshmem_sysv_component.use_hp);
175205

176206
return OSHMEM_SUCCESS;
177207
}

‎oshmem/mca/sshmem/sysv/sshmem_sysv_module.c

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,8 @@ segment_create(map_segment_t *ds_buf,
188188
*/
189189
flags = IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR;
190190
#if defined (SHM_HUGETLB)
191-
flags |= (mca_sshmem_sysv_component.use_hp ? SHM_HUGETLB : 0);
191+
flags |= ((0 != mca_sshmem_sysv_component.use_hp) ? SHM_HUGETLB : 0);
192+
size = ((size + sshmem_sysv_gethugepagesize() - 1) / sshmem_sysv_gethugepagesize()) * sshmem_sysv_gethugepagesize();
192193
#endif
193194

194195
/* Create a new shared memory segment and save the shmid. */
@@ -206,7 +207,7 @@ segment_create(map_segment_t *ds_buf,
206207
return OSHMEM_ERROR;
207208
}
208209

209-
/* Attach to the sement */
210+
/* Attach to the segment */
210211
addr = shmat(shmid, (void *) mca_sshmem_base_start_address, 0);
211212
if (addr == (void *) -1L) {
212213
opal_show_help("help-oshmem-sshmem.txt",
@@ -294,7 +295,7 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
294295
shmctl(ds_buf->seg_id, IPC_RMID, NULL );
295296
}
296297

297-
if (mca_sshmem_sysv_component.use_hp > 0) {
298+
if (mca_sshmem_sysv_component.use_hp != 0) {
298299
/**
299300
* Workaround kernel panic when detaching huge pages from user space simultanously from several processes
300301
* dont detach here instead let kernel do it during process cleanup
@@ -336,3 +337,36 @@ segment_unlink(map_segment_t *ds_buf)
336337
return OSHMEM_SUCCESS;
337338
}
338339

340+
/*
341+
* Get current huge page size
342+
*
343+
*/
344+
size_t sshmem_sysv_gethugepagesize(void)
345+
{
346+
static size_t huge_page_size = 0;
347+
char buf[256];
348+
int size_kb;
349+
FILE *f;
350+
351+
/* Cache the huge page size value */
352+
if (huge_page_size == 0) {
353+
f = fopen("/proc/meminfo", "r");
354+
if (f != NULL) {
355+
while (fgets(buf, sizeof(buf), f)) {
356+
if (sscanf(buf, "Hugepagesize: %d kB", &size_kb) == 1) {
357+
huge_page_size = size_kb * 1024L;
358+
break;
359+
}
360+
}
361+
fclose(f);
362+
}
363+
364+
if (huge_page_size == 0) {
365+
huge_page_size = 2 * 1024L *1024L;
366+
}
367+
}
368+
369+
return huge_page_size;
370+
}
371+
372+

‎oshmem/mca/sshmem/verbs/sshmem_verbs_component.c

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ verbs_runtime_query(mca_base_module_t **module,
176176
}
177177

178178
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
179-
if (!rc) {
179+
if (!rc && (0 != mca_sshmem_verbs_component.has_shared_mr)) {
180180
struct ibv_exp_reg_shared_mr_in in_smr;
181181

182182
access_flag = IBV_ACCESS_LOCAL_WRITE |
@@ -190,14 +190,17 @@ verbs_runtime_query(mca_base_module_t **module,
190190
if (NULL == ib_mr) {
191191
if (mca_sshmem_verbs_component.has_shared_mr == 1)
192192
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
193-
194193
mca_sshmem_verbs_component.has_shared_mr = 0;
195-
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
196194
} else {
197195
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
198196
mca_sshmem_verbs_component.has_shared_mr = 1;
199197
}
200198
}
199+
#else
200+
if (!rc && mca_sshmem_verbs_component.has_shared_mr == 1) {
201+
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
202+
}
203+
mca_sshmem_verbs_component.has_shared_mr = 0;
201204
#endif /* MPAGE_ENABLE */
202205
}
203206

@@ -308,6 +311,15 @@ verbs_register(void)
308311
MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
309312
}
310313

314+
mca_sshmem_verbs_component.has_shared_mr = -1;
315+
index = mca_base_component_var_register (&mca_sshmem_verbs_component.super.base_version,
316+
"shared_mr", "Shared memory region usage "
317+
"[0 - off, 1 - on, -1 - auto] (default: -1)", MCA_BASE_VAR_TYPE_INT,
318+
NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
319+
OPAL_INFO_LVL_4,
320+
MCA_BASE_VAR_SCOPE_ALL_EQ,
321+
&mca_sshmem_verbs_component.has_shared_mr);
322+
311323
return OSHMEM_SUCCESS;
312324
}
313325

0 commit comments

Comments
 (0)