Skip to content

Commit 1501855

Browse files
bosilcabosilca
committed
Clean the temporary arrays from the non blocking collectives.
Signed-off-by: George Bosilca <[email protected]>
1 parent 855b523 commit 1501855

File tree

4 files changed

+93
-60
lines changed

4 files changed

+93
-60
lines changed

‎ompi/include/ompi/memchecker.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,7 @@ static inline int memchecker_request(MPI_Request *request)
308308
opal_memchecker_base_isdefined (&(*request)->req_status._ucount, sizeof(size_t));
309309
#endif
310310

311-
opal_memchecker_base_isdefined ((void*)&(*request)->req_complete, sizeof(volatile _Bool));
311+
opal_memchecker_base_isdefined ((void*)&(*request)->req_complete, sizeof(volatile void*));
312312
opal_memchecker_base_isdefined ((void*)&(*request)->req_state, sizeof(volatile ompi_request_state_t));
313313
opal_memchecker_base_isdefined (&(*request)->req_persistent, sizeof(_Bool));
314314
opal_memchecker_base_isdefined (&(*request)->req_f_to_c_index, sizeof(int));

‎ompi/mca/coll/base/coll_base_util.c

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
33
* University Research and Technology
44
* Corporation. All rights reserved.
5-
* Copyright (c) 2004-2020 The University of Tennessee and The University
5+
* Copyright (c) 2004-2021 The University of Tennessee and The University
66
* of Tennessee Research Foundation. All rights
77
* reserved.
88
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -127,14 +127,28 @@ int ompi_rounddown(int num, int factor)
127127
return num * factor; /* floor(num / factor) * factor */
128128
}
129129

130-
static void release_objs_callback(struct ompi_coll_base_nbc_request_t *request) {
131-
if (NULL != request->data.objs.objs[0]) {
132-
OBJ_RELEASE(request->data.objs.objs[0]);
133-
request->data.objs.objs[0] = NULL;
130+
/**
131+
* Release all objects and arrays stored into the nbc_request.
132+
* The release_arrays are temporary memory to stored the values
133+
* converted from Fortran, and should dissapear in same time as the
134+
* request itself.
135+
*/
136+
static void
137+
release_objs_callback(struct ompi_coll_base_nbc_request_t *request)
138+
{
139+
if (NULL != request->data.refcounted.objs.objs[0]) {
140+
OBJ_RELEASE(request->data.refcounted.objs.objs[0]);
141+
request->data.refcounted.objs.objs[0] = NULL;
142+
}
143+
if (NULL != request->data.refcounted.objs.objs[1]) {
144+
OBJ_RELEASE(request->data.refcounted.objs.objs[1]);
145+
request->data.refcounted.objs.objs[1] = NULL;
134146
}
135-
if (NULL != request->data.objs.objs[1]) {
136-
OBJ_RELEASE(request->data.objs.objs[1]);
137-
request->data.objs.objs[1] = NULL;
147+
for(int i = 0; i < OMPI_REQ_NB_RELEASE_ARRAYS; i++ ) {
148+
if( NULL == request->data.release_arrays[i] )
149+
break;
150+
free(request->data.release_arrays[i]);
151+
request->data.release_arrays[i] = NULL;
138152
}
139153
}
140154

@@ -168,12 +182,12 @@ int ompi_coll_base_retain_op( ompi_request_t *req, ompi_op_t *op,
168182
}
169183
if (!ompi_op_is_intrinsic(op)) {
170184
OBJ_RETAIN(op);
171-
request->data.op.op = op;
185+
request->data.refcounted.op.op = op;
172186
retain = true;
173187
}
174188
if (!ompi_datatype_is_predefined(type)) {
175189
OBJ_RETAIN(type);
176-
request->data.op.datatype = type;
190+
request->data.refcounted.op.datatype = type;
177191
retain = true;
178192
}
179193
if (OPAL_UNLIKELY(retain)) {
@@ -207,12 +221,12 @@ int ompi_coll_base_retain_datatypes( ompi_request_t *req, ompi_datatype_t *stype
207221
}
208222
if (NULL != stype && !ompi_datatype_is_predefined(stype)) {
209223
OBJ_RETAIN(stype);
210-
request->data.types.stype = stype;
224+
request->data.refcounted.types.stype = stype;
211225
retain = true;
212226
}
213227
if (NULL != rtype && !ompi_datatype_is_predefined(rtype)) {
214228
OBJ_RETAIN(rtype);
215-
request->data.types.rtype = rtype;
229+
request->data.refcounted.types.rtype = rtype;
216230
retain = true;
217231
}
218232
if (OPAL_UNLIKELY(retain)) {
@@ -237,21 +251,21 @@ static void release_vecs_callback(ompi_coll_base_nbc_request_t *request) {
237251
} else {
238252
scount = rcount = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm);
239253
}
240-
if (NULL != request->data.vecs.stypes) {
254+
if (NULL != request->data.refcounted.vecs.stypes) {
241255
for (int i=0; i<scount; i++) {
242-
if (NULL != request->data.vecs.stypes[i]) {
243-
OMPI_DATATYPE_RELEASE_NO_NULLIFY(request->data.vecs.stypes[i]);
256+
if (NULL != request->data.refcounted.vecs.stypes[i]) {
257+
OMPI_DATATYPE_RELEASE_NO_NULLIFY(request->data.refcounted.vecs.stypes[i]);
244258
}
245259
}
246-
request->data.vecs.stypes = NULL;
260+
request->data.refcounted.vecs.stypes = NULL;
247261
}
248-
if (NULL != request->data.vecs.rtypes) {
262+
if (NULL != request->data.refcounted.vecs.rtypes) {
249263
for (int i=0; i<rcount; i++) {
250-
if (NULL != request->data.vecs.rtypes[i]) {
251-
OMPI_DATATYPE_RELEASE_NO_NULLIFY(request->data.vecs.rtypes[i]);
264+
if (NULL != request->data.refcounted.vecs.rtypes[i]) {
265+
OMPI_DATATYPE_RELEASE_NO_NULLIFY(request->data.refcounted.vecs.rtypes[i]);
252266
}
253267
}
254-
request->data.vecs.rtypes = NULL;
268+
request->data.refcounted.vecs.rtypes = NULL;
255269
}
256270
}
257271

@@ -304,8 +318,8 @@ int ompi_coll_base_retain_datatypes_w( ompi_request_t *req,
304318
}
305319
}
306320
if (OPAL_UNLIKELY(retain)) {
307-
request->data.vecs.stypes = (ompi_datatype_t **) stypes;
308-
request->data.vecs.rtypes = (ompi_datatype_t **) rtypes;
321+
request->data.refcounted.vecs.stypes = (ompi_datatype_t **) stypes;
322+
request->data.refcounted.vecs.rtypes = (ompi_datatype_t **) rtypes;
309323
if (req->req_persistent) {
310324
request->cb.req_free = req->req_free;
311325
req->req_free = free_vecs_callback;
@@ -319,15 +333,16 @@ int ompi_coll_base_retain_datatypes_w( ompi_request_t *req,
319333
return OMPI_SUCCESS;
320334
}
321335

322-
static void nbc_req_cons(ompi_coll_base_nbc_request_t *req)
336+
static void nbc_req_constructor(ompi_coll_base_nbc_request_t *req)
323337
{
324338
req->cb.req_complete_cb = NULL;
325339
req->req_complete_cb_data = NULL;
326-
req->data.objs.objs[0] = NULL;
327-
req->data.objs.objs[1] = NULL;
340+
req->data.refcounted.objs.objs[0] = NULL;
341+
req->data.refcounted.objs.objs[1] = NULL;
342+
req->data.release_arrays[0] = NULL;
328343
}
329344

330-
OBJ_CLASS_INSTANCE(ompi_coll_base_nbc_request_t, ompi_request_t, nbc_req_cons, NULL);
345+
OBJ_CLASS_INSTANCE(ompi_coll_base_nbc_request_t, ompi_request_t, nbc_req_constructor, NULL);
331346

332347
/* File reading functions */
333348
static void skiptonewline (FILE *fptr, int *fileline)

‎ompi/mca/coll/base/coll_base_util.h

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
33
* University Research and Technology
44
* Corporation. All rights reserved.
5-
* Copyright (c) 2004-2020 The University of Tennessee and The University
5+
* Copyright (c) 2004-2021 The University of Tennessee and The University
66
* of Tennessee Research Foundation. All rights
77
* reserved.
88
* Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -34,6 +34,8 @@
3434

3535
BEGIN_C_DECLS
3636

37+
#define OMPI_REQ_NB_RELEASE_ARRAYS 4
38+
3739
/**
3840
* Request structure to be returned by non-blocking
3941
* collective operations.
@@ -45,22 +47,25 @@ struct ompi_coll_base_nbc_request_t {
4547
ompi_request_free_fn_t req_free;
4648
} cb;
4749
void *req_complete_cb_data;
48-
union {
49-
struct {
50-
ompi_op_t *op;
51-
ompi_datatype_t *datatype;
52-
} op;
53-
struct {
54-
ompi_datatype_t *stype;
55-
ompi_datatype_t *rtype;
56-
} types;
57-
struct {
58-
opal_object_t *objs[2];
59-
} objs;
60-
struct {
61-
ompi_datatype_t * const *stypes;
62-
ompi_datatype_t * const *rtypes;
63-
} vecs;
50+
struct {
51+
union {
52+
struct {
53+
ompi_op_t *op;
54+
ompi_datatype_t *datatype;
55+
} op;
56+
struct {
57+
ompi_datatype_t *stype;
58+
ompi_datatype_t *rtype;
59+
} types;
60+
struct {
61+
opal_object_t *objs[2];
62+
} objs;
63+
struct {
64+
ompi_datatype_t * const *stypes;
65+
ompi_datatype_t * const *rtypes;
66+
} vecs;
67+
} refcounted;
68+
void* release_arrays[OMPI_REQ_NB_RELEASE_ARRAYS];
6469
} data;
6570
};
6671

‎ompi/mpi/fortran/mpif-h/ialltoallw_f.c

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
33
* University Research and Technology
44
* Corporation. All rights reserved.
5-
* Copyright (c) 2004-2005 The University of Tennessee and The University
5+
* Copyright (c) 2004-2021 The University of Tennessee and The University
66
* of Tennessee Research Foundation. All rights
77
* reserved.
88
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -24,6 +24,7 @@
2424
#include "ompi/mpi/fortran/mpif-h/bindings.h"
2525
#include "ompi/mpi/fortran/base/constants.h"
2626
#include "ompi/communicator/communicator.h"
27+
#include "ompi/mca/coll/base/coll_base_util.h"
2728

2829
#if OMPI_BUILD_MPI_PROFILING
2930
#if OPAL_HAVE_WEAK_SYMBOLS
@@ -77,7 +78,7 @@ void ompi_ialltoallw_f(char *sendbuf, MPI_Fint *sendcounts,
7778
MPI_Comm c_comm;
7879
MPI_Datatype *c_sendtypes = NULL, *c_recvtypes;
7980
MPI_Request c_request;
80-
int size, c_ierr;
81+
int size, idx = 0, c_ierr;
8182
OMPI_ARRAY_NAME_DECL(sendcounts);
8283
OMPI_ARRAY_NAME_DECL(sdispls);
8384
OMPI_ARRAY_NAME_DECL(recvcounts);
@@ -107,22 +108,34 @@ void ompi_ialltoallw_f(char *sendbuf, MPI_Fint *sendcounts,
107108
recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf);
108109

109110
c_ierr = PMPI_Ialltoallw(sendbuf,
110-
OMPI_ARRAY_NAME_CONVERT(sendcounts),
111-
OMPI_ARRAY_NAME_CONVERT(sdispls),
112-
c_sendtypes,
113-
recvbuf,
114-
OMPI_ARRAY_NAME_CONVERT(recvcounts),
115-
OMPI_ARRAY_NAME_CONVERT(rdispls),
116-
c_recvtypes, c_comm, &c_request);
111+
OMPI_ARRAY_NAME_CONVERT(sendcounts),
112+
OMPI_ARRAY_NAME_CONVERT(sdispls),
113+
c_sendtypes,
114+
recvbuf,
115+
OMPI_ARRAY_NAME_CONVERT(recvcounts),
116+
OMPI_ARRAY_NAME_CONVERT(rdispls),
117+
c_recvtypes, c_comm, &c_request);
117118
if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
118119
if (MPI_SUCCESS == c_ierr) *request = PMPI_Request_c2f(c_request);
119120

120-
OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
121-
OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
122-
OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
123-
OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
124-
if (NULL != c_sendtypes) {
125-
free(c_sendtypes);
121+
if ( REQUEST_COMPLETE(c_request)) {
122+
if (NULL != c_sendtypes) {
123+
free(c_sendtypes);
124+
}
125+
free(c_recvtypes);
126+
OMPI_ARRAY_FINT_2_INT_CLEANUP(sendcounts);
127+
OMPI_ARRAY_FINT_2_INT_CLEANUP(sdispls);
128+
OMPI_ARRAY_FINT_2_INT_CLEANUP(recvcounts);
129+
OMPI_ARRAY_FINT_2_INT_CLEANUP(rdispls);
130+
} else {
131+
ompi_coll_base_nbc_request_t* nb_request = (ompi_coll_base_nbc_request_t*)c_request;
132+
if (sendcounts == OMPI_ARRAY_NAME_CONVERT(sendcounts)) {
133+
nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sendcounts);
134+
nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(recvcounts);
135+
}
136+
if (sdispls == OMPI_ARRAY_NAME_CONVERT(sdispls)) {
137+
nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(sdispls);
138+
nb_request->data.release_arrays[idx++] = OMPI_ARRAY_NAME_CONVERT(rdispls);
139+
}
126140
}
127-
free(c_recvtypes);
128141
}

0 commit comments

Comments
 (0)