Skip to content

Commit 3e559a1

Browse files
ggouaillardetggouaillardet
committed
coll/inter: fix non standard ddt handling
 - correctly handle non zero lower bound ddt - correctly handle ddt with size > extent Thanks Yuki Matsumoto for the report
1 parent 488d037 commit 3e559a1

File tree

8 files changed

+82
-123
lines changed

8 files changed

+82
-123
lines changed

‎ompi/mca/coll/inter/coll_inter_allgather.c

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2010 University of Houston. All rights reserved.
13-
* Copyright (c) 2015 Research Organization for Information Science
13+
* Copyright (c) 2015-2016 Research Organization for Information Science
1414
* and Technology (RIST). All rights reserved.
1515
* $COPYRIGHT$
1616
*
@@ -49,26 +49,22 @@ mca_coll_inter_allgather_inter(const void *sbuf, int scount,
4949
mca_coll_base_module_t *module)
5050
{
5151
int rank, root = 0, size, rsize, err;
52-
char *ptmp = NULL;
53-
ptrdiff_t slb, sextent, incr;
52+
char *ptmp_free = NULL, *ptmp;
53+
ptrdiff_t gap, span;
5454
ompi_request_t *req[2];
5555

5656
rank = ompi_comm_rank(comm);
5757
size = ompi_comm_size(comm->c_local_comm);
5858
rsize = ompi_comm_remote_size(comm);
5959

6060
/* Perform the gather locally at the root */
61-
err = ompi_datatype_get_extent(sdtype, &slb, &sextent);
62-
if (OMPI_SUCCESS != err) {
63-
return OMPI_ERROR;
64-
}
65-
6661
if ( scount > 0 ) {
67-
incr = sextent * scount;
68-
ptmp = (char*)malloc(size * incr);
69-
if (NULL == ptmp) {
62+
span = opal_datatype_span(&sdtype->super, scount*size, &gap);
63+
ptmp_free = (char*)malloc(span);
64+
if (NULL == ptmp_free) {
7065
return OMPI_ERR_OUT_OF_RESOURCE;
7166
}
67+
ptmp = ptmp_free - gap;
7268

7369
err = comm->c_local_comm->c_coll.coll_gather(sbuf, scount, sdtype,
7470
ptmp, scount, sdtype,
@@ -112,8 +108,8 @@ mca_coll_inter_allgather_inter(const void *sbuf, int scount,
112108
}
113109

114110
exit:
115-
if (NULL != ptmp) {
116-
free(ptmp);
111+
if (NULL != ptmp_free) {
112+
free(ptmp_free);
117113
}
118114

119115
return err;

‎ompi/mca/coll/inter/coll_inter_allgatherv.c

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2010 University of Houston. All rights reserved.
13-
* Copyright (c) 2015 Research Organization for Information Science
13+
* Copyright (c) 2015-2016 Research Organization for Information Science
1414
* and Technology (RIST). All rights reserved.
1515
* $COPYRIGHT$
1616
*
@@ -49,10 +49,7 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
4949
{
5050
int i, rank, size, size_local, total=0, err;
5151
int *count=NULL,*displace=NULL;
52-
char *ptmp=NULL;
53-
MPI_Aint incr;
54-
MPI_Aint extent;
55-
MPI_Aint lb;
52+
char *ptmp_free=NULL, *ptmp;
5653
ompi_datatype_t *ndtype = NULL;
5754
ompi_request_t *req[2];
5855

@@ -81,22 +78,19 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
8178
for (i = 1; i < size_local; i++) {
8279
displace[i] = displace[i-1] + count[i-1];
8380
}
84-
/* Perform the gatherv locally with the first process as root */
85-
err = ompi_datatype_get_extent(sdtype, &lb, &extent);
86-
if (OMPI_SUCCESS != err) {
87-
err = OMPI_ERROR;
88-
goto exit;
89-
}
90-
incr = 0;
81+
total = 0;
9182
for (i = 0; i < size_local; i++) {
92-
incr = incr + extent*count[i];
83+
total = total + count[i];
9384
}
94-
if ( incr > 0 ) {
95-
ptmp = (char*)malloc(incr);
96-
if (NULL == ptmp) {
85+
if ( total > 0 ) {
86+
ptrdiff_t gap, span;
87+
span = opal_datatype_span(&sdtype->super, total, &gap);
88+
ptmp_free = (char*)malloc(span);
89+
if (NULL == ptmp_free) {
9790
err = OMPI_ERR_OUT_OF_RESOURCE;
9891
goto exit;
9992
}
93+
ptmp = ptmp_free - gap;
10094
}
10195
}
10296
err = comm->c_local_comm->c_coll.coll_gatherv(sbuf, scount, sdtype,
@@ -111,9 +105,6 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
111105
ompi_datatype_commit(&ndtype);
112106

113107
if (0 == rank) {
114-
for (i = 0; i < size_local; i++) {
115-
total = total + count[i];
116-
}
117108
/* Exchange data between roots */
118109
err = MCA_PML_CALL(irecv(rbuf, 1, ndtype, 0,
119110
MCA_COLL_BASE_TAG_ALLGATHERV, comm,
@@ -144,8 +135,8 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
144135
if( NULL != ndtype ) {
145136
ompi_datatype_destroy(&ndtype);
146137
}
147-
if (NULL != ptmp) {
148-
free(ptmp);
138+
if (NULL != ptmp_free) {
139+
free(ptmp_free);
149140
}
150141
if (NULL != displace) {
151142
free(displace);

‎ompi/mca/coll/inter/coll_inter_allreduce.c

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2007 University of Houston. All rights reserved.
13-
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
14-
* Copyright (c) 2015 Research Organization for Information Science
13+
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
14+
* Copyright (c) 2015-2016 Research Organization for Information Science
1515
* and Technology (RIST). All rights reserved.
1616
* $COPYRIGHT$
1717
*
@@ -48,23 +48,20 @@ mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
4848
mca_coll_base_module_t *module)
4949
{
5050
int err, rank, root = 0;
51-
ptrdiff_t lb, extent;
5251
char *tmpbuf = NULL, *pml_buffer = NULL;
5352
ompi_request_t *req[2];
53+
ptrdiff_t gap, span;
5454

5555
rank = ompi_comm_rank(comm);
5656

5757
/* Perform the reduction locally */
58-
err = ompi_datatype_get_extent(dtype, &lb, &extent);
59-
if (OMPI_SUCCESS != err) {
60-
return OMPI_ERROR;
61-
}
58+
span = opal_datatype_span(&dtype->super, count, &gap);
6259

63-
tmpbuf = (char *) malloc(count * extent);
60+
tmpbuf = (char *) malloc(span);
6461
if (NULL == tmpbuf) {
6562
return OMPI_ERR_OUT_OF_RESOURCE;
6663
}
67-
pml_buffer = tmpbuf - lb;
64+
pml_buffer = tmpbuf - gap;
6865

6966
err = comm->c_local_comm->c_coll.coll_reduce(sbuf, pml_buffer, count,
7067
dtype, op, root,

‎ompi/mca/coll/inter/coll_inter_gather.c

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2007 University of Houston. All rights reserved.
13-
* Copyright (c) 2015 Research Organization for Information Science
13+
* Copyright (c) 2015-2016 Research Organization for Information Science
1414
* and Technology (RIST). All rights reserved.
1515
* $COPYRIGHT$
1616
*
@@ -47,11 +47,7 @@ mca_coll_inter_gather_inter(const void *sbuf, int scount,
4747
{
4848
int err;
4949
int rank;
50-
int size,size_local;
51-
char *ptmp = NULL;
52-
MPI_Aint incr;
53-
MPI_Aint extent;
54-
MPI_Aint lb;
50+
int size;
5551

5652
size = ompi_comm_remote_size(comm);
5753
rank = ompi_comm_rank(comm);
@@ -61,17 +57,18 @@ mca_coll_inter_gather_inter(const void *sbuf, int scount,
6157
err = OMPI_SUCCESS;
6258
} else if (MPI_ROOT != root) {
6359
/* Perform the gather locally with the first process as root */
64-
err = ompi_datatype_get_extent(sdtype, &lb, &extent);
65-
if (OMPI_SUCCESS != err) {
66-
return OMPI_ERROR;
67-
}
60+
char *ptmp_free = NULL, *ptmp;
61+
int size_local;
62+
ptrdiff_t gap, span;
6863

69-
incr = extent * scount;
7064
size_local = ompi_comm_size(comm->c_local_comm);
71-
ptmp = (char*)malloc(size_local * incr);
72-
if (NULL == ptmp) {
65+
span = opal_datatype_span(&sdtype->super, scount*size_local, &gap);
66+
67+
ptmp_free = (char*)malloc(span);
68+
if (NULL == ptmp_free) {
7369
return OMPI_ERR_OUT_OF_RESOURCE;
7470
}
71+
ptmp = ptmp_free - gap;
7572

7673
err = comm->c_local_comm->c_coll.coll_gather(sbuf, scount, sdtype,
7774
ptmp, scount, sdtype,
@@ -86,9 +83,7 @@ mca_coll_inter_gather_inter(const void *sbuf, int scount,
8683
return err;
8784
}
8885
}
89-
if (NULL != ptmp) {
90-
free(ptmp);
91-
}
86+
free(ptmp_free);
9287
} else {
9388
/* I am the root, loop receiving the data. */
9489
err = MCA_PML_CALL(recv(rbuf, rcount*size, rdtype, 0,

‎ompi/mca/coll/inter/coll_inter_gatherv.c

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2010 University of Houston. All rights reserved.
13-
* Copyright (c) 2015 Research Organization for Information Science
13+
* Copyright (c) 2015-2016 Research Organization for Information Science
1414
* and Technology (RIST). All rights reserved.
1515
* $COPYRIGHT$
1616
*
@@ -46,10 +46,7 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
4646
{
4747
int i, rank, size, size_local, total=0, err;
4848
int *count=NULL, *displace=NULL;
49-
char *ptmp=NULL;
50-
MPI_Aint incr;
51-
MPI_Aint extent;
52-
MPI_Aint lb;
49+
char *ptmp_free=NULL, *ptmp;
5350
ompi_datatype_t *ndtype;
5451

5552
if (MPI_PROC_NULL == root) { /* do nothing */
@@ -92,21 +89,18 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
9289
displace[i] = displace[i-1] + count[i-1];
9390
}
9491
/* Perform the gatherv locally with the first process as root */
95-
err = ompi_datatype_get_extent(sdtype, &lb, &extent);
96-
if (OMPI_SUCCESS != err) {
97-
err = OMPI_ERROR;
98-
goto exit;
99-
}
100-
incr = 0;
10192
for (i = 0; i < size_local; i++) {
102-
incr = incr + extent*count[i];
93+
total = total + count[i];
10394
}
104-
if ( incr > 0 ) {
105-
ptmp = (char*)malloc(incr);
106-
if (NULL == ptmp) {
95+
if ( total > 0 ) {
96+
ptrdiff_t gap, span;
97+
span = opal_datatype_span(&sdtype->super, total, &gap);
98+
ptmp_free = (char*)malloc(span);
99+
if (NULL == ptmp_free) {
107100
err = OMPI_ERR_OUT_OF_RESOURCE;
108101
goto exit;
109102
}
103+
ptmp = ptmp_free - gap;
110104
}
111105
}
112106
err = comm->c_local_comm->c_coll.coll_gatherv(sbuf, scount, sdtype,
@@ -118,18 +112,15 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
118112
}
119113

120114
if (0 == rank) {
121-
for (i = 0; i < size_local; i++) {
122-
total = total + count[i];
123-
}
124115
/* First process sends data to the root */
125116
err = MCA_PML_CALL(send(ptmp, total, sdtype, root,
126117
MCA_COLL_BASE_TAG_GATHERV,
127118
MCA_PML_BASE_SEND_STANDARD, comm));
128119
}
129120

130121
exit:
131-
if (NULL != ptmp) {
132-
free(ptmp);
122+
if (NULL != ptmp_free) {
123+
free(ptmp_free);
133124
}
134125
if (NULL != displace) {
135126
free(displace);

‎ompi/mca/coll/inter/coll_inter_reduce.c

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2007 University of Houston. All rights reserved.
13-
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
14-
* Copyright (c) 2015 Research Organization for Information Science
13+
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
14+
* Copyright (c) 2015-2016 Research Organization for Information Science
1515
* and Technology (RIST). All rights reserved.
1616
* $COPYRIGHT$
1717
*
@@ -47,9 +47,6 @@ mca_coll_inter_reduce_inter(const void *sbuf, void *rbuf, int count,
4747
mca_coll_base_module_t *module)
4848
{
4949
int rank, err;
50-
ptrdiff_t true_lb, true_extent, lb, extent;
51-
char *free_buffer = NULL;
52-
char *pml_buffer = NULL;
5350

5451
/* Initialize */
5552
rank = ompi_comm_rank(comm);
@@ -58,15 +55,18 @@ mca_coll_inter_reduce_inter(const void *sbuf, void *rbuf, int count,
5855
/* do nothing */
5956
err = OMPI_SUCCESS;
6057
} else if (MPI_ROOT != root) {
58+
ptrdiff_t gap, span;
59+
char *free_buffer = NULL;
60+
char *pml_buffer = NULL;
61+
6162
/* Perform the reduce locally with the first process as root */
62-
ompi_datatype_get_extent(dtype, &lb, &extent);
63-
ompi_datatype_get_true_extent(dtype, &true_lb, &true_extent);
63+
span = opal_datatype_span(&dtype->super, count, &gap);
6464

65-
free_buffer = (char*)malloc(true_extent + (count - 1) * extent);
65+
free_buffer = (char*)malloc(span);
6666
if (NULL == free_buffer) {
6767
return OMPI_ERR_OUT_OF_RESOURCE;
6868
}
69-
pml_buffer = free_buffer - true_lb;
69+
pml_buffer = free_buffer - gap;
7070

7171
err = comm->c_local_comm->c_coll.coll_reduce(sbuf, pml_buffer, count,
7272
dtype, op, 0, comm->c_local_comm,

‎ompi/mca/coll/inter/coll_inter_scatter.c

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2008 University of Houston. All rights reserved.
13-
* Copyright (c) 2015 Research Organization for Information Science
13+
* Copyright (c) 2015-2016 Research Organization for Information Science
1414
* and Technology (RIST). All rights reserved.
1515
* $COPYRIGHT$
1616
*
@@ -44,9 +44,7 @@ mca_coll_inter_scatter_inter(const void *sbuf, int scount,
4444
int root, struct ompi_communicator_t *comm,
4545
mca_coll_base_module_t *module)
4646
{
47-
int rank, size, size_local, err;
48-
char *ptmp = NULL;
49-
ptrdiff_t lb, incr;
47+
int rank, size, err;
5048

5149
/* Initialize */
5250

@@ -58,18 +56,18 @@ mca_coll_inter_scatter_inter(const void *sbuf, int scount,
5856
err = OMPI_SUCCESS;
5957
} else if (MPI_ROOT != root) {
6058
/* First process receives the data from root */
59+
char *ptmp_free = NULL, *ptmp;
6160
if(0 == rank) {
62-
err = ompi_datatype_get_extent(rdtype, &lb, &incr);
63-
if (OMPI_SUCCESS != err) {
64-
return OMPI_ERROR;
65-
}
61+
int size_local;
62+
ptrdiff_t gap, span;
6663

67-
incr *= rcount;
6864
size_local = ompi_comm_size(comm->c_local_comm);
69-
ptmp = (char*)malloc(size_local * incr);
70-
if (NULL == ptmp) {
65+
span = opal_datatype_span(&rdtype->super, rcount*size_local, &gap);
66+
ptmp_free = malloc(span);
67+
if (NULL == ptmp_free) {
7168
return OMPI_ERR_OUT_OF_RESOURCE;
7269
}
70+
ptmp = ptmp_free - gap;
7371

7472
err = MCA_PML_CALL(recv(ptmp, rcount*size_local, rdtype,
7573
root, MCA_COLL_BASE_TAG_SCATTER,
@@ -83,8 +81,8 @@ mca_coll_inter_scatter_inter(const void *sbuf, int scount,
8381
rbuf, rcount, rdtype,
8482
0, comm->c_local_comm,
8583
comm->c_local_comm->c_coll.coll_scatter_module);
86-
if (NULL != ptmp) {
87-
free(ptmp);
84+
if (NULL != ptmp_free) {
85+
free(ptmp_free);
8886
}
8987
} else {
9088
/* Root sends data to the first process in the remote group */

0 commit comments

Comments
 (0)