Skip to content
This repository was archived by the owner on Sep 30, 2022. It is now read-only.

Commit f3bad94

Browse files
author
rhc54
committed
Merge pull request #1004 from ggouaillardet/topic/v1.10/nbc_ineighbor_alltoallw_in_place
coll/libnbc: do not handle MPI_IN_PLACE in neighborhood collectives
2 parents 1609d1d + ea2fcb9 commit f3bad94

15 files changed

+106
-290
lines changed

‎ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c‎

Lines changed: 17 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
/*
2-
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
3-
* University Research and Technology
4-
* Corporation. All rights reserved.
5-
* Copyright (c) 2006 The Technical University of Chemnitz. All
6-
* rights reserved.
2+
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
3+
* University Research and Technology
4+
* Corporation. All rights reserved.
5+
* Copyright (c) 2006 The Technical University of Chemnitz. All
6+
* rights reserved.
7+
* Copyright (c) 2016 Research Organization for Information Science
8+
* and Technology (RIST). All rights reserved.
79
*
810
* Author(s): Torsten Hoefler <[email protected]>
911
*
@@ -56,14 +58,11 @@ int ompi_coll_libnbc_ineighbor_allgather(void *sbuf, int scount, MPI_Datatype st
5658
res = MPI_Type_extent(rtype, &rcvext);
5759
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
5860

59-
char inplace;
6061
NBC_Schedule *schedule;
6162
#ifdef NBC_CACHE_SCHEDULE
6263
NBC_Ineighbor_allgather_args *args, *found, search;
6364
#endif
6465

65-
NBC_IN_PLACE(sbuf, rbuf, inplace);
66-
6766
handle->tmpbuf=NULL;
6867

6968
#ifdef NBC_CACHE_SCHEDULE
@@ -93,41 +92,17 @@ int ompi_coll_libnbc_ineighbor_allgather(void *sbuf, int scount, MPI_Datatype st
9392
res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
9493
if(res != NBC_OK) return res;
9594

96-
if(inplace) { /* we need an extra buffer to be deadlock-free */
97-
handle->tmpbuf = malloc(indegree*rcvext*rcount);
98-
99-
for(i = 0; i < indegree; i++) {
100-
if (MPI_PROC_NULL != srcs[i]) {
101-
res = NBC_Sched_recv((char*)0+i*rcount*rcvext, true, rcount, rtype, srcs[i], schedule);
102-
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
103-
}
104-
}
105-
for(i = 0; i < outdegree; i++) {
106-
if (MPI_PROC_NULL != dsts[i]) {
107-
res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
108-
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
109-
}
95+
/* simply loop over neighbors and post send/recv operations */
96+
for(i = 0; i < indegree; i++) {
97+
if (MPI_PROC_NULL != srcs[i]) {
98+
res = NBC_Sched_recv((char*)rbuf+i*rcount*rcvext, false, rcount, rtype, srcs[i], schedule);
99+
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
110100
}
111-
/* unpack from buffer */
112-
for(i = 0; i < indegree; i++) {
113-
res = NBC_Sched_barrier(schedule);
114-
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
115-
res = NBC_Sched_copy((char*)0+i*rcount*rcvext, true, rcount, rtype, (char*)rbuf+i*rcount*rcvext, false, rcount, rtype, schedule);
116-
if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
117-
}
118-
} else { /* non INPLACE case */
119-
/* simply loop over neighbors and post send/recv operations */
120-
for(i = 0; i < indegree; i++) {
121-
if (MPI_PROC_NULL != srcs[i]) {
122-
res = NBC_Sched_recv((char*)rbuf+i*rcount*rcvext, false, rcount, rtype, srcs[i], schedule);
123-
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
124-
}
125-
}
126-
for(i = 0; i < outdegree; i++) {
127-
if (MPI_PROC_NULL != dsts[i]) {
128-
res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
129-
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
130-
}
101+
}
102+
for(i = 0; i < outdegree; i++) {
103+
if (MPI_PROC_NULL != dsts[i]) {
104+
res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
105+
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
131106
}
132107
}
133108
}

‎ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c‎

Lines changed: 17 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
/*
2-
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
3-
* University Research and Technology
4-
* Corporation. All rights reserved.
5-
* Copyright (c) 2006 The Technical University of Chemnitz. All
6-
* rights reserved.
2+
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
3+
* University Research and Technology
4+
* Corporation. All rights reserved.
5+
* Copyright (c) 2006 The Technical University of Chemnitz. All
6+
* rights reserved.
7+
* Copyright (c) 2016 Research Organization for Information Science
8+
* and Technology (RIST). All rights reserved.
79
*
810
* Author(s): Torsten Hoefler <[email protected]>
911
*
@@ -57,14 +59,11 @@ int ompi_coll_libnbc_ineighbor_allgatherv(void *sbuf, int scount, MPI_Datatype s
5759
res = MPI_Type_extent(rtype, &rcvext);
5860
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
5961

60-
char inplace;
6162
NBC_Schedule *schedule;
6263
#ifdef NBC_CACHE_SCHEDULE
6364
NBC_Ineighbor_allgatherv_args *args, *found, search;
6465
#endif
6566

66-
NBC_IN_PLACE(sbuf, rbuf, inplace);
67-
6867
handle->tmpbuf=NULL;
6968

7069
#ifdef NBC_CACHE_SCHEDULE
@@ -94,50 +93,17 @@ int ompi_coll_libnbc_ineighbor_allgatherv(void *sbuf, int scount, MPI_Datatype s
9493
res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
9594
if(res != NBC_OK) return res;
9695

97-
if(inplace) { /* we need an extra buffer to be deadlock-free */
98-
int sumrcounts=0;
99-
int offset=0;
100-
for(i=0; i<indegree; ++i) sumrcounts += rcounts[i];
101-
handle->tmpbuf = malloc(rcvext*sumrcounts);
102-
103-
for(i = 0; i < indegree; i++) {
104-
if(srcs[i] != MPI_PROC_NULL) {
105-
res = NBC_Sched_recv((char*)0+offset, true, rcounts[i], rtype, srcs[i], schedule);
106-
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
107-
}
108-
offset += rcounts[i]*rcvext;
109-
}
110-
for(i = 0; i < outdegree; i++) {
111-
if(dsts[i] != MPI_PROC_NULL) {
112-
res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
113-
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
114-
}
115-
}
116-
/* unpack from buffer */
117-
offset=0;
118-
for(i = 0; i < indegree; i++) {
119-
if(srcs[i] != MPI_PROC_NULL) {
120-
res = NBC_Sched_barrier(schedule);
121-
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
122-
res = NBC_Sched_copy((char*)0+offset, true, rcounts[i], rtype, (char*)rbuf+displs[i]*rcvext, false, rcounts[i], rtype, schedule);
123-
if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
124-
}
125-
offset += rcounts[i]*rcvext;
96+
/* simply loop over neighbors and post send/recv operations */
97+
for(i = 0; i < indegree; i++) {
98+
if(srcs[i] != MPI_PROC_NULL) {
99+
res = NBC_Sched_recv((char*)rbuf+displs[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
100+
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
126101
}
127-
} else { /* non INPLACE case */
128-
129-
/* simply loop over neighbors and post send/recv operations */
130-
for(i = 0; i < indegree; i++) {
131-
if(srcs[i] != MPI_PROC_NULL) {
132-
res = NBC_Sched_recv((char*)rbuf+displs[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
133-
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
134-
}
135-
}
136-
for(i = 0; i < outdegree; i++) {
137-
if(dsts[i] != MPI_PROC_NULL) {
138-
res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
139-
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
140-
}
102+
}
103+
for(i = 0; i < outdegree; i++) {
104+
if(dsts[i] != MPI_PROC_NULL) {
105+
res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
106+
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
141107
}
142108
}
143109
}

‎ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c‎

Lines changed: 17 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
/*
2-
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
3-
* University Research and Technology
4-
* Corporation. All rights reserved.
5-
* Copyright (c) 2006 The Technical University of Chemnitz. All
6-
* rights reserved.
2+
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
3+
* University Research and Technology
4+
* Corporation. All rights reserved.
5+
* Copyright (c) 2006 The Technical University of Chemnitz. All
6+
* rights reserved.
7+
* Copyright (c) 2016 Research Organization for Information Science
8+
* and Technology (RIST). All rights reserved.
79
*
810
* Author(s): Torsten Hoefler <[email protected]>
911
*
@@ -55,14 +57,11 @@ int ompi_coll_libnbc_ineighbor_alltoall(void *sbuf, int scount, MPI_Datatype sty
5557
res = MPI_Type_extent(rtype, &rcvext);
5658
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
5759

58-
char inplace;
5960
NBC_Schedule *schedule;
6061
#ifdef NBC_CACHE_SCHEDULE
6162
NBC_Ineighbor_alltoall_args *args, *found, search;
6263
#endif
6364

64-
NBC_IN_PLACE(sbuf, rbuf, inplace);
65-
6665
handle->tmpbuf=NULL;
6766

6867
#ifdef NBC_CACHE_SCHEDULE
@@ -92,41 +91,17 @@ int ompi_coll_libnbc_ineighbor_alltoall(void *sbuf, int scount, MPI_Datatype sty
9291
res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
9392
if(res != NBC_OK) return res;
9493

95-
if(inplace) { /* we need an extra buffer to be deadlock-free */
96-
handle->tmpbuf = malloc(indegree*rcvext*rcount);
97-
98-
for(i = 0; i < indegree; i++) {
99-
if (MPI_PROC_NULL != srcs[i]) {
100-
res = NBC_Sched_recv((char*)0+i*rcount*rcvext, true, rcount, rtype, srcs[i], schedule);
101-
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
102-
}
103-
}
104-
for(i = 0; i < outdegree; i++) {
105-
if (MPI_PROC_NULL != dsts[i]) {
106-
res = NBC_Sched_send((char*)sbuf+i*scount*sndext, false, scount, stype, dsts[i], schedule);
107-
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
108-
}
94+
/* simply loop over neighbors and post send/recv operations */
95+
for(i = 0; i < indegree; i++) {
96+
if (MPI_PROC_NULL != srcs[i]) {
97+
res = NBC_Sched_recv((char*)rbuf+i*rcount*rcvext, false, rcount, rtype, srcs[i], schedule);
98+
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
10999
}
110-
/* unpack from buffer */
111-
for(i = 0; i < indegree; i++) {
112-
res = NBC_Sched_barrier(schedule);
113-
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
114-
res = NBC_Sched_copy((char*)0+i*rcount*rcvext, true, rcount, rtype, (char*)rbuf+i*rcount*rcvext, false, rcount, rtype, schedule);
115-
if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
116-
}
117-
} else { /* non INPLACE case */
118-
/* simply loop over neighbors and post send/recv operations */
119-
for(i = 0; i < indegree; i++) {
120-
if (MPI_PROC_NULL != srcs[i]) {
121-
res = NBC_Sched_recv((char*)rbuf+i*rcount*rcvext, false, rcount, rtype, srcs[i], schedule);
122-
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
123-
}
124-
}
125-
for(i = 0; i < outdegree; i++) {
126-
if (MPI_PROC_NULL != dsts[i]) {
127-
res = NBC_Sched_send((char*)sbuf+i*scount*sndext, false, scount, stype, dsts[i], schedule);
128-
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
129-
}
100+
}
101+
for(i = 0; i < outdegree; i++) {
102+
if (MPI_PROC_NULL != dsts[i]) {
103+
res = NBC_Sched_send((char*)sbuf+i*scount*sndext, false, scount, stype, dsts[i], schedule);
104+
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
130105
}
131106
}
132107
}

‎ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c‎

Lines changed: 17 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
/*
2-
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
3-
* University Research and Technology
4-
* Corporation. All rights reserved.
5-
* Copyright (c) 2006 The Technical University of Chemnitz. All
6-
* rights reserved.
2+
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
3+
* University Research and Technology
4+
* Corporation. All rights reserved.
5+
* Copyright (c) 2006 The Technical University of Chemnitz. All
6+
* rights reserved.
7+
* Copyright (c) 2016 Research Organization for Information Science
8+
* and Technology (RIST). All rights reserved.
79
*
810
* Author(s): Torsten Hoefler <[email protected]>
911
*
@@ -57,14 +59,11 @@ int ompi_coll_libnbc_ineighbor_alltoallv(void *sbuf, int *scounts, int *sdispls,
5759
res = MPI_Type_extent(rtype, &rcvext);
5860
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
5961

60-
char inplace;
6162
NBC_Schedule *schedule;
6263
#ifdef NBC_CACHE_SCHEDULE
6364
NBC_Ineighbor_alltoallv_args *args, *found, search;
6465
#endif
6566

66-
NBC_IN_PLACE(sbuf, rbuf, inplace);
67-
6867
handle->tmpbuf=NULL;
6968

7069
#ifdef NBC_CACHE_SCHEDULE
@@ -94,50 +93,17 @@ int ompi_coll_libnbc_ineighbor_alltoallv(void *sbuf, int *scounts, int *sdispls,
9493
res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
9594
if(res != NBC_OK) return res;
9695

97-
if(inplace) { /* we need an extra buffer to be deadlock-free */
98-
int sumrcounts=0;
99-
int offset=0;
100-
for(i=0; i<indegree; ++i) sumrcounts += rcounts[i];
101-
handle->tmpbuf = malloc(rcvext*sumrcounts);
102-
103-
for(i = 0; i < indegree; i++) {
104-
if(srcs[i] != MPI_PROC_NULL) {
105-
res = NBC_Sched_recv((char*)0+offset, true, rcounts[i], rtype, srcs[i], schedule);
106-
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
107-
}
108-
offset += rcounts[i]*rcvext;
109-
}
110-
111-
for(i = 0; i < outdegree; i++) {
112-
if(dsts[i] != MPI_PROC_NULL) {
113-
res = NBC_Sched_send((char*)sbuf+sdispls[i]*sndext, false, scounts[i], stype, dsts[i], schedule);
114-
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
115-
}
96+
/* simply loop over neighbors and post send/recv operations */
97+
for(i = 0; i < indegree; i++) {
98+
if(srcs[i] != MPI_PROC_NULL) {
99+
res = NBC_Sched_recv((char*)rbuf+rdispls[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
100+
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
116101
}
117-
/* unpack from buffer */
118-
offset=0;
119-
for(i = 0; i < indegree; i++) {
120-
if(srcs[i] != MPI_PROC_NULL) {
121-
res = NBC_Sched_barrier(schedule);
122-
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
123-
res = NBC_Sched_copy((char*)0+offset, true, rcounts[i], rtype, (char*)rbuf+rdispls[i]*rcvext, false, rcounts[i], rtype, schedule);
124-
if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
125-
}
126-
offset += rcounts[i]*rcvext;
127-
}
128-
} else { /* non INPLACE case */
129-
/* simply loop over neighbors and post send/recv operations */
130-
for(i = 0; i < indegree; i++) {
131-
if(srcs[i] != MPI_PROC_NULL) {
132-
res = NBC_Sched_recv((char*)rbuf+rdispls[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
133-
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
134-
}
135-
}
136-
for(i = 0; i < outdegree; i++) {
137-
if(dsts[i] != MPI_PROC_NULL) {
138-
res = NBC_Sched_send((char*)sbuf+sdispls[i]*sndext, false, scounts[i], stype, dsts[i], schedule);
139-
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
140-
}
102+
}
103+
for(i = 0; i < outdegree; i++) {
104+
if(dsts[i] != MPI_PROC_NULL) {
105+
res = NBC_Sched_send((char*)sbuf+sdispls[i]*sndext, false, scounts[i], stype, dsts[i], schedule);
106+
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
141107
}
142108
}
143109
}

0 commit comments

Comments
 (0)