Skip to content

Commit 4c0d347

Browse files
edgargabrieledgargabriel
authored
Merge pull request #4230 from edgargabriel/topic/no-smart-fview
io/ompio: add a new grouping option avoiding communication
2 parents 4c98e6b + 76a8c67 commit 4c0d347

File tree

4 files changed

+71
-57
lines changed

4 files changed

+71
-57
lines changed

‎ompi/mca/common/ompio/common_ompio_file_view.c‎

Lines changed: 66 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
#include "ompi/mca/fcoll/base/base.h"
3131
#include "ompi/mca/topo/topo.h"
3232

33-
static OMPI_MPI_OFFSET_TYPE get_contiguous_chunk_size (mca_io_ompio_file_t *);
33+
static OMPI_MPI_OFFSET_TYPE get_contiguous_chunk_size (mca_io_ompio_file_t *, int flag);
3434
static int datatype_duplicate (ompi_datatype_t *oldtype, ompi_datatype_t **newtype );
3535
static int datatype_duplicate (ompi_datatype_t *oldtype, ompi_datatype_t **newtype )
3636
{
@@ -139,7 +139,13 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh,
139139
// in orig_file type, No need to set args on this one.
140140
ompi_datatype_duplicate (newfiletype, &fh->f_filetype);
141141

142-
fh->f_cc_size = get_contiguous_chunk_size (fh);
142+
143+
if( SIMPLE_PLUS == mca_io_ompio_grouping_option ) {
144+
fh->f_cc_size = get_contiguous_chunk_size (fh, 1);
145+
}
146+
else {
147+
fh->f_cc_size = get_contiguous_chunk_size (fh, 0);
148+
}
143149

144150
if (opal_datatype_is_contiguous_memory_layout(&etype->super,1)) {
145151
if (opal_datatype_is_contiguous_memory_layout(&filetype->super,1) &&
@@ -166,7 +172,7 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh,
166172
}
167173
}
168174

169-
if ( SIMPLE != mca_io_ompio_grouping_option ) {
175+
if ( SIMPLE != mca_io_ompio_grouping_option || SIMPLE_PLUS != mca_io_ompio_grouping_option ) {
170176

171177
ret = mca_io_ompio_fview_based_grouping(fh,
172178
&num_groups,
@@ -179,6 +185,7 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh,
179185
else {
180186
int done=0;
181187
int ndims;
188+
182189
if ( fh->f_comm->c_flags & OMPI_COMM_CART ){
183190
ret = fh->f_comm->c_topo->topo.cart.cartdim_get( fh->f_comm, &ndims);
184191
if ( OMPI_SUCCESS != ret ){
@@ -253,7 +260,7 @@ int mca_common_ompio_set_view (mca_io_ompio_file_t *fh,
253260
return ret;
254261
}
255262

256-
OMPI_MPI_OFFSET_TYPE get_contiguous_chunk_size (mca_io_ompio_file_t *fh)
263+
OMPI_MPI_OFFSET_TYPE get_contiguous_chunk_size (mca_io_ompio_file_t *fh, int flag)
257264
{
258265
int uniform = 0;
259266
OMPI_MPI_OFFSET_TYPE avg[3] = {0,0,0};
@@ -268,60 +275,66 @@ OMPI_MPI_OFFSET_TYPE get_contiguous_chunk_size (mca_io_ompio_file_t *fh)
268275
** 2. each section in the file view has exactly the same size
269276
*/
270277

271-
for (i=0 ; i<(int)fh->f_iov_count ; i++) {
272-
avg[0] += fh->f_decoded_iov[i].iov_len;
273-
if (i && 0 == uniform) {
274-
if (fh->f_decoded_iov[i].iov_len != fh->f_decoded_iov[i-1].iov_len) {
275-
uniform = 1;
278+
if ( flag ) {
279+
global_avg[0] = MCA_IO_DEFAULT_FILE_VIEW_SIZE;
280+
}
281+
else {
282+
for (i=0 ; i<(int)fh->f_iov_count ; i++) {
283+
avg[0] += fh->f_decoded_iov[i].iov_len;
284+
if (i && 0 == uniform) {
285+
if (fh->f_decoded_iov[i].iov_len != fh->f_decoded_iov[i-1].iov_len) {
286+
uniform = 1;
287+
}
276288
}
277289
}
278-
}
279-
if ( 0 != fh->f_iov_count ) {
280-
avg[0] = avg[0]/fh->f_iov_count;
281-
}
282-
avg[1] = (OMPI_MPI_OFFSET_TYPE) fh->f_iov_count;
283-
avg[2] = (OMPI_MPI_OFFSET_TYPE) uniform;
284-
285-
fh->f_comm->c_coll->coll_allreduce (avg,
286-
global_avg,
287-
3,
288-
OMPI_OFFSET_DATATYPE,
289-
MPI_SUM,
290-
fh->f_comm,
291-
fh->f_comm->c_coll->coll_allreduce_module);
292-
global_avg[0] = global_avg[0]/fh->f_size;
293-
global_avg[1] = global_avg[1]/fh->f_size;
294-
290+
if ( 0 != fh->f_iov_count ) {
291+
avg[0] = avg[0]/fh->f_iov_count;
292+
}
293+
avg[1] = (OMPI_MPI_OFFSET_TYPE) fh->f_iov_count;
294+
avg[2] = (OMPI_MPI_OFFSET_TYPE) uniform;
295+
296+
fh->f_comm->c_coll->coll_allreduce (avg,
297+
global_avg,
298+
3,
299+
OMPI_OFFSET_DATATYPE,
300+
MPI_SUM,
301+
fh->f_comm,
302+
fh->f_comm->c_coll->coll_allreduce_module);
303+
global_avg[0] = global_avg[0]/fh->f_size;
304+
global_avg[1] = global_avg[1]/fh->f_size;
305+
295306
#if 0
296-
/* Disabling the feature since we are not using it anyway. Saves us one allreduce operation. */
297-
int global_uniform=0;
298-
299-
if ( global_avg[0] == avg[0] &&
300-
global_avg[1] == avg[1] &&
301-
0 == avg[2] &&
302-
0 == global_avg[2] ) {
303-
uniform = 0;
304-
}
305-
else {
306-
uniform = 1;
307+
/* Disabling the feature since we are not using it anyway. Saves us one allreduce operation. */
308+
int global_uniform=0;
309+
310+
if ( global_avg[0] == avg[0] &&
311+
global_avg[1] == avg[1] &&
312+
0 == avg[2] &&
313+
0 == global_avg[2] ) {
314+
uniform = 0;
315+
}
316+
else {
317+
uniform = 1;
318+
}
319+
320+
/* second confirmation round to see whether all processes agree
321+
** on having a uniform file view or not
322+
*/
323+
fh->f_comm->c_coll->coll_allreduce (&uniform,
324+
&global_uniform,
325+
1,
326+
MPI_INT,
327+
MPI_MAX,
328+
fh->f_comm,
329+
fh->f_comm->c_coll->coll_allreduce_module);
330+
331+
if ( 0 == global_uniform ){
332+
/* yes, everybody agrees on having a uniform file view */
333+
fh->f_flags |= OMPIO_UNIFORM_FVIEW;
334+
}
335+
#endif
307336
}
308337

309-
/* second confirmation round to see whether all processes agree
310-
** on having a uniform file view or not
311-
*/
312-
fh->f_comm->c_coll->coll_allreduce (&uniform,
313-
&global_uniform,
314-
1,
315-
MPI_INT,
316-
MPI_MAX,
317-
fh->f_comm,
318-
fh->f_comm->c_coll->coll_allreduce_module);
319-
320-
if ( 0 == global_uniform ){
321-
/* yes, everybody agrees on having a uniform file view */
322-
fh->f_flags |= OMPIO_UNIFORM_FVIEW;
323-
}
324-
#endif
325338
return global_avg[0];
326339
}
327340

‎ompi/mca/io/ompio/io_ompio.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ OMPI_DECLSPEC extern int mca_io_ompio_coll_timing_info;
111111
#define OPTIMIZE_GROUPING 4
112112
#define SIMPLE 5
113113
#define NO_REFINEMENT 6
114-
114+
#define SIMPLE_PLUS 7
115115

116116
#define OMPIO_UNIFORM_DIST_THRESHOLD 0.5
117117
#define OMPIO_CONTG_THRESHOLD 1048576

‎ompi/mca/io/ompio/io_ompio_aggregators.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -497,8 +497,9 @@ int mca_io_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh,
497497
fh->f_flags |= OMPIO_AGGREGATOR_IS_SET;
498498

499499
if (-1 == num_aggregators) {
500-
if ( SIMPLE == mca_io_ompio_grouping_option ||
501-
NO_REFINEMENT == mca_io_ompio_grouping_option ) {
500+
if ( SIMPLE == mca_io_ompio_grouping_option ||
501+
NO_REFINEMENT == mca_io_ompio_grouping_option ||
502+
SIMPLE_PLUS == mca_io_ompio_grouping_option ) {
502503
fh->f_aggregator_index = 0;
503504
fh->f_final_num_aggrs = fh->f_init_num_aggrs;
504505
fh->f_procs_per_group = fh->f_init_procs_per_group;

‎ompi/mca/io/ompio/io_ompio_component.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ static int register_component(void)
212212
"Option for grouping of processes in the aggregator selection "
213213
"1: Data volume based grouping 2: maximizing group size uniformity 3: maximimze "
214214
"data contiguity 4: hybrid optimization 5: simple (default) "
215-
"6: skip refinement step",
215+
"6: skip refinement step 7: simple+: grouping based on default file view",
216216
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
217217
OPAL_INFO_LVL_9,
218218
MCA_BASE_VAR_SCOPE_READONLY,

0 commit comments

Comments
 (0)