Skip to content

Commit cc098ba

Browse files
committed
adding external32size field to supplement remote_sizes[]
opal datatypes map MPI_LONG and MPI_INT8_T to the same OPAL_DATATYPE_INT8. This makes the remote_sizes[] array indexed by the predefined OPAL_DATATYPE_* entries inadequate to identify the external32 size for entries in a datatype's description[]. A datatype's desc[] entries have elem.common with info about the predefined OMPI type that makes up the description. This PR adds an elem.external32size. Details for setting that field:: * the OPAL datatype initializers (in OPAL_DATATYPE_INIT_DESC_PREDEFINED) create opal_datatype->desc.desc as an offset into opal_datatype_predefined_elem_desc, using the same offsets for multiple items. With the addition of an external32size field that data diverges for the different OMPI types that are using the same OPAL construction, so it had to be copied into its own space for the predefined datatypes, see the malloc/memcpy in opal_datatype_set_external32_size(). * the field is initialized for the predefined OMPI datatypes using opal_datatype_set_external32_size() performed by OMPI, then in opal_datatype_add() when description[] entries are being added it's always either pulling from the .external32size set in one of those base datatypes, or it's using that field out of a previously created description[] entry. Details for using that field: * Not all the OMPI types will have an elem.external32size set, so I didn't remove remote_sizes[], rather a non-zero external32size is used first, and if unavailable then the remote_sizes[] of the elem.common.type is used. This is found in opal_datatype_pack/unpack.c There's a similar issue in opal_datatype_compute_remote_size() that used a ptypes[] array that counted the number of each predefined OPAL_DATATYPE_* contained in the datatype. This was used to count a total remote_size. It doesn't have enough info for the same reason, so I expanded the ptypes into a struct with two entries .bdt_count[] for counting the number of each predefined OPAL_DATATYPE_* .external32size for counting the total bytes represented by the external32 size for the entries put in bdt_count[] Signed-off-by: Mark Allen <[email protected]>
1 parent 9a794e3 commit cc098ba

16 files changed

+186
-19
lines changed

ompi/datatype/ompi_datatype.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,8 @@ OMPI_DECLSPEC int ompi_datatype_unpack_external( const char datarep[], const voi
388388
OMPI_DECLSPEC int ompi_datatype_pack_external_size( const char datarep[], int incount,
389389
ompi_datatype_t *datatype, MPI_Aint *size);
390390

391+
OMPI_DECLSPEC int ompi_set_external32_sizes(void);
392+
391393
#define OMPI_DATATYPE_RETAIN(ddt) \
392394
{ \
393395
if( !ompi_datatype_is_predefined((ddt)) ) { \

ompi/datatype/ompi_datatype_external32.c

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,3 +97,39 @@ int32_t ompi_datatype_default_convertors_fini( void )
9797

9898
return OMPI_SUCCESS;
9999
}
100+
101+
int
102+
ompi_set_external32_sizes() {
103+
opal_datatype_set_external32_size(&MPI_PACKED->super, 1);
104+
opal_datatype_set_external32_size(&MPI_BYTE->super, 1);
105+
opal_datatype_set_external32_size(&MPI_CHAR->super, 1);
106+
opal_datatype_set_external32_size(&MPI_UNSIGNED_CHAR->super, 1);
107+
opal_datatype_set_external32_size(&MPI_SIGNED_CHAR->super, 1);
108+
opal_datatype_set_external32_size(&MPI_WCHAR->super, 2);
109+
opal_datatype_set_external32_size(&MPI_SHORT->super, 2);
110+
opal_datatype_set_external32_size(&MPI_UNSIGNED_SHORT->super, 2);
111+
opal_datatype_set_external32_size(&MPI_INT->super, 4);
112+
opal_datatype_set_external32_size(&MPI_UNSIGNED->super, 4);
113+
opal_datatype_set_external32_size(&MPI_LONG->super, 4);
114+
opal_datatype_set_external32_size(&MPI_UNSIGNED_LONG->super, 4);
115+
opal_datatype_set_external32_size(&MPI_FLOAT->super, 4);
116+
opal_datatype_set_external32_size(&MPI_DOUBLE->super, 8);
117+
opal_datatype_set_external32_size(&MPI_LONG_DOUBLE->super, 16);
118+
opal_datatype_set_external32_size(&MPI_CHARACTER->super, 1);
119+
opal_datatype_set_external32_size(&MPI_LOGICAL->super, 4);
120+
opal_datatype_set_external32_size(&MPI_INTEGER->super, 4);
121+
opal_datatype_set_external32_size(&MPI_REAL->super, 4);
122+
opal_datatype_set_external32_size(&MPI_DOUBLE_PRECISION->super, 8);
123+
opal_datatype_set_external32_size(&MPI_COMPLEX->super, 2*4);
124+
opal_datatype_set_external32_size(&MPI_DOUBLE_COMPLEX->super, 2*8);
125+
opal_datatype_set_external32_size(&MPI_INTEGER1->super, 1);
126+
opal_datatype_set_external32_size(&MPI_INTEGER2->super, 2);
127+
opal_datatype_set_external32_size(&MPI_INTEGER4->super, 4);
128+
opal_datatype_set_external32_size(&MPI_INTEGER8->super, 8);
129+
opal_datatype_set_external32_size(&MPI_LONG_LONG_INT->super, 8);
130+
opal_datatype_set_external32_size(&MPI_UNSIGNED_LONG_LONG->super, 8);
131+
opal_datatype_set_external32_size(&MPI_REAL4->super, 4);
132+
opal_datatype_set_external32_size(&MPI_REAL8->super, 8);
133+
opal_datatype_set_external32_size(&MPI_REAL16->super, 16);
134+
return 0;
135+
}

ompi/datatype/ompi_datatype_get_elements.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,10 @@ int ompi_datatype_get_elements (ompi_datatype_t *datatype, size_t ucount, size_t
4949
there are no leftover bytes */
5050
if (!ompi_datatype_is_predefined(datatype)) {
5151
if (0 != internal_count) {
52-
opal_datatype_compute_ptypes(&datatype->super);
52+
opal_datatype_compute_ptypes(&datatype->super, opal_convertor_get_remote_sizes(ompi_mpi_external32_convertor));
5353
/* count the basic elements in the datatype */
5454
for (i = OPAL_DATATYPE_FIRST_TYPE, total = 0 ; i < OPAL_DATATYPE_MAX_PREDEFINED ; ++i) {
55-
total += datatype->super.ptypes[i];
55+
total += datatype->super.ptypes->bdt_count[i];
5656
}
5757
internal_count = total * internal_count;
5858
}

ompi/datatype/ompi_datatype_internal.h

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,7 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX
494494
.name = OPAL_DATATYPE_INIT_NAME(TYPE ## SIZE), \
495495
.desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(TYPE ## SIZE), \
496496
.opt_desc = OPAL_DATATYPE_INIT_DESC_PREDEFINED(TYPE ## SIZE), \
497-
.ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY(TYPE ## SIZE) \
497+
.ptypes = OPAL_DATATYPE_INIT_PTYPES_ARRAY(TYPE ## SIZE, SIZE) \
498498
}
499499

500500
#define OMPI_DATATYPE_INIT_PREDEFINED_BASIC_TYPE_FORTRAN( TYPE, NAME, SIZE, ALIGN, FLAGS ) \
@@ -571,8 +571,20 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX
571571
#define OMPI_DATATYPE_INITIALIZER_UNSIGNED OPAL_DATATYPE_INITIALIZER_UINT8
572572
#endif
573573

574-
#define OMPI_DATATYPE_INITIALIZER_LONG OPAL_DATATYPE_INITIALIZER_LONG
575-
#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG OPAL_DATATYPE_INITIALIZER_ULONG
574+
//#define OMPI_DATATYPE_INITIALIZER_LONG OPAL_DATATYPE_INITIALIZER_LONG
575+
//#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG OPAL_DATATYPE_INITIALIZER_ULONG
576+
#if 1
577+
#if SIZEOF_LONG == 4
578+
#define OMPI_DATATYPE_INITIALIZER_LONG OPAL_DATATYPE_INITIALIZER_INT4
579+
#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG OPAL_DATATYPE_INITIALIZER_UINT4
580+
#elif SIZEOF_LONG == 8
581+
#define OMPI_DATATYPE_INITIALIZER_LONG OPAL_DATATYPE_INITIALIZER_INT8
582+
#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG OPAL_DATATYPE_INITIALIZER_UINT8
583+
#elif SIZEOF_LONG == 16
584+
#define OMPI_DATATYPE_INITIALIZER_LONG OPAL_DATATYPE_INITIALIZER_INT16
585+
#define OMPI_DATATYPE_INITIALIZER_UNSIGNED_LONG OPAL_DATATYPE_INITIALIZER_UINT16
586+
#endif
587+
#endif
576588

577589
#if SIZEOF_LONG_LONG == 4
578590
#define OMPI_DATATYPE_INITIALIZER_LONG_LONG_INT OPAL_DATATYPE_INITIALIZER_INT4

ompi/datatype/ompi_datatype_module.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -663,6 +663,9 @@ int32_t ompi_datatype_init( void )
663663
}
664664
}
665665
ompi_datatype_default_convertors_init();
666+
667+
ompi_set_external32_sizes();
668+
666669
return OMPI_SUCCESS;
667670
}
668671

opal/datatype/Makefile.am

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ libdatatype_la_SOURCES = \
6666
opal_datatype_pack.c \
6767
opal_datatype_position.c \
6868
opal_datatype_resize.c \
69-
opal_datatype_unpack.c
69+
opal_datatype_unpack.c \
70+
opal_datatype_setexternal32.c
7071

7172
libdatatype_la_LIBADD = libdatatype_reliable.la
7273

opal/datatype/opal_convertor.c

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -461,21 +461,27 @@ opal_datatype_compute_remote_size( const opal_datatype_t* pData,
461461
size_t length = 0;
462462

463463
if (opal_datatype_is_predefined(pData)) {
464+
if (pData->desc.desc->elem.common.external32size != 0) {
465+
return pData->desc.desc->elem.common.external32size;
466+
}
464467
return sizes[pData->desc.desc->elem.common.type];
465468
}
466469

467470
if( OPAL_UNLIKELY(NULL == pData->ptypes) ) {
468471
/* Allocate and fill the array of types used in the datatype description */
469-
opal_datatype_compute_ptypes( (opal_datatype_t*)pData );
472+
opal_datatype_compute_ptypes( (opal_datatype_t*)pData , sizes );
470473
}
474+
return pData->ptypes->external32size;
471475

476+
#if 0
472477
for( int i = OPAL_DATATYPE_FIRST_TYPE; typeMask && (i < OPAL_DATATYPE_MAX_PREDEFINED); i++ ) {
473478
if( typeMask & ((uint32_t)1 << i) ) {
474479
length += (pData->ptypes[i] * sizes[i]);
475480
typeMask ^= ((uint32_t)1 << i);
476481
}
477482
}
478483
return length;
484+
#endif
479485
}
480486

481487
/**
@@ -772,3 +778,9 @@ void opal_datatype_dump_stack( const dt_stack_t* pStack, int stack_pos,
772778
}
773779
opal_output( 0, "\n" );
774780
}
781+
782+
size_t*
783+
opal_convertor_get_remote_sizes(opal_convertor_t* pConvertor)
784+
{
785+
return pConvertor->master->remote_sizes;
786+
}

opal/datatype/opal_convertor.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,9 @@ OPAL_DECLSPEC int
397397
opal_convertor_generic_simple_position( opal_convertor_t* pConvertor,
398398
size_t* position );
399399

400+
OPAL_DECLSPEC size_t*
401+
opal_convertor_get_remote_sizes(opal_convertor_t* pConvertor);
402+
400403
END_C_DECLS
401404

402405
#endif /* OPAL_CONVERTOR_H_HAS_BEEN_INCLUDED */

opal/datatype/opal_datatype.h

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,18 @@ struct dt_type_desc_t {
101101
};
102102
typedef struct dt_type_desc_t dt_type_desc_t;
103103

104+
/*
105+
* ptypes structure
106+
*
107+
* There used to just be a ptypes[] array that was a count of how
108+
* many of each base OPAL_DATATYPE_ was in a given datatype. This
109+
* keeps that but adds another field for the total external32size
110+
* of all those types combined.
111+
*/
112+
typedef struct {
113+
size_t bdt_count[OPAL_DATATYPE_MAX_SUPPORTED];
114+
size_t external32size;
115+
} ptypes_t;
104116

105117
/*
106118
* The datatype description.
@@ -127,7 +139,7 @@ struct opal_datatype_t {
127139
dt_type_desc_t opt_desc; /**< short description of the data used when conversion is useless
128140
or in the send case (without conversion) */
129141

130-
size_t *ptypes; /**< array of basic predefined types that facilitate the computing
142+
ptypes_t *ptypes; /**< array of basic predefined types that facilitate the computing
131143
of the remote size in heterogeneous environments. The length of the
132144
array is dependent on the maximum number of predefined datatypes of
133145
all language interfaces (because Fortran is not known at the OPAL
@@ -316,7 +328,7 @@ OPAL_DECLSPEC int32_t
316328
opal_datatype_copy_content_same_ddt( const opal_datatype_t* pData, int32_t count,
317329
char* pDestBuf, char* pSrcBuf );
318330

319-
OPAL_DECLSPEC int opal_datatype_compute_ptypes( opal_datatype_t* datatype );
331+
OPAL_DECLSPEC int opal_datatype_compute_ptypes( opal_datatype_t* datatype, const size_t* sizes );
320332

321333
/* Compute the span in memory of count datatypes. This function help with temporary
322334
* memory allocations for receiving already typed data (such as those used for reduce
@@ -341,6 +353,13 @@ opal_datatype_span( const opal_datatype_t* pData, size_t count,
341353
return true_extent + extent * (count - 1);
342354
}
343355

356+
/*
357+
* For use with the bottom level MPI datatypes
358+
* Set the .external32size field in the datatype's
359+
* description[].elem.common entry
360+
*/
361+
int opal_datatype_set_external32_size(opal_datatype_t *pData, int e32sz);
362+
344363
#if OPAL_ENABLE_DEBUG
345364
/*
346365
* Set a breakpoint to this function in your favorite debugger

opal/datatype/opal_datatype_add.c

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -279,11 +279,14 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
279279
* predefined non contiguous datatypes (like MPI_SHORT_INT).
280280
*/
281281
if( (pdtAdd->flags & (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_DATA)) == (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_DATA) ) {
282-
if( NULL != pdtBase->ptypes )
283-
pdtBase->ptypes[pdtAdd->id] += count;
282+
if( NULL != pdtBase->ptypes ) {
283+
pdtBase->ptypes->bdt_count[pdtAdd->id] += count;
284+
pdtBase->ptypes->external32size += count * pdtAdd->desc.desc->elem.common.external32size;
285+
}
284286

285287
pLast->elem.common.flags = pdtAdd->flags & ~(OPAL_DATATYPE_FLAG_COMMITTED);
286288
pLast->elem.common.type = pdtAdd->id;
289+
pLast->elem.common.external32size = pdtAdd->desc.desc->elem.common.external32size;
287290
pLast->elem.disp = disp;
288291
pLast->elem.extent = (ptrdiff_t)count * extent;
289292
/* assume predefined datatypes without extent, aka. contiguous */
@@ -305,7 +308,12 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
305308
pdtBase->flags |= (pdtAdd->flags & OPAL_DATATYPE_FLAG_USER_UB);
306309
if( (NULL != pdtBase->ptypes) && (NULL != pdtAdd->ptypes) ) {
307310
for( i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++ )
308-
if( pdtAdd->ptypes[i] != 0 ) pdtBase->ptypes[i] += (count * pdtAdd->ptypes[i]);
311+
if( pdtAdd->ptypes->bdt_count[i] != 0 ) pdtBase->ptypes->bdt_count[i] += (count * pdtAdd->ptypes->bdt_count[i]);
312+
pdtBase->ptypes->external32size += pdtAdd->ptypes->external32size;
313+
} else if (NULL != pdtBase->ptypes) {
314+
/* (We had ptypes data but didn't update it, must invalidate) */
315+
free(pdtBase->ptypes);
316+
pdtBase->ptypes = NULL;
309317
}
310318
if( 1 == pdtAdd->desc.used ) {
311319
pLast->elem = pdtAdd->desc.desc[0].elem;

0 commit comments

Comments
 (0)