Optimized datatype description.

Move toward a base type of vector (count, type, blocklen, extent, disp)
with disp and extent applying toward the count repertition and blocklen
being a contiguous memory of type type.
Implement 2 optimizations on this description used during type_commit:
- collapse: successive similar datatype descriptions are collapsed
together with an increased count.
- fusion: fuse successive datatype descriptions in order to minimize the
number of resulting memcpy during pack/unpack.

Fixes at the OMPI datatype level including:
 - Fix the create_hindexed and vector creation.
 - Fix the handling of [get|set]_elements and _count.
 - Correctly compute the dispacement for block indexed types.
 - Support the MPI_LB and MPI_UB deprecation, aka. OMPI_ENABLE_MPI1_COMPAT.

Signed-off-by: George Bosilca <[email protected]>

Author: bosilca

ompi/datatype/ompi_datatype.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2009-2013 The University of Tennessee and The University
+ * Copyright (c) 2009-2019 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.

ompi/datatype/ompi_datatype_create_indexed.c

@@ -87,10 +87,10 @@ int32_t ompi_datatype_create_hindexed( int count, const int* pBlockLength, const
         return ompi_datatype_duplicate( &ompi_mpi_datatype_null.dt, newType);
     }
 
+    ompi_datatype_type_extent( oldType, &extent );
     disp = pDisp[i];
     dLength = pBlockLength[i];
     endat = disp + dLength * extent;
-    ompi_datatype_type_extent( oldType, &extent );
 
     pdt = ompi_datatype_create( (count - i) * (2 + oldType->super.desc.used) );
     for( i += 1; i < count; i++ ) {
@@ -162,17 +162,17 @@ int32_t ompi_datatype_create_hindexed_block( int count, int bLength, const ptrdi
     pdt = ompi_datatype_create( count * (2 + oldType->super.desc.used) );
     disp = pDisp[0];
     dLength = bLength;
-    endat = disp + dLength;
+    endat = disp + dLength * extent;
     for( i = 1; i < count; i++ ) {
         if( endat == pDisp[i] ) {
             /* contiguous with the previsious */
             dLength += bLength;
-            endat += bLength;
+            endat += bLength * extent;
         } else {
             ompi_datatype_add( pdt, oldType, dLength, disp, extent );
             disp = pDisp[i];
             dLength = bLength;
-            endat = disp + bLength;
+            endat = disp + bLength * extent;
         }
     }
     ompi_datatype_add( pdt, oldType, dLength, disp, extent );

ompi/datatype/ompi_datatype_external.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2019 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -26,7 +26,6 @@
 #include <stdio.h>
 
 #include "ompi/runtime/params.h"
-#include "ompi/communicator/communicator.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "opal/datatype/opal_convertor.h"
 

opal/datatype/opal_convertor.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2019 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
@@ -324,8 +324,9 @@ int32_t opal_convertor_unpack( opal_convertor_t* pConv,
     return pConv->fAdvance( pConv, iov, out_size, max_data );
 }
 
-static inline int opal_convertor_create_stack_with_pos_contig( opal_convertor_t* pConvertor,
-                                                               size_t starting_point, const size_t* sizes )
+static inline int
+opal_convertor_create_stack_with_pos_contig( opal_convertor_t* pConvertor,
+                                             size_t starting_point, const size_t* sizes )
 {
     dt_stack_t* pStack;   /* pointer to the position on the stack */
     const opal_datatype_t* pData = pConvertor->pDesc;
@@ -349,7 +350,7 @@ static inline int opal_convertor_create_stack_with_pos_contig( opal_convertor_t*
     pStack[0].disp     = count * extent;
 
     /* now compute the number of pending bytes */
-    count = starting_point - count * pData->size;
+    count = starting_point % pData->size;
     /**
      * We save the current displacement starting from the begining
      * of this data.
@@ -370,9 +371,9 @@ static inline int opal_convertor_create_stack_with_pos_contig( opal_convertor_t*
     return OPAL_SUCCESS;
 }
 
-static inline
-int opal_convertor_create_stack_at_begining( opal_convertor_t* convertor,
-                                             const size_t* sizes )
+static inline int
+opal_convertor_create_stack_at_begining( opal_convertor_t* convertor,
+                                         const size_t* sizes )
 {
     dt_stack_t* pStack = convertor->pStack;
     dt_elem_desc_t* pElems;
@@ -402,7 +403,7 @@ int opal_convertor_create_stack_at_begining( opal_convertor_t* convertor,
         pStack[1].count = pElems[0].loop.loops;
         pStack[1].type  = OPAL_DATATYPE_LOOP;
     } else {
-        pStack[1].count = pElems[0].elem.count;
+        pStack[1].count = pElems[0].elem.count * pElems[0].elem.blocklen;
         pStack[1].type  = pElems[0].elem.common.type;
     }
     return OPAL_SUCCESS;

opal/datatype/opal_datatype.h

@@ -224,13 +224,41 @@ opal_datatype_is_contiguous_memory_layout( const opal_datatype_t* datatype, int3
 }
 
 
-OPAL_DECLSPEC void opal_datatype_dump( const opal_datatype_t* pData );
+OPAL_DECLSPEC void
+opal_datatype_dump( const opal_datatype_t* pData );
+
 /* data creation functions */
-OPAL_DECLSPEC int32_t opal_datatype_clone( const opal_datatype_t * src_type, opal_datatype_t * dest_type );
-OPAL_DECLSPEC int32_t opal_datatype_create_contiguous( int count, const opal_datatype_t* oldType, opal_datatype_t** newType );
-OPAL_DECLSPEC int32_t opal_datatype_resize( opal_datatype_t* type, ptrdiff_t lb, ptrdiff_t extent );
-OPAL_DECLSPEC int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtAdd, size_t count,
-                                         ptrdiff_t disp, ptrdiff_t extent );
+
+/**
+ * Create a duplicate of the source datatype.
+ */
+OPAL_DECLSPEC int32_t
+opal_datatype_clone( const opal_datatype_t* src_type,
+                     opal_datatype_t* dest_type );
+/**
+ * A contiguous array of identical datatypes.
+ */
+OPAL_DECLSPEC int32_t
+opal_datatype_create_contiguous( int count, const opal_datatype_t* oldType,
+                                 opal_datatype_t** newType );
+/**
+ * Add a new datatype to the base type description. The count is the number
+ * repetitions of the same element to be added, and the extent is the extent
+ * of each element. The displacement is the initial displacement of the
+ * first element.
+ */
+OPAL_DECLSPEC int32_t
+opal_datatype_add( opal_datatype_t* pdtBase,
+                   const opal_datatype_t* pdtAdd, size_t count,
+                   ptrdiff_t disp, ptrdiff_t extent );
+
+/**
+ * Alter the lb and extent of an existing datatype in place.
+ */
+OPAL_DECLSPEC int32_t
+opal_datatype_resize( opal_datatype_t* type,
+                      ptrdiff_t lb,
+                      ptrdiff_t extent );
 
 static inline int32_t
 opal_datatype_type_lb( const opal_datatype_t* pData, ptrdiff_t* disp )

opal/datatype/opal_datatype_add.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2019 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
@@ -281,15 +281,23 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
     if( (pdtAdd->flags & (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_DATA)) == (OPAL_DATATYPE_FLAG_PREDEFINED | OPAL_DATATYPE_FLAG_DATA) ) {
         if( NULL != pdtBase->ptypes )
             pdtBase->ptypes[pdtAdd->id] += count;
+
+        pLast->elem.common.flags     = pdtAdd->flags & ~(OPAL_DATATYPE_FLAG_COMMITTED);
         pLast->elem.common.type      = pdtAdd->id;
-        pLast->elem.count            = count;
         pLast->elem.disp             = disp;
-        pLast->elem.extent           = extent;
-        pdtBase->desc.used++;
-        pLast->elem.common.flags     = pdtAdd->flags & ~(OPAL_DATATYPE_FLAG_COMMITTED);
-        if( (extent != (ptrdiff_t)pdtAdd->size) && (count > 1) ) {  /* gaps around the datatype */
-            pLast->elem.common.flags &= ~(OPAL_DATATYPE_FLAG_CONTIGUOUS | OPAL_DATATYPE_FLAG_NO_GAPS);
+        pLast->elem.extent           = count * extent;
+        /* assume predefined datatypes without extent, aka. contiguous */
+        pLast->elem.count            = 1;
+        pLast->elem.blocklen         = count;
+        if( extent != (ptrdiff_t)pdtAdd->size ) {  /* not contiguous: let's fix */
+            pLast->elem.count            = count;
+            pLast->elem.blocklen         = 1;
+            pLast->elem.extent           = extent;
+            if( count > 1 ) {  /* gaps around the predefined datatype */
+                pLast->elem.common.flags &= ~(OPAL_DATATYPE_FLAG_CONTIGUOUS | OPAL_DATATYPE_FLAG_NO_GAPS);
+            }
         }
+        pdtBase->desc.used++;
     } else {
         /* keep trace of the total number of basic datatypes in the datatype definition */
         pdtBase->loops += pdtAdd->loops;
@@ -299,13 +307,40 @@ int32_t opal_datatype_add( opal_datatype_t* pdtBase, const opal_datatype_t* pdtA
             for( i = OPAL_DATATYPE_FIRST_TYPE; i < OPAL_DATATYPE_MAX_PREDEFINED; i++ )
                 if( pdtAdd->ptypes[i] != 0 ) pdtBase->ptypes[i] += (count * pdtAdd->ptypes[i]);
         }
-        if( (1 == pdtAdd->desc.used) && (extent == (pdtAdd->ub - pdtAdd->lb)) &&
-            (extent == pdtAdd->desc.desc[0].elem.extent) ){
+        if( 1 == pdtAdd->desc.used ) {
             pLast->elem        = pdtAdd->desc.desc[0].elem;
-            pLast->elem.count *= count;
             pLast->elem.disp  += disp;
+            if( 1 == count ) {
+                /* Extent only has a meaning when there are multiple elements. Bail out */
+            } else if( 1 == pLast->elem.count ) {
+                /* The size and true_extent of the added datatype are identical, signaling a datatype
+                 * that is mostly contiguous with the exception of the initial and final gaps. These
+                 * gaps do not matter here as they will amended (the initial gaps being shifted by the
+                 * new displacement and the final gap being replaced with the new gap
+                 */
+                if( pdtAdd->desc.desc[0].elem.extent == extent ) {
+                    /* pure bliss everything is fully contiguous and we can collapse
+                     * everything by updating the blocklen and extent
+                     */
+                    pLast->elem.blocklen *= count;
+                    pLast->elem.extent   *= count;
+                } else {
+                    pLast->elem.count = count;
+                    pLast->elem.extent = extent;
+                }
+            } else if( extent == (ptrdiff_t)(pLast->elem.count * pLast->elem.extent) ) {
+                /* It's just a repetition of the same element, increase the count */
+                pLast->elem.count *= count;
+            } else {
+                /* No luck here, no optimization can be applied. Fall back to the
+                 * normal case where we add a loop around the datatype.
+                 */
+                goto build_loop;
+            }
             pdtBase->desc.used++;
         } else {
+
+build_loop:
             /* if the extent of the datatype is the same as the extent of the loop
              * description of the datatype then we simply have to update the main loop.
              */

opal/datatype/opal_datatype_copy.h

@@ -48,37 +48,37 @@ static inline void _predefined_data( const dt_elem_desc_t* ELEM,
                                      unsigned char* DESTINATION,
                                      size_t* SPACE )
 {
-    size_t _copy_count = (COUNT);
-    size_t _copy_blength;
     const ddt_elem_desc_t* _elem = &((ELEM)->elem);
     unsigned char* _source = (SOURCE) + _elem->disp;
     unsigned char* _destination = (DESTINATION) + _elem->disp;
+    size_t total_count = _elem->count * _elem->blocklen;
+    size_t do_now, do_now_bytes;
 
-    _copy_blength = opal_datatype_basicDatatypes[_elem->common.type]->size;
+    assert( (COUNT) == total_count);
+    assert( total_count <= ((*SPACE) / opal_datatype_basicDatatypes[_elem->common.type]->size) );
 
-    if( _copy_blength == (size_t)_elem->extent ) {
-        _copy_blength *= _copy_count;
-        OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_blength, (SOURCE_BASE),
-                                    (DATATYPE), (TOTAL_COUNT) );
-        /* the extent and the size of the basic datatype are equals */
-        DO_DEBUG( opal_output( 0, "copy 1. %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
-                               STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, _copy_blength, *(SPACE) ); );
-        MEM_OP( _destination, _source, _copy_blength );
-        _source      += _copy_blength;
-        _destination += _copy_blength;
-    } else {
-        for(size_t _i = 0; _i < _copy_count; _i++ ) {
-            OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_blength, (SOURCE_BASE),
-                                        (DATATYPE), (TOTAL_COUNT) );
-            DO_DEBUG( opal_output( 0, "copy 2. %s( %p, %p, %lu ) => space %lu\n",
-                                   STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, (unsigned long)_copy_blength, (unsigned long)(*(SPACE) - (_i * _copy_blength)) ); );
-            MEM_OP( _destination, _source, _copy_blength );
-            _source      += _elem->extent;
+    /* We don't a prologue and epilogue here as we are __always__ working
+     * with full copies of the data description.
+     */
+
+    /**
+     * Compute how many full blocklen we need to do and do them.
+     */
+    do_now = _elem->count;
+    if( 0 != do_now ) {
+        do_now_bytes = _elem->blocklen * opal_datatype_basicDatatypes[_elem->common.type]->size;
+        for(size_t _i = 0; _i < do_now; _i++ ) {
+            OPAL_DATATYPE_SAFEGUARD_POINTER( _source, do_now_bytes, (SOURCE_BASE),
+                                            (DATATYPE), (TOTAL_COUNT) );
+            DO_DEBUG( opal_output( 0, "copy %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
+                                   STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, do_now_bytes, *(SPACE) ); );
+            MEM_OP( _destination, _source, do_now_bytes );
             _destination += _elem->extent;
+            _source      += _elem->extent;
+            *(SPACE)     -= do_now_bytes;
         }
-        _copy_blength *= _copy_count;
+        (COUNT)      -= total_count;
     }
-    *(SPACE)      -= _copy_blength;
 }
 
 static inline void _contiguous_loop( const dt_elem_desc_t* ELEM,
@@ -147,12 +147,10 @@ static inline int32_t _copy_content_same_ddt( const opal_datatype_t* datatype, i
         if( (ptrdiff_t)datatype->size == extent ) {  /* all contiguous == no gaps around */
             size_t total_length = iov_len_local;
             size_t memop_chunk = opal_datatype_memop_block_size;
+            OPAL_DATATYPE_SAFEGUARD_POINTER( source, iov_len_local,
+                                             (unsigned char*)source_base, datatype, count );
             while( total_length > 0 ) {
                 if( memop_chunk > total_length ) memop_chunk = total_length;
-                OPAL_DATATYPE_SAFEGUARD_POINTER( destination, memop_chunk,
-                                            (unsigned char*)destination_base, datatype, count );
-                OPAL_DATATYPE_SAFEGUARD_POINTER( source, memop_chunk,
-                                            (unsigned char*)source_base, datatype, count );
                 DO_DEBUG( opal_output( 0, "copy c1. %s( %p, %p, %lu ) => space %lu\n",
                                        STRINGIFY(MEM_OP_NAME), (void*)destination, (void*)source, (unsigned long)memop_chunk, (unsigned long)total_length ); );
                 MEM_OP( destination, source, memop_chunk );
@@ -184,17 +182,12 @@ static inline int32_t _copy_content_same_ddt( const opal_datatype_t* datatype, i
     pos_desc = 0;
     stack_pos = 0;
 
-    if( datatype->opt_desc.desc != NULL ) {
-        description = datatype->opt_desc.desc;
-    } else {
+    description = datatype->opt_desc.desc;
+    if( NULL == description ) {
         description = datatype->desc.desc;
     }
 
-    if( description[0].elem.common.type == OPAL_DATATYPE_LOOP )
-        count_desc = description[0].loop.loops;
-    else
-        count_desc = description[0].elem.count;
-    pElem = &(description[pos_desc]);
+    UPDATE_INTERNAL_COUNTERS( description, 0, pElem, count_desc );
 
     while( 1 ) {
         while( OPAL_LIKELY(pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) ) {

opal/datatype/opal_datatype_get_count.c

@@ -69,14 +69,14 @@ ssize_t opal_datatype_get_element_count( const opal_datatype_t* datatype, size_t
         while( pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
             /* now here we have a basic datatype */
             const opal_datatype_t* basic_type = BASIC_DDT_FROM_ELEM(pElems[pos_desc]);
-            local_size = pElems[pos_desc].elem.count * basic_type->size;
+            local_size = (pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen) * basic_type->size;
             if( local_size >= iSize ) {
                 local_size = iSize / basic_type->size;
                 nbElems += (int32_t)local_size;
                 iSize -= local_size * basic_type->size;
                 return (iSize == 0 ? nbElems : -1);
             }
-            nbElems += pElems[pos_desc].elem.count;
+            nbElems += (pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen);
             iSize -= local_size;
             pos_desc++;  /* advance to the next data */
         }
@@ -131,7 +131,7 @@ int32_t opal_datatype_set_element_count( const opal_datatype_t* datatype, size_t
         while( pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
             /* now here we have a basic datatype */
             const opal_datatype_t* basic_type = BASIC_DDT_FROM_ELEM(pElems[pos_desc]);
-            local_length = pElems[pos_desc].elem.count;
+            local_length = (pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen);
             if( local_length >= count ) {
                 *length += count * basic_type->size;
                 return 0;
@@ -188,8 +188,8 @@ int opal_datatype_compute_ptypes( opal_datatype_t* datatype )
         }
         while( pElems[pos_desc].elem.common.flags & OPAL_DATATYPE_FLAG_DATA ) {
             /* now here we have a basic datatype */
-            datatype->ptypes[pElems[pos_desc].elem.common.type] += pElems[pos_desc].elem.count;
-            nbElems += pElems[pos_desc].elem.count;
+            datatype->ptypes[pElems[pos_desc].elem.common.type] += pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen;
+            nbElems += pElems[pos_desc].elem.count * pElems[pos_desc].elem.blocklen;
 
             DUMP( "  compute_ptypes-add: type %d count %"PRIsize_t" (total type %"PRIsize_t" total %lld)\n",
                   pElems[pos_desc].elem.common.type, datatype->ptypes[pElems[pos_desc].elem.common.type],