Skip to content

Commit 88b3b24

Browse files
edgargabrieledgargabriel
authored
Merge pull request #6697 from edgargabriel/topic/external32-v2
Topic/external32 v2
2 parents d65fae1 + 27b2ec7 commit 88b3b24

22 files changed

+437
-206
lines changed

‎ompi/mca/common/ompio/Makefile.am

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ headers = \
2525
common_ompio_aggregators.h \
2626
common_ompio_print_queue.h \
2727
common_ompio_request.h \
28+
common_ompio_buffer.h \
2829
common_ompio.h
2930

3031
sources = \
@@ -34,6 +35,7 @@ sources = \
3435
common_ompio_file_open.c \
3536
common_ompio_file_view.c \
3637
common_ompio_file_read.c \
38+
common_ompio_buffer.c \
3739
common_ompio_file_write.c
3840

3941

@@ -74,10 +76,6 @@ else
7476
ompidir = $(includedir)
7577
endif
7678

77-
if OPAL_cuda_support
78-
headers += common_ompio_cuda.h
79-
sources += common_ompio_cuda.c
80-
endif
8179

8280
# These two rules will sym link the "noinst" libtool library filename
8381
# to the installable libtool library filename in the case where we are

‎ompi/mca/common/ompio/common_ompio.h

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* University of Stuttgart. All rights reserved.
1111
* Copyright (c) 2004-2005 The Regents of the University of California.
1212
* All rights reserved.
13-
* Copyright (c) 2008-2016 University of Houston. All rights reserved.
13+
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
1414
* Copyright (c) 2018 Research Organization for Information Science
1515
* and Technology (RIST). All rights reserved.
1616
* Copyright (c) 2018 DataDirect Networks. All rights reserved.
@@ -66,7 +66,7 @@
6666
#define OMPIO_LOCK_ENTIRE_FILE 0x00000080
6767
#define OMPIO_LOCK_NEVER 0x00000100
6868
#define OMPIO_LOCK_NOT_THIS_OP 0x00000200
69-
69+
#define OMPIO_DATAREP_NATIVE 0x00000400
7070

7171
#define OMPIO_ROOT 0
7272

@@ -157,7 +157,8 @@ struct ompio_file_t {
157157
ompi_communicator_t *f_comm;
158158
const char *f_filename;
159159
char *f_datarep;
160-
opal_convertor_t *f_convertor;
160+
opal_convertor_t *f_mem_convertor;
161+
opal_convertor_t *f_file_convertor;
161162
opal_info_t *f_info;
162163
int32_t f_flags;
163164
void *f_fs_ptr;
@@ -253,10 +254,16 @@ OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at (ompio_file_t *fh, OMPI_MPI_O
253254
const void *buf, int count, struct ompi_datatype_t *datatype,
254255
ompi_request_t **request);
255256

257+
OMPI_DECLSPEC int mca_common_ompio_file_write_all (ompio_file_t *fh, const void *buf,
258+
int count, struct ompi_datatype_t *datatype,
259+
ompi_status_public_t *status);
260+
256261
OMPI_DECLSPEC int mca_common_ompio_file_write_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
257262
int count, struct ompi_datatype_t *datatype,
258263
ompi_status_public_t *status);
259264

265+
OMPI_DECLSPEC int mca_common_ompio_file_iwrite_all (ompio_file_t *fp, const void *buf,
266+
int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
260267

261268
OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
262269
int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
@@ -282,10 +289,16 @@ OMPI_DECLSPEC int mca_common_ompio_file_iread_at (ompio_file_t *fh, OMPI_MPI_OFF
282289
void *buf, int count, struct ompi_datatype_t *datatype,
283290
ompi_request_t **request);
284291

292+
OMPI_DECLSPEC int mca_common_ompio_file_read_all (ompio_file_t *fh, void *buf, int count, struct ompi_datatype_t *datatype,
293+
ompi_status_public_t * status);
294+
285295
OMPI_DECLSPEC int mca_common_ompio_file_read_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
286296
void *buf, int count, struct ompi_datatype_t *datatype,
287297
ompi_status_public_t * status);
288298

299+
OMPI_DECLSPEC int mca_common_ompio_file_iread_all (ompio_file_t *fp, void *buf, int count, struct ompi_datatype_t *datatype,
300+
ompi_request_t **request);
301+
289302
OMPI_DECLSPEC int mca_common_ompio_file_iread_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset,
290303
void *buf, int count, struct ompi_datatype_t *datatype,
291304
ompi_request_t **request);
@@ -318,6 +331,7 @@ OMPI_DECLSPEC int mca_common_ompio_decode_datatype (struct ompio_file_t *fh,
318331
int count,
319332
const void *buf,
320333
size_t *max_data,
334+
opal_convertor_t *convertor,
321335
struct iovec **iov,
322336
uint32_t *iov_count);
323337

‎ompi/mca/common/ompio/common_ompio_cuda.c renamed to ‎ompi/mca/common/ompio/common_ompio_buffer.c

Lines changed: 34 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* University of Stuttgart. All rights reserved.
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
12-
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
12+
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
1313
* $COPYRIGHT$
1414
*
1515
* Additional copyrights may follow
@@ -27,18 +27,19 @@
2727
#include "opal/mca/allocator/allocator.h"
2828
#include "opal/mca/allocator/base/base.h"
2929
#include "common_ompio.h"
30-
#include "common_ompio_cuda.h"
30+
#include "common_ompio_buffer.h"
3131

3232

33-
static opal_mutex_t mca_common_ompio_cuda_mutex; /* lock for thread safety */
33+
static opal_mutex_t mca_common_ompio_buffer_mutex; /* lock for thread safety */
3434
static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL;
3535
static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL;
3636

37-
static opal_atomic_int32_t mca_common_ompio_cuda_init = 0;
37+
static opal_atomic_int32_t mca_common_ompio_buffer_init = 0;
3838
static int32_t mca_common_ompio_pagesize=4096;
39-
static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size );
40-
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf );
39+
static void* mca_common_ompio_buffer_alloc_seg ( void *ctx, size_t *size );
40+
static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf );
4141

42+
#if OPAL_CUDA_SUPPORT
4243
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is_gpu,
4344
int *is_managed)
4445
{
@@ -57,8 +58,9 @@ void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is
5758

5859
return;
5960
}
61+
#endif
6062

61-
static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
63+
static void* mca_common_ompio_buffer_alloc_seg ( void*ctx, size_t *size )
6264
{
6365
char *buf=NULL;
6466
size_t realsize, numpages;
@@ -67,64 +69,67 @@ static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
6769
realsize = numpages * mca_common_ompio_pagesize;
6870

6971
buf = malloc ( realsize);
72+
#if OPAL_CUDA_SUPPORT
7073
if ( NULL != buf ) {
7174
mca_common_cuda_register ( ( char *)buf, realsize, NULL );
7275
}
76+
#endif
7377
*size = realsize;
7478
return buf;
7579
}
7680

77-
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf )
81+
static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf )
7882
{
7983
if ( NULL != buf ) {
84+
#if OPAL_CUDA_SUPPORT
8085
mca_common_cuda_unregister ( (char *) buf, NULL );
86+
#endif
8187
free ( buf );
8288
}
8389
return;
8490
}
8591

86-
int mca_common_ompio_cuda_alloc_init ( void )
92+
int mca_common_ompio_buffer_alloc_init ( void )
8793
{
8894
bool thread_safe=true;
8995

90-
if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_cuda_init, 1) > 1)
96+
if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_buffer_init, 1) > 1)
9197
return OMPI_SUCCESS;
9298

9399
/* initialize static objects */
94-
OBJ_CONSTRUCT(&mca_common_ompio_cuda_mutex, opal_mutex_t);
100+
OBJ_CONSTRUCT(&mca_common_ompio_buffer_mutex, opal_mutex_t);
95101

96-
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex );
102+
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex );
97103
/* lookup name of the allocator to use */
98104
if(NULL == (mca_common_ompio_allocator_component = mca_allocator_component_lookup("basic"))) {
99-
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
105+
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
100106
return OMPI_ERR_BUFFER;
101107
}
102108

103109
/* create an instance of the allocator */
104110
mca_common_ompio_allocator = mca_common_ompio_allocator_component->allocator_init(thread_safe,
105-
mca_common_ompio_cuda_alloc_seg,
106-
mca_common_ompio_cuda_free_seg,
111+
mca_common_ompio_buffer_alloc_seg,
112+
mca_common_ompio_buffer_free_seg,
107113
NULL);
108114
if(NULL == mca_common_ompio_allocator) {
109-
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
115+
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
110116
return OMPI_ERR_BUFFER;
111117
}
112118

113-
// mca_common_ompio_pagesize = sysconf(_SC_PAGESIZE);
114119
mca_common_ompio_pagesize = opal_getpagesize();
115120

116-
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
121+
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
117122
return OMPI_SUCCESS;
118123
}
119124

120-
int mca_common_ompio_cuda_alloc_fini ( void )
125+
int mca_common_ompio_buffer_alloc_fini ( void )
121126
{
122127
if ( NULL != mca_common_ompio_allocator ) {
123-
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
128+
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
124129
mca_common_ompio_allocator->alc_finalize(mca_common_ompio_allocator);
125130
mca_common_ompio_allocator=NULL;
126-
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
127-
OBJ_DESTRUCT (&mca_common_ompio_cuda_mutex);
131+
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
132+
OBJ_DESTRUCT (&mca_common_ompio_buffer_mutex);
128133
}
129134

130135
return OMPI_SUCCESS;
@@ -134,31 +139,31 @@ void *mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize )
134139
{
135140
char *tmp=NULL;
136141

137-
if ( !mca_common_ompio_cuda_init ){
138-
mca_common_ompio_cuda_alloc_init ();
142+
if ( !mca_common_ompio_buffer_init ){
143+
mca_common_ompio_buffer_alloc_init ();
139144
}
140145

141-
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
146+
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
142147
tmp = mca_common_ompio_allocator->alc_alloc (mca_common_ompio_allocator,
143148
bufsize, 0 );
144-
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
149+
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
145150
return tmp;
146151
}
147152

148153
void mca_common_ompio_release_buf ( ompio_file_t *fh, void *buf )
149154
{
150155

151-
if ( !mca_common_ompio_cuda_init ){
156+
if ( !mca_common_ompio_buffer_init ){
152157
/* Should not happen. You can not release a buf without
153158
** having it allocated first.
154159
*/
155160
opal_output (1, "error in mca_common_ompio_release_buf: allocator not initialized\n");
156161
}
157162

158-
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
163+
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
159164
mca_common_ompio_allocator->alc_free (mca_common_ompio_allocator,
160165
buf);
161-
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
166+
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
162167

163168
return;
164169
}

‎ompi/mca/common/ompio/common_ompio_cuda.h renamed to ‎ompi/mca/common/ompio/common_ompio_buffer.h

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* University of Stuttgart. All rights reserved.
1111
* Copyright (c) 2004-2005 The Regents of the University of California.
1212
* All rights reserved.
13-
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
13+
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
1414
* $COPYRIGHT$
1515
*
1616
* Additional copyrights may follow
@@ -22,9 +22,9 @@
2222
#define MCA_COMMON_OMPIO_CUDA_H
2323

2424

25-
#define OMPIO_CUDA_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
26-
opal_convertor_clone ( _fh->f_convertor, _convertor, 0); \
27-
opal_convertor_prepare_for_send ( _convertor, &(_datatype->super), _count, _buf );\
25+
#define OMPIO_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
26+
OBJ_CONSTRUCT( _convertor, opal_convertor_t); \
27+
opal_convertor_copy_and_prepare_for_send ( _fh->f_file_convertor, &(_datatype->super), _count, _buf, CONVERTOR_SEND_CONVERSION, _convertor ); \
2828
opal_convertor_get_packed_size( _convertor, &_max_data ); \
2929
_tbuf = mca_common_ompio_alloc_buf (_fh, _max_data); \
3030
if ( NULL == _tbuf ) { \
@@ -40,11 +40,30 @@
4040
_decoded_iov->iov_len = _max_data; \
4141
_iov_count=1;}
4242

43+
#define OMPIO_PREPARE_READ_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
44+
OBJ_CONSTRUCT( _convertor, opal_convertor_t); \
45+
opal_convertor_copy_and_prepare_for_recv ( _fh->f_file_convertor, &(_datatype->super), _count, _buf, 0, _convertor ); \
46+
opal_convertor_get_packed_size( _convertor, &_max_data ); \
47+
_tbuf = mca_common_ompio_alloc_buf (_fh, _max_data); \
48+
if ( NULL == _tbuf ) { \
49+
opal_output(1, "common_ompio: error allocating memory\n"); \
50+
return OMPI_ERR_OUT_OF_RESOURCE; \
51+
} \
52+
_decoded_iov = (struct iovec *) malloc ( sizeof ( struct iovec )); \
53+
if ( NULL == _decoded_iov ) { \
54+
opal_output(1, "common_ompio: could not allocate memory.\n"); \
55+
return OMPI_ERR_OUT_OF_RESOURCE; \
56+
} \
57+
_decoded_iov->iov_base = _tbuf; \
58+
_decoded_iov->iov_len = _max_data; \
59+
_iov_count=1;}
4360

61+
#if OPAL_CUDA_SUPPORT
4462
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf,
4563
int *is_gpu, int *is_managed);
46-
int mca_common_ompio_cuda_alloc_init ( void );
47-
int mca_common_ompio_cuda_alloc_fini ( void );
64+
#endif
65+
int mca_common_ompio_buffer_alloc_init ( void );
66+
int mca_common_ompio_buffer_alloc_fini ( void );
4867

4968

5069
void* mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize);

0 commit comments

Comments
 (0)