Skip to content

Commit 87a79fa

Browse files
jsquyresjsquyres
authored
Merge pull request #2215 from hjelmn/v2.x_comm
v2.x communicator code updates
2 parents 768cfcf + a7b8d16 commit 87a79fa

File tree

10 files changed

+926
-1399
lines changed

10 files changed

+926
-1399
lines changed

‎ompi/communicator/comm.c

Lines changed: 227 additions & 288 deletions
Large diffs are not rendered by default.

‎ompi/communicator/comm_cid.c

100644100755
Lines changed: 628 additions & 1010 deletions
Large diffs are not rendered by default.

‎ompi/communicator/comm_init.c

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -206,10 +206,6 @@ int ompi_comm_init(void)
206206
OBJ_RETAIN(&ompi_mpi_group_null.group);
207207
OBJ_RETAIN(&ompi_mpi_errors_are_fatal.eh);
208208

209-
/* initialize the comm_reg stuff for multi-threaded comm_cid
210-
allocation */
211-
ompi_comm_reg_init();
212-
213209
/* initialize communicator requests (for ompi_comm_idup) */
214210
ompi_comm_request_init ();
215211

@@ -328,27 +324,15 @@ int ompi_comm_finalize(void)
328324
}
329325
}
330326

331-
332327
OBJ_DESTRUCT (&ompi_mpi_communicators);
333328
OBJ_DESTRUCT (&ompi_comm_f_to_c_table);
334329

335-
/* finalize the comm_reg stuff */
336-
ompi_comm_reg_finalize();
337-
338330
/* finalize communicator requests */
339331
ompi_comm_request_fini ();
340332

341333
return OMPI_SUCCESS;
342334
}
343335

344-
/*
345-
* For linking only. To be checked.
346-
*/
347-
int ompi_comm_link_function(void)
348-
{
349-
return OMPI_SUCCESS;
350-
}
351-
352336
/********************************************************************************/
353337
/********************************************************************************/
354338
/********************************************************************************/

‎ompi/communicator/comm_request.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ static void ompi_comm_request_destruct (ompi_comm_request_t *request)
234234
{
235235
OBJ_DESTRUCT(&request->schedule);
236236
}
237+
237238
OBJ_CLASS_INSTANCE(ompi_comm_request_t, ompi_request_t,
238239
ompi_comm_request_construct,
239240
ompi_comm_request_destruct);
@@ -257,10 +258,10 @@ ompi_comm_request_t *ompi_comm_request_get (void)
257258
void ompi_comm_request_return (ompi_comm_request_t *request)
258259
{
259260
if (request->context) {
260-
free (request->context);
261-
request->context = NULL;
261+
OBJ_RELEASE (request->context);
262262
}
263263

264+
OMPI_REQUEST_FINI(&request->super);
264265
opal_free_list_return (&ompi_comm_requests, (opal_free_list_item_t *) request);
265266
}
266267

‎ompi/communicator/comm_request.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
typedef struct ompi_comm_request_t {
2323
ompi_request_t super;
2424

25-
void *context;
25+
opal_object_t *context;
2626
opal_list_t schedule;
2727
} ompi_comm_request_t;
2828
OBJ_CLASS_DECLARATION(ompi_comm_request_t);

‎ompi/communicator/communicator.h

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,6 @@ static inline bool ompi_comm_peer_invalid(ompi_communicator_t* comm, int peer_id
351351
* Initialise MPI_COMM_WORLD and MPI_COMM_SELF
352352
*/
353353
int ompi_comm_init(void);
354-
OMPI_DECLSPEC int ompi_comm_link_function(void);
355354

356355
/**
357356
* extract the local group from a communicator
@@ -493,24 +492,27 @@ ompi_communicator_t* ompi_comm_allocate (int local_group_size,
493492
* @param mode: combination of input
494493
* OMPI_COMM_CID_INTRA: intra-comm
495494
* OMPI_COMM_CID_INTER: inter-comm
495+
* OMPI_COMM_CID_GROUP: only decide CID within the ompi_group_t
496+
* associated with the communicator. arg0
497+
* must point to an int which will be used
498+
* as the pml tag for communication.
496499
* OMPI_COMM_CID_INTRA_BRIDGE: 2 intracomms connected by
497-
* a bridge comm. local_leader
498-
* and remote leader are in this
499-
* case an int (rank in bridge-comm).
500+
* a bridge comm. arg0 and arg1 must point
501+
* to integers representing the local and
502+
* remote leader ranks. the remote leader rank
503+
* is a rank in the bridgecomm.
500504
* OMPI_COMM_CID_INTRA_PMIX: 2 intracomms, leaders talk
501-
* through PMIx. lleader and rleader
502-
* are the required contact information.
505+
* through PMIx. arg0 must point to an integer
506+
* representing the local leader rank. arg1
507+
* must point to a string representing the
508+
* port of the remote leader.
503509
* @param send_first: to avoid a potential deadlock for
504510
* the OOB version.
505511
* This routine has to be thread safe in the final version.
506512
*/
507-
OMPI_DECLSPEC int ompi_comm_nextcid ( ompi_communicator_t* newcomm,
508-
ompi_communicator_t* oldcomm,
509-
ompi_communicator_t* bridgecomm,
510-
void* local_leader,
511-
void* remote_leader,
512-
int mode,
513-
int send_first);
513+
OMPI_DECLSPEC int ompi_comm_nextcid (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
514+
ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
515+
bool send_first, int mode);
514516

515517
/**
516518
* allocate new communicator ID (non-blocking)
@@ -522,10 +524,9 @@ OMPI_DECLSPEC int ompi_comm_nextcid ( ompi_communicator_t* newcomm,
522524
* OMPI_COMM_CID_INTER: inter-comm
523525
* This routine has to be thread safe in the final version.
524526
*/
525-
OMPI_DECLSPEC int ompi_comm_nextcid_nb (ompi_communicator_t* newcomm,
526-
ompi_communicator_t* comm,
527-
ompi_communicator_t* bridgecomm,
528-
int mode, ompi_request_t **req);
527+
OMPI_DECLSPEC int ompi_comm_nextcid_nb (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
528+
ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
529+
bool send_first, int mode, ompi_request_t **req);
529530

530531
/**
531532
* shut down the communicator infrastructure.
@@ -618,18 +619,25 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm,
618619
int high );
619620

620621

621-
OMPI_DECLSPEC int ompi_comm_activate ( ompi_communicator_t** newcomm,
622-
ompi_communicator_t* comm,
623-
ompi_communicator_t* bridgecomm,
624-
void* local_leader,
625-
void* remote_leader,
626-
int mode,
627-
int send_first );
622+
OMPI_DECLSPEC int ompi_comm_activate (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
623+
ompi_communicator_t *bridgecomm, const void *arg0,
624+
const void *arg1, bool send_first, int mode);
628625

629-
OMPI_DECLSPEC int ompi_comm_activate_nb (ompi_communicator_t **newcomm,
630-
ompi_communicator_t *comm,
631-
ompi_communicator_t *bridgecomm,
632-
int mode, ompi_request_t **req);
626+
/**
627+
* Non-blocking variant of comm_activate.
628+
*
629+
* @param[inout] newcomm New communicator
630+
* @param[in] comm Parent communicator
631+
* @param[in] bridgecomm Bridge communicator (used for PMIX and bridge modes)
632+
* @param[in] arg0 Mode argument 0
633+
* @param[in] arg1 Mode argument 1
634+
* @param[in] send_first Send first from this process (PMIX mode only)
635+
* @param[in] mode Collective mode
636+
* @param[out] req New request object to track this operation
637+
*/
638+
OMPI_DECLSPEC int ompi_comm_activate_nb (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
639+
ompi_communicator_t *bridgecomm, const void *arg0,
640+
const void *arg1, bool send_first, int mode, ompi_request_t **req);
633641

634642
/**
635643
* a simple function to dump the structure
@@ -639,14 +647,6 @@ int ompi_comm_dump ( ompi_communicator_t *comm );
639647
/* setting name */
640648
int ompi_comm_set_name (ompi_communicator_t *comm, const char *name );
641649

642-
/*
643-
* these are the init and finalize functions for the comm_reg
644-
* stuff. These routines are necessary for handling multi-threading
645-
* scenarious in the communicator_cid allocation
646-
*/
647-
void ompi_comm_reg_init(void);
648-
void ompi_comm_reg_finalize(void);
649-
650650
/* global variable to save the number od dynamic communicators */
651651
extern int ompi_comm_num_dyncomm;
652652

‎ompi/dpm/dpm.c

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -469,25 +469,25 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
469469
new_group_pointer = MPI_GROUP_NULL;
470470

471471
/* allocate comm_cid */
472-
rc = ompi_comm_nextcid ( newcomp, /* new communicator */
473-
comm, /* old communicator */
474-
NULL, /* bridge comm */
475-
&root, /* local leader */
476-
(void*)port_string, /* rendezvous point */
477-
OMPI_COMM_CID_INTRA_PMIX, /* mode */
478-
send_first ); /* send or recv first */
472+
rc = ompi_comm_nextcid ( newcomp, /* new communicator */
473+
comm, /* old communicator */
474+
NULL, /* bridge comm */
475+
&root, /* local leader */
476+
(void*)port_string, /* rendezvous point */
477+
send_first, /* send or recv first */
478+
OMPI_COMM_CID_INTRA_PMIX); /* mode */
479479
if (OMPI_SUCCESS != rc) {
480480
goto exit;
481481
}
482482

483483
/* activate comm and init coll-component */
484-
rc = ompi_comm_activate ( &newcomp, /* new communicator */
485-
comm, /* old communicator */
486-
NULL, /* bridge comm */
487-
&root, /* local leader */
488-
(void*)port_string, /* rendezvous point */
489-
OMPI_COMM_CID_INTRA_PMIX, /* mode */
490-
send_first ); /* send or recv first */
484+
rc = ompi_comm_activate ( &newcomp, /* new communicator */
485+
comm, /* old communicator */
486+
NULL, /* bridge comm */
487+
&root, /* local leader */
488+
(void*)port_string, /* rendezvous point */
489+
send_first, /* send or recv first */
490+
OMPI_COMM_CID_INTRA_PMIX); /* mode */
491491
if (OMPI_SUCCESS != rc) {
492492
goto exit;
493493
}
@@ -500,7 +500,7 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
500500
exit:
501501
if (OMPI_SUCCESS != rc) {
502502
if (MPI_COMM_NULL != newcomp && NULL != newcomp) {
503-
OBJ_RETAIN(newcomp);
503+
OBJ_RELEASE(newcomp);
504504
newcomp = MPI_COMM_NULL;
505505
}
506506
}

‎ompi/mpi/c/intercomm_create.c

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -14,6 +15,8 @@
1415
* Copyright (c) 2012-2013 Inria. All rights reserved.
1516
* Copyright (c) 2014-2015 Research Organization for Information Science
1617
* and Technology (RIST). All rights reserved.
18+
* Copyright (c) 2016 Los Alamos National Security, LLC. All rights
19+
* reserved.
1720
* $COPYRIGHT$
1821
*
1922
* Additional copyrights may follow
@@ -196,26 +199,15 @@ int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader,
196199
new_group_pointer = MPI_GROUP_NULL;
197200

198201
/* Determine context id. It is identical to f_2_c_handle */
199-
rc = ompi_comm_nextcid ( newcomp, /* new comm */
200-
local_comm, /* old comm */
201-
bridge_comm, /* bridge comm */
202-
&lleader, /* local leader */
203-
&rleader, /* remote_leader */
204-
OMPI_COMM_CID_INTRA_BRIDGE, /* mode */
205-
-1 ); /* send_first */
206-
202+
rc = ompi_comm_nextcid (newcomp, local_comm, bridge_comm, &lleader,
203+
&rleader, false, OMPI_COMM_CID_INTRA_BRIDGE);
207204
if ( MPI_SUCCESS != rc ) {
208205
goto err_exit;
209206
}
210207

211208
/* activate comm and init coll-module */
212-
rc = ompi_comm_activate ( &newcomp,
213-
local_comm, /* old comm */
214-
bridge_comm, /* bridge comm */
215-
&lleader, /* local leader */
216-
&rleader, /* remote_leader */
217-
OMPI_COMM_CID_INTRA_BRIDGE, /* mode */
218-
-1 ); /* send_first */
209+
rc = ompi_comm_activate (&newcomp, local_comm, bridge_comm, &lleader, &rleader,
210+
false, OMPI_COMM_CID_INTRA_BRIDGE);
219211
if ( MPI_SUCCESS != rc ) {
220212
goto err_exit;
221213
}

‎ompi/mpi/c/intercomm_merge.c

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -14,6 +15,8 @@
1415
* Copyright (c) 2012-2013 Inria. All rights reserved.
1516
* Copyright (c) 2015 Research Organization for Information Science
1617
* and Technology (RIST). All rights reserved.
18+
* Copyright (c) 2016 Los Alamos National Security, LLC. All rights
19+
* reserved.
1720
* $COPYRIGHT$
1821
*
1922
* Additional copyrights may follow
@@ -115,26 +118,16 @@ int MPI_Intercomm_merge(MPI_Comm intercomm, int high,
115118
OBJ_RELEASE(new_group_pointer);
116119
new_group_pointer = MPI_GROUP_NULL;
117120

118-
/* Determine context id. It is identical to f_2_c_handle */
119-
rc = ompi_comm_nextcid ( newcomp, /* new comm */
120-
intercomm, /* old comm */
121-
NULL, /* bridge comm */
122-
NULL, /* local leader */
123-
NULL, /* remote_leader */
124-
OMPI_COMM_CID_INTER, /* mode */
125-
-1 ); /* send_first */
121+
/* Determine context id */
122+
rc = ompi_comm_nextcid (newcomp, intercomm, NULL, NULL, NULL, false,
123+
OMPI_COMM_CID_INTER);
126124
if ( OMPI_SUCCESS != rc ) {
127125
goto exit;
128126
}
129127

130128
/* activate communicator and init coll-module */
131-
rc = ompi_comm_activate( &newcomp, /* new comm */
132-
intercomm, /* old comm */
133-
NULL, /* bridge comm */
134-
NULL, /* local leader */
135-
NULL, /* remote_leader */
136-
OMPI_COMM_CID_INTER, /* mode */
137-
-1 ); /* send_first */
129+
rc = ompi_comm_activate (&newcomp, intercomm, NULL, NULL, NULL, false,
130+
OMPI_COMM_CID_INTER);
138131
if ( OMPI_SUCCESS != rc ) {
139132
goto exit;
140133
}

‎ompi/request/req_wait.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ int ompi_request_default_wait_all( size_t count,
196196
rptr = requests;
197197
for (i = 0; i < count; i++) {
198198
request = *rptr++;
199-
199+
200200
if( request->req_state == OMPI_REQUEST_INACTIVE ) {
201201
completed++;
202202
continue;

0 commit comments

Comments
 (0)