Skip to content
This repository was archived by the owner on Sep 30, 2022. It is now read-only.

v2.x: oshmem: get rid of oshmem_proc_t and use ompi_proc_t instead #1358

Merged
merged 1 commit into from
Sep 8, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion ompi/proc/proc.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,16 @@ BEGIN_C_DECLS
* a tag increases the memory consumed by Open MPI, so should only be done
* if unavoidable.
*/

#define OMPI_PROC_PADDING_SIZE 16

struct ompi_proc_t {
opal_proc_t super;

/* endpoint data */
void *proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MAX];

char padding[16]; /* for future extensions (OSHMEM uses this area also)*/
char padding[OMPI_PROC_PADDING_SIZE]; /* for future extensions (OSHMEM uses this area also)*/
};
typedef struct ompi_proc_t ompi_proc_t;
OBJ_CLASS_DECLARATION(ompi_proc_t);
Expand Down
4 changes: 3 additions & 1 deletion oshmem/include/oshmem/types.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/*
* Copyright (c) 2013 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand All @@ -16,7 +18,7 @@
* dependencies.
*/

struct oshmem_proc_t;
struct ompi_proc_t;
struct oshmem_group_t;
struct oshmem_op_t;

Expand Down
4 changes: 3 additions & 1 deletion oshmem/mca/atomic/mxm/atomic_mxm_cswap.c
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/*
* Copyright (c) 2013 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -72,7 +74,7 @@ int mca_atomic_mxm_cswap(void *target,
return OSHMEM_ERR_BAD_PARAM;
}

ptl_id = oshmem_proc_group_all(pe)->transport_ids[0];
ptl_id = OSHMEM_PROC_DATA(oshmem_proc_group_all(pe))->transport_ids[0];
if (MXM_PTL_SHM == ptl_id) {
ptl_id = MXM_PTL_RDMA;
}
Expand Down
4 changes: 3 additions & 1 deletion oshmem/mca/atomic/mxm/atomic_mxm_fadd.c
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/*
* Copyright (c) 2013 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -74,7 +76,7 @@ int mca_atomic_mxm_fadd(void *target,
return OSHMEM_ERR_BAD_PARAM;
}

ptl_id = oshmem_proc_group_all(pe)->transport_ids[0];
ptl_id = OSHMEM_PROC_DATA(oshmem_proc_group_all(pe))->transport_ids[0];
if (MXM_PTL_SHM == ptl_id) {
ptl_id = MXM_PTL_RDMA;
}
Expand Down
8 changes: 4 additions & 4 deletions oshmem/mca/memheap/base/memheap_base_mkey.c
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
/*
* Copyright (c) 2013-2015 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
Expand Down Expand Up @@ -93,7 +93,7 @@ int mca_memheap_seg_cmp(const void *k, const void *v)
*/
static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg, int all_trs)
{
oshmem_proc_t *proc;
ompi_proc_t *proc;
int i, n, tr_id;
sshmem_mkey_t *mkey;

Expand All @@ -111,7 +111,7 @@ static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg, int all_trs)
MEMHEAP_VERBOSE(5, "found %d transports to %d", n, pe);
for (i = 0; i < n; i++) {
if (!all_trs) {
tr_id = proc->transport_ids[i];
tr_id = OSHMEM_PROC_DATA(proc)->transport_ids[i];
}
else {
tr_id = i;
Expand Down Expand Up @@ -171,7 +171,7 @@ static void unpack_remote_mkeys(opal_buffer_t *msg, int remote_pe)
int32_t n;
int32_t tr_id;
int i;
oshmem_proc_t *proc;
ompi_proc_t *proc;

proc = oshmem_proc_group_find(oshmem_group_all, remote_pe);
cnt = 1;
Expand Down
6 changes: 3 additions & 3 deletions oshmem/mca/scoll/fca/scoll_fca_module.c
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/**
* Copyright (c) 2013 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2014 Research Organization for Information Science
* Copyright (c) 2014-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -104,7 +104,7 @@ static int have_remote_peers(struct oshmem_group_t *group,
size_t size,
int *local_peers)
{
struct oshmem_proc_t *proc;
struct ompi_proc_t *proc;
size_t i;
int ret;

Expand All @@ -128,7 +128,7 @@ static int have_remote_peers(struct oshmem_group_t *group,
static int _get_local_ranks(mca_scoll_fca_module_t *fca_module)
{
struct oshmem_group_t *comm = fca_module->comm;
oshmem_proc_t* proc;
ompi_proc_t* proc;
int i, rank;

/* Count the local ranks */
Expand Down
4 changes: 3 additions & 1 deletion oshmem/mca/spml/base/base.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
* Copyright (c) 2013 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -57,7 +59,7 @@ OSHMEM_DECLSPEC int mca_spml_base_spml_selected(const char *name);
* Verify that all new procs are using the currently selected component
*/
OSHMEM_DECLSPEC int mca_spml_base_spml_check_selected(const char *my_spml,
oshmem_proc_t **procs,
ompi_proc_t **procs,
size_t nprocs);

OSHMEM_DECLSPEC int mca_spml_base_wait(void* addr,
Expand Down
4 changes: 2 additions & 2 deletions oshmem/mca/spml/base/spml_base_request.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -72,7 +72,7 @@ struct mca_spml_base_request_t {
void *req_addr; /**< pointer to application buffer */
size_t req_count; /**< count of user datatype elements *//* TODO: Need to remove since we are going to remove datatype*/
int32_t req_peer; /**< peer process - rank of process executing the parallel program */
oshmem_proc_t* req_proc; /**< peer process */
ompi_proc_t* req_proc; /**< peer process */
uint64_t req_sequence; /**< sequence number for shmem one sided ordering */
};
typedef struct mca_spml_base_request_t mca_spml_base_request_t;
Expand Down
36 changes: 18 additions & 18 deletions oshmem/mca/spml/ikrit/spml_ikrit.c
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
/*
* Copyright (c) 2013-2015 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2014 Research Organization for Information Science
* Copyright (c) 2014-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
Expand Down Expand Up @@ -331,31 +331,31 @@ int mca_spml_ikrit_enable(bool enable)

static int create_ptl_idx(int dst_pe)
{
oshmem_proc_t *proc;
ompi_proc_t *proc;

proc = oshmem_proc_group_find(oshmem_group_all, dst_pe);

proc->transport_ids = (char *) malloc(MXM_PTL_LAST * sizeof(char));
if (!proc->transport_ids)
OSHMEM_PROC_DATA(proc)->transport_ids = (char *) malloc(MXM_PTL_LAST * sizeof(char));
if (NULL == OSHMEM_PROC_DATA(proc)->transport_ids)
return OSHMEM_ERROR;

proc->num_transports = 1;
OSHMEM_PROC_DATA(proc)->num_transports = 1;
#if MXM_API < MXM_VERSION(2,0)
if (oshmem_my_proc_id() == dst_pe)
proc->transport_ids[0] = MXM_PTL_SELF;
OSHMEM_PROC_DATA(proc)->transport_ids[0] = MXM_PTL_SELF;
else
#endif
proc->transport_ids[0] = MXM_PTL_RDMA;
OSHMEM_PROC_DATA(proc)->transport_ids[0] = MXM_PTL_RDMA;
return OSHMEM_SUCCESS;
}

static void destroy_ptl_idx(int dst_pe)
{
oshmem_proc_t *proc;
ompi_proc_t *proc;

proc = oshmem_proc_group_find(oshmem_group_all, dst_pe);
if (proc->transport_ids)
free(proc->transport_ids);
if (NULL != OSHMEM_PROC_DATA(proc)->transport_ids)
free(OSHMEM_PROC_DATA(proc)->transport_ids);
}

static void mxm_peer_construct(mxm_peer_t *p)
Expand All @@ -375,7 +375,7 @@ OBJ_CLASS_INSTANCE( mxm_peer_t,
mxm_peer_construct,
mxm_peer_destruct);

int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs)
int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs)
{
size_t i, n;
int my_rank = oshmem_my_proc_id();
Expand Down Expand Up @@ -406,7 +406,7 @@ int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs)
return OSHMEM_SUCCESS;
}

int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs)
{
spml_ikrit_mxm_ep_conn_info_t *ep_info = NULL;
spml_ikrit_mxm_ep_conn_info_t *ep_hw_rdma_info = NULL;
Expand All @@ -420,7 +420,7 @@ int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
mxm_error_t err;
size_t i, n;
int rc = OSHMEM_ERROR;
oshmem_proc_t *proc_self;
ompi_proc_t *proc_self;
int my_rank = oshmem_my_proc_id();

OBJ_CONSTRUCT(&mca_spml_ikrit.active_peers, opal_list_t);
Expand Down Expand Up @@ -588,9 +588,9 @@ int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
continue;

/* use zcopy for put/get via sysv shared memory */
procs[i]->transport_ids[0] = MXM_PTL_SHM;
procs[i]->transport_ids[1] = MXM_PTL_RDMA;
procs[i]->num_transports = 2;
OSHMEM_PROC_DATA(procs[i])->transport_ids[0] = MXM_PTL_SHM;
OSHMEM_PROC_DATA(procs[i])->transport_ids[1] = MXM_PTL_RDMA;
OSHMEM_PROC_DATA(procs[i])->num_transports = 2;
}

SPML_VERBOSE(50, "*** ADDED PROCS ***");
Expand Down Expand Up @@ -742,7 +742,7 @@ int mca_spml_ikrit_deregister(sshmem_mkey_t *mkeys)

static inline int get_ptl_id(int dst)
{
oshmem_proc_t *proc;
ompi_proc_t *proc;

/* get endpoint and btl */
proc = oshmem_proc_group_all(dst);
Expand All @@ -751,7 +751,7 @@ static inline int get_ptl_id(int dst)
oshmem_shmem_abort(-1);
return -1;
}
return proc->transport_ids[0];
return OSHMEM_PROC_DATA(proc)->transport_ids[0];
}

int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, sshmem_mkey_t *mkeys)
Expand Down
6 changes: 4 additions & 2 deletions oshmem/mca/spml/ikrit/spml_ikrit.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
* All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -161,8 +163,8 @@ extern int mca_spml_ikrit_oob_get_mkeys(int pe,
uint32_t seg,
sshmem_mkey_t *mkeys);

extern int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs);
extern int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs);
extern int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs);
extern int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs);
extern int mca_spml_ikrit_fence(void);
extern int spml_ikrit_progress(void);

Expand Down
6 changes: 4 additions & 2 deletions oshmem/mca/spml/spml.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* Copyright (c) 2014 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -166,9 +168,9 @@ typedef int (*mca_spml_base_module_oob_get_mkeys_fn_t)(int pe,
* @return OSHMEM_SUCCESS or failure status.
*
*/
typedef int (*mca_spml_base_module_add_procs_fn_t)(oshmem_proc_t** procs,
typedef int (*mca_spml_base_module_add_procs_fn_t)(ompi_proc_t** procs,
size_t nprocs);
typedef int (*mca_spml_base_module_del_procs_fn_t)(oshmem_proc_t** procs,
typedef int (*mca_spml_base_module_del_procs_fn_t)(ompi_proc_t** procs,
size_t nprocs);

/**
Expand Down
6 changes: 3 additions & 3 deletions oshmem/mca/spml/ucx/spml_ucx.c
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2013 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2014 Research Organization for Information Science
* Copyright (c) 2014-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -79,7 +79,7 @@ int mca_spml_ucx_enable(bool enable)
return OSHMEM_SUCCESS;
}

int mca_spml_ucx_del_procs(oshmem_proc_t** procs, size_t nprocs)
int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs)
{
size_t i, n;
int my_rank = oshmem_my_proc_id();
Expand Down Expand Up @@ -174,7 +174,7 @@ static void dump_address(int pe, char *addr, size_t len)
#endif
}

int mca_spml_ucx_add_procs(oshmem_proc_t** procs, size_t nprocs)
int mca_spml_ucx_add_procs(ompi_proc_t** procs, size_t nprocs)
{
size_t i, n;
int rc = OSHMEM_ERROR;
Expand Down
6 changes: 4 additions & 2 deletions oshmem/mca/spml/ucx/spml_ucx.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/*
* Copyright (c) 2013 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -92,8 +94,8 @@ extern int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys);
extern void mca_spml_ucx_rmkey_unpack(sshmem_mkey_t *mkey, int pe);
extern void mca_spml_ucx_rmkey_free(sshmem_mkey_t *mkey);

extern int mca_spml_ucx_add_procs(oshmem_proc_t** procs, size_t nprocs);
extern int mca_spml_ucx_del_procs(oshmem_proc_t** procs, size_t nprocs);
extern int mca_spml_ucx_add_procs(ompi_proc_t** procs, size_t nprocs);
extern int mca_spml_ucx_del_procs(ompi_proc_t** procs, size_t nprocs);
extern int mca_spml_ucx_fence(void);
extern int mca_spml_ucx_quiet(void);
extern int spml_ucx_progress(void);
Expand Down
Loading