Skip to content

Commit 10601d9

Browse files
alex-mikheevalex-mikheev
committed
OSHMEM: adds support for mkey caching by spml
It improves cpu cache hit ratio. Signed-off-by: Alex Mikheev <[email protected]>
1 parent aa728f1 commit 10601d9

20 files changed

+451
-390
lines changed

‎oshmem/mca/atomic/mxm/atomic_mxm.h

+68-6
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
/* This component does uses SPML:IKRIT */
2121
#include "oshmem/mca/spml/ikrit/spml_ikrit.h"
22+
#include "oshmem/runtime/runtime.h"
2223

2324

2425
BEGIN_C_DECLS
@@ -60,15 +61,76 @@ struct mca_atomic_mxm_module_t {
6061
typedef struct mca_atomic_mxm_module_t mca_atomic_mxm_module_t;
6162
OBJ_CLASS_DECLARATION(mca_atomic_mxm_module_t);
6263

63-
END_C_DECLS
6464

65-
/* move to spml/ikrit */
66-
static inline mxm_mem_key_t *to_mxm_mkey(sshmem_mkey_t *mkey) {
65+
static inline uint8_t mca_atomic_mxm_order(size_t nlong)
66+
{
67+
if (OPAL_LIKELY(8 == nlong)) {
68+
return 3;
69+
}
70+
71+
if (OPAL_LIKELY(4 == nlong)) {
72+
return 2;
73+
}
74+
75+
if (2 == nlong) {
76+
return 1;
77+
}
6778

68-
if (0 == mkey->len) {
69-
return &mxm_empty_mem_key;
79+
if (1 == nlong) {
80+
return 0;
7081
}
71-
return (mxm_mem_key_t *)mkey->u.data;
82+
83+
ATOMIC_ERROR("Type size must be 1/2/4 or 8 bytes.");
84+
oshmem_shmem_abort(-1);
85+
return OSHMEM_ERR_BAD_PARAM;
7286
}
7387

88+
static inline void mca_atomic_mxm_req_init(mxm_send_req_t *sreq, int pe, void *target, size_t nlong)
89+
{
90+
uint8_t nlong_order;
91+
void *remote_addr;
92+
mxm_mem_key_t *mkey;
93+
94+
nlong_order = mca_atomic_mxm_order(nlong);
95+
96+
mkey = mca_spml_ikrit_get_mkey(pe, target, MXM_PTL_RDMA, &remote_addr);
97+
98+
/* mxm request init */
99+
sreq->base.state = MXM_REQ_NEW;
100+
sreq->base.mq = mca_atomic_mxm_spml_self->mxm_mq;
101+
sreq->base.conn = mca_atomic_mxm_spml_self->mxm_peers[pe].mxm_hw_rdma_conn;
102+
sreq->base.completed_cb = NULL;
103+
sreq->base.data_type = MXM_REQ_DATA_BUFFER;
104+
105+
sreq->base.data.buffer.memh = MXM_INVALID_MEM_HANDLE;
106+
sreq->base.data.buffer.length = nlong;
107+
108+
sreq->op.atomic.remote_vaddr = (uintptr_t) remote_addr;
109+
sreq->op.atomic.remote_mkey = mkey;
110+
sreq->op.atomic.order = nlong_order;
111+
112+
sreq->flags = 0;
113+
}
114+
115+
static inline void mca_atomic_mxm_post(mxm_send_req_t *sreq)
116+
{
117+
mxm_error_t mxm_err;
118+
119+
mxm_err = mxm_req_send(sreq);
120+
if (OPAL_UNLIKELY(MXM_OK != mxm_err)) {
121+
ATOMIC_ERROR("mxm_req_send failed, mxm_error = %d",
122+
mxm_err);
123+
oshmem_shmem_abort(-1);
124+
}
125+
126+
mxm_req_wait(&sreq->base);
127+
if (OPAL_UNLIKELY(MXM_OK != sreq->base.error)) {
128+
ATOMIC_ERROR("mxm_req_wait got non MXM_OK error: %d",
129+
sreq->base.error);
130+
oshmem_shmem_abort(-1);
131+
}
132+
}
133+
134+
END_C_DECLS
135+
74136
#endif /* MCA_ATOMIC_MXM_H */

‎oshmem/mca/atomic/mxm/atomic_mxm_cswap.c

+2-63
Original file line numberDiff line numberDiff line change
@@ -31,81 +31,20 @@ int mca_atomic_mxm_cswap(void *target,
3131
size_t nlong,
3232
int pe)
3333
{
34-
unsigned my_pe;
35-
uint8_t nlong_order;
36-
void *remote_addr;
3734
mxm_send_req_t sreq;
38-
mxm_error_t mxm_err;
39-
sshmem_mkey_t *r_mkey;
4035

41-
my_pe = oshmem_my_proc_id();
42-
mxm_err = MXM_OK;
36+
mca_atomic_mxm_req_init(&sreq, pe, target, nlong);
4337

44-
switch (nlong) {
45-
case 1:
46-
nlong_order = 0;
47-
break;
48-
case 2:
49-
nlong_order = 1;
50-
break;
51-
case 4:
52-
nlong_order = 2;
53-
break;
54-
case 8:
55-
nlong_order = 3;
56-
break;
57-
default:
58-
ATOMIC_ERROR("[#%d] Type size must be 1/2/4 or 8 bytes.", my_pe);
59-
oshmem_shmem_abort(-1);
60-
return OSHMEM_ERR_BAD_PARAM;
61-
}
62-
63-
r_mkey = mca_memheap_base_get_cached_mkey(pe, target, MXM_PTL_RDMA, &remote_addr);
64-
if (!r_mkey) {
65-
ATOMIC_ERROR("[#%d] %p is not address of symmetric variable",
66-
my_pe, target);
67-
oshmem_shmem_abort(-1);
68-
return OSHMEM_ERR_BAD_PARAM;
69-
}
70-
71-
/* mxm request init */
72-
sreq.base.state = MXM_REQ_NEW;
73-
sreq.base.mq = mca_atomic_mxm_spml_self->mxm_mq;
74-
sreq.base.conn = mca_atomic_mxm_spml_self->mxm_peers[pe].mxm_hw_rdma_conn;
75-
sreq.base.completed_cb = NULL;
76-
sreq.base.data_type = MXM_REQ_DATA_BUFFER;
77-
78-
/* set data */
7938
sreq.base.data.buffer.ptr = (void *) value;
80-
sreq.base.data.buffer.length = nlong;
81-
sreq.base.data.buffer.memh = MXM_INVALID_MEM_HANDLE;
82-
83-
sreq.op.atomic.remote_vaddr = (uintptr_t) remote_addr;
84-
sreq.flags = 0;
85-
sreq.op.atomic.remote_mkey = to_mxm_mkey(r_mkey);
86-
sreq.op.atomic.order = nlong_order;
87-
8839
if (NULL == cond) {
8940
sreq.opcode = MXM_REQ_OP_ATOMIC_SWAP;
9041
} else {
9142
memcpy(&sreq.op.atomic.value, cond, nlong);
9243
sreq.opcode = MXM_REQ_OP_ATOMIC_CSWAP;
9344
}
9445

95-
if (MXM_OK != (mxm_err = mxm_req_send(&sreq))) {
96-
ATOMIC_ERROR("[#%d] mxm_req_send failed, mxm_error = %d",
97-
my_pe, mxm_err);
98-
oshmem_shmem_abort(-1);
99-
return OSHMEM_ERROR;
100-
}
46+
mca_atomic_mxm_post(&sreq);
10147

102-
mxm_req_wait(&sreq.base);
103-
if (MXM_OK != sreq.base.error) {
104-
ATOMIC_ERROR("[#%d] mxm_req_wait got non MXM_OK error: %d",
105-
my_pe, sreq.base.error);
106-
oshmem_shmem_abort(-1);
107-
return OSHMEM_ERROR;
108-
}
10948
memcpy(prev, value, nlong);
11049

11150
return OSHMEM_SUCCESS;

‎oshmem/mca/atomic/mxm/atomic_mxm_fadd.c

+3-69
Original file line numberDiff line numberDiff line change
@@ -32,86 +32,20 @@ int mca_atomic_mxm_fadd(void *target,
3232
int pe,
3333
struct oshmem_op_t *op)
3434
{
35-
unsigned my_pe;
36-
uint8_t nlong_order;
37-
void *remote_addr;
3835
mxm_send_req_t sreq;
39-
mxm_error_t mxm_err;
40-
sshmem_mkey_t *r_mkey;
4136
static char dummy_buf[8];
4237

43-
my_pe = oshmem_my_proc_id();
44-
mxm_err = MXM_OK;
38+
mca_atomic_mxm_req_init(&sreq, pe, target, nlong);
4539

46-
switch (nlong) {
47-
case 1:
48-
nlong_order = 0;
49-
break;
50-
case 2:
51-
nlong_order = 1;
52-
break;
53-
case 4:
54-
nlong_order = 2;
55-
break;
56-
case 8:
57-
nlong_order = 3;
58-
break;
59-
default:
60-
ATOMIC_ERROR("[#%d] Type size must be 1/2/4 or 8 bytes.", my_pe);
61-
oshmem_shmem_abort(-1);
62-
return OSHMEM_ERR_BAD_PARAM;
63-
}
64-
65-
r_mkey = mca_memheap_base_get_cached_mkey(pe, target, MXM_PTL_RDMA, &remote_addr);
66-
if (!r_mkey) {
67-
ATOMIC_ERROR("[#%d] %p is not address of symmetric variable",
68-
my_pe, target);
69-
oshmem_shmem_abort(-1);
70-
return OSHMEM_ERR_BAD_PARAM;
71-
}
72-
73-
/* mxm request init */
74-
sreq.base.state = MXM_REQ_NEW;
75-
sreq.base.mq = mca_atomic_mxm_spml_self->mxm_mq;
76-
sreq.base.conn = mca_atomic_mxm_spml_self->mxm_peers[pe].mxm_hw_rdma_conn;
77-
sreq.base.completed_cb = NULL;
78-
sreq.base.data_type = MXM_REQ_DATA_BUFFER;
79-
80-
sreq.op.atomic.remote_vaddr = (uintptr_t) remote_addr;
81-
sreq.op.atomic.remote_mkey = to_mxm_mkey(r_mkey);
8240
memcpy(&sreq.op.atomic.value, value, nlong);
83-
sreq.op.atomic.order = nlong_order;
84-
85-
/* Do we need atomic 'add' or atomic 'fetch and add'? */
41+
sreq.opcode = MXM_REQ_OP_ATOMIC_FADD;
8642
if (NULL == prev) {
8743
sreq.base.data.buffer.ptr = dummy_buf;
88-
sreq.base.data.buffer.length = nlong;
89-
sreq.base.data.buffer.memh = MXM_INVALID_MEM_HANDLE;
90-
sreq.flags = 0;
91-
sreq.opcode = MXM_REQ_OP_ATOMIC_FADD;
9244
} else {
9345
sreq.base.data.buffer.ptr = prev;
94-
sreq.base.data.buffer.length = nlong;
95-
sreq.base.data.buffer.memh = MXM_INVALID_MEM_HANDLE;
96-
sreq.flags = 0;
97-
98-
sreq.opcode = MXM_REQ_OP_ATOMIC_FADD;
9946
}
10047

101-
if (MXM_OK != (mxm_err = mxm_req_send(&sreq))) {
102-
ATOMIC_ERROR("[#%d] mxm_req_send failed, mxm_error = %d",
103-
my_pe, mxm_err);
104-
oshmem_shmem_abort(-1);
105-
return OSHMEM_ERROR;
106-
}
107-
108-
mxm_req_wait(&sreq.base);
109-
if (MXM_OK != sreq.base.error) {
110-
ATOMIC_ERROR("[#%d] mxm_req_wait got non MXM_OK error: %d",
111-
my_pe, sreq.base.error);
112-
oshmem_shmem_abort(-1);
113-
return OSHMEM_ERROR;
114-
}
48+
mca_atomic_mxm_post(&sreq);
11549

11650
return OSHMEM_SUCCESS;
11751
}

0 commit comments

Comments
 (0)