Skip to content

Commit 514aee6

Browse files
rleonjgunthorpe
rleon
authored and
jgunthorpe
committed
RDMA: Globally allocate and release QP memory
Convert QP object to follow IB/core general allocation scheme. That change allows us to make sure that restrack properly kref the memory. Link: https://lore.kernel.org/r/48e767124758aeecc433360ddd85eaa6325b34d9.1627040189.git.leonro@nvidia.com Reviewed-by: Gal Pressman <[email protected]> #efa Tested-by: Gal Pressman <[email protected]> Reviewed-by: Dennis Dalessandro <[email protected]> #rdma and core Tested-by: Dennis Dalessandro <[email protected]> Signed-off-by: Leon Romanovsky <[email protected]> Tested-by: Tatyana Nikolova <[email protected]> Signed-off-by: Jason Gunthorpe <[email protected]>
1 parent 44da373 commit 514aee6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+404
-549
lines changed

‎drivers/infiniband/core/core_priv.h

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -322,13 +322,14 @@ _ib_create_qp(struct ib_device *dev, struct ib_pd *pd,
322322
struct ib_uqp_object *uobj, const char *caller)
323323
{
324324
struct ib_qp *qp;
325+
int ret;
325326

326327
if (!dev->ops.create_qp)
327328
return ERR_PTR(-EOPNOTSUPP);
328329

329-
qp = dev->ops.create_qp(pd, attr, udata);
330-
if (IS_ERR(qp))
331-
return qp;
330+
qp = rdma_zalloc_drv_obj_numa(dev, ib_qp);
331+
if (!qp)
332+
return ERR_PTR(-ENOMEM);
332333

333334
qp->device = dev;
334335
qp->pd = pd;
@@ -337,23 +338,36 @@ _ib_create_qp(struct ib_device *dev, struct ib_pd *pd,
337338

338339
qp->qp_type = attr->qp_type;
339340
qp->rwq_ind_tbl = attr->rwq_ind_tbl;
340-
qp->send_cq = attr->send_cq;
341-
qp->recv_cq = attr->recv_cq;
342341
qp->srq = attr->srq;
343-
qp->rwq_ind_tbl = attr->rwq_ind_tbl;
344342
qp->event_handler = attr->event_handler;
345343
qp->port = attr->port_num;
346344

347-
atomic_set(&qp->usecnt, 0);
348345
spin_lock_init(&qp->mr_lock);
349346
INIT_LIST_HEAD(&qp->rdma_mrs);
350347
INIT_LIST_HEAD(&qp->sig_mrs);
351348

352349
rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP);
353350
WARN_ONCE(!udata && !caller, "Missing kernel QP owner");
354351
rdma_restrack_set_name(&qp->res, udata ? NULL : caller);
352+
ret = dev->ops.create_qp(qp, attr, udata);
353+
if (ret)
354+
goto err_create;
355+
356+
/*
357+
* TODO: The mlx4 internally overwrites send_cq and recv_cq.
358+
* Unfortunately, it is not an easy task to fix that driver.
359+
*/
360+
qp->send_cq = attr->send_cq;
361+
qp->recv_cq = attr->recv_cq;
362+
355363
rdma_restrack_add(&qp->res);
356364
return qp;
365+
366+
err_create:
367+
rdma_restrack_put(&qp->res);
368+
kfree(qp);
369+
return ERR_PTR(ret);
370+
357371
}
358372

359373
struct rdma_dev_addr;

‎drivers/infiniband/core/device.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2654,6 +2654,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
26542654
SET_DEVICE_OP(dev_ops, get_hw_stats);
26552655
SET_DEVICE_OP(dev_ops, get_link_layer);
26562656
SET_DEVICE_OP(dev_ops, get_netdev);
2657+
SET_DEVICE_OP(dev_ops, get_numa_node);
26572658
SET_DEVICE_OP(dev_ops, get_port_immutable);
26582659
SET_DEVICE_OP(dev_ops, get_vector_affinity);
26592660
SET_DEVICE_OP(dev_ops, get_vf_config);
@@ -2710,6 +2711,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
27102711
SET_OBJ_SIZE(dev_ops, ib_cq);
27112712
SET_OBJ_SIZE(dev_ops, ib_mw);
27122713
SET_OBJ_SIZE(dev_ops, ib_pd);
2714+
SET_OBJ_SIZE(dev_ops, ib_qp);
27132715
SET_OBJ_SIZE(dev_ops, ib_rwq_ind_table);
27142716
SET_OBJ_SIZE(dev_ops, ib_srq);
27152717
SET_OBJ_SIZE(dev_ops, ib_ucontext);

‎drivers/infiniband/core/restrack.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,7 @@ void rdma_restrack_del(struct rdma_restrack_entry *res)
343343
rt = &dev->res[res->type];
344344

345345
old = xa_erase(&rt->xa, res->id);
346-
if (res->type == RDMA_RESTRACK_MR || res->type == RDMA_RESTRACK_QP)
346+
if (res->type == RDMA_RESTRACK_MR)
347347
return;
348348
WARN_ON(old != res);
349349

‎drivers/infiniband/core/verbs.c

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1963,30 +1963,32 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)
19631963
rdma_rw_cleanup_mrs(qp);
19641964

19651965
rdma_counter_unbind_qp(qp, true);
1966-
rdma_restrack_del(&qp->res);
19671966
ret = qp->device->ops.destroy_qp(qp, udata);
1968-
if (!ret) {
1969-
if (alt_path_sgid_attr)
1970-
rdma_put_gid_attr(alt_path_sgid_attr);
1971-
if (av_sgid_attr)
1972-
rdma_put_gid_attr(av_sgid_attr);
1973-
if (pd)
1974-
atomic_dec(&pd->usecnt);
1975-
if (scq)
1976-
atomic_dec(&scq->usecnt);
1977-
if (rcq)
1978-
atomic_dec(&rcq->usecnt);
1979-
if (srq)
1980-
atomic_dec(&srq->usecnt);
1981-
if (ind_tbl)
1982-
atomic_dec(&ind_tbl->usecnt);
1983-
if (sec)
1984-
ib_destroy_qp_security_end(sec);
1985-
} else {
1967+
if (ret) {
19861968
if (sec)
19871969
ib_destroy_qp_security_abort(sec);
1970+
return ret;
19881971
}
19891972

1973+
if (alt_path_sgid_attr)
1974+
rdma_put_gid_attr(alt_path_sgid_attr);
1975+
if (av_sgid_attr)
1976+
rdma_put_gid_attr(av_sgid_attr);
1977+
if (pd)
1978+
atomic_dec(&pd->usecnt);
1979+
if (scq)
1980+
atomic_dec(&scq->usecnt);
1981+
if (rcq)
1982+
atomic_dec(&rcq->usecnt);
1983+
if (srq)
1984+
atomic_dec(&srq->usecnt);
1985+
if (ind_tbl)
1986+
atomic_dec(&ind_tbl->usecnt);
1987+
if (sec)
1988+
ib_destroy_qp_security_end(sec);
1989+
1990+
rdma_restrack_del(&qp->res);
1991+
kfree(qp);
19901992
return ret;
19911993
}
19921994
EXPORT_SYMBOL(ib_destroy_qp_user);

‎drivers/infiniband/hw/bnxt_re/ib_verbs.c

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -815,7 +815,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
815815
if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) {
816816
rc = bnxt_re_destroy_gsi_sqp(qp);
817817
if (rc)
818-
goto sh_fail;
818+
return rc;
819819
}
820820

821821
mutex_lock(&rdev->qp_lock);
@@ -826,10 +826,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
826826
ib_umem_release(qp->rumem);
827827
ib_umem_release(qp->sumem);
828828

829-
kfree(qp);
830829
return 0;
831-
sh_fail:
832-
return rc;
833830
}
834831

835832
static u8 __from_ib_qp_type(enum ib_qp_type type)
@@ -1402,27 +1399,22 @@ static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
14021399
return rc;
14031400
}
14041401

1405-
struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
1406-
struct ib_qp_init_attr *qp_init_attr,
1407-
struct ib_udata *udata)
1402+
int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1403+
struct ib_udata *udata)
14081404
{
1405+
struct ib_pd *ib_pd = ib_qp->pd;
14091406
struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
14101407
struct bnxt_re_dev *rdev = pd->rdev;
14111408
struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1412-
struct bnxt_re_qp *qp;
1409+
struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
14131410
int rc;
14141411

14151412
rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
14161413
if (!rc) {
14171414
rc = -EINVAL;
1418-
goto exit;
1415+
goto fail;
14191416
}
14201417

1421-
qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1422-
if (!qp) {
1423-
rc = -ENOMEM;
1424-
goto exit;
1425-
}
14261418
qp->rdev = rdev;
14271419
rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
14281420
if (rc)
@@ -1465,16 +1457,14 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
14651457
mutex_unlock(&rdev->qp_lock);
14661458
atomic_inc(&rdev->qp_count);
14671459

1468-
return &qp->ib_qp;
1460+
return 0;
14691461
qp_destroy:
14701462
bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
14711463
free_umem:
14721464
ib_umem_release(qp->rumem);
14731465
ib_umem_release(qp->sumem);
14741466
fail:
1475-
kfree(qp);
1476-
exit:
1477-
return ERR_PTR(rc);
1467+
return rc;
14781468
}
14791469

14801470
static u8 __from_ib_qp_state(enum ib_qp_state state)

‎drivers/infiniband/hw/bnxt_re/ib_verbs.h

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,9 @@ struct bnxt_re_srq {
7878
};
7979

8080
struct bnxt_re_qp {
81+
struct ib_qp ib_qp;
8182
struct list_head list;
8283
struct bnxt_re_dev *rdev;
83-
struct ib_qp ib_qp;
8484
spinlock_t sq_lock; /* protect sq */
8585
spinlock_t rq_lock; /* protect rq */
8686
struct bnxt_qplib_qp qplib_qp;
@@ -179,9 +179,8 @@ int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
179179
int bnxt_re_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
180180
int bnxt_re_post_srq_recv(struct ib_srq *srq, const struct ib_recv_wr *recv_wr,
181181
const struct ib_recv_wr **bad_recv_wr);
182-
struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd,
183-
struct ib_qp_init_attr *qp_init_attr,
184-
struct ib_udata *udata);
182+
int bnxt_re_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,
183+
struct ib_udata *udata);
185184
int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
186185
int qp_attr_mask, struct ib_udata *udata);
187186
int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,

‎drivers/infiniband/hw/bnxt_re/main.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -709,6 +709,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
709709
INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah),
710710
INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq),
711711
INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd),
712+
INIT_RDMA_OBJ_SIZE(ib_qp, bnxt_re_qp, ib_qp),
712713
INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq),
713714
INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx),
714715
};

‎drivers/infiniband/hw/cxgb4/iw_cxgb4.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -989,9 +989,8 @@ int c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);
989989
int c4iw_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attrs,
990990
struct ib_udata *udata);
991991
int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);
992-
struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
993-
struct ib_qp_init_attr *attrs,
994-
struct ib_udata *udata);
992+
int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
993+
struct ib_udata *udata);
995994
int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
996995
int attr_mask, struct ib_udata *udata);
997996
int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,

‎drivers/infiniband/hw/cxgb4/provider.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ static const struct ib_device_ops c4iw_dev_ops = {
499499
INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq),
500500
INIT_RDMA_OBJ_SIZE(ib_mw, c4iw_mw, ibmw),
501501
INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),
502+
INIT_RDMA_OBJ_SIZE(ib_qp, c4iw_qp, ibqp),
502503
INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq),
503504
INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext),
504505
};

‎drivers/infiniband/hw/cxgb4/qp.c

Lines changed: 13 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2103,16 +2103,15 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
21032103
ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq);
21042104

21052105
c4iw_put_wr_wait(qhp->wr_waitp);
2106-
2107-
kfree(qhp);
21082106
return 0;
21092107
}
21102108

2111-
struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
2112-
struct ib_udata *udata)
2109+
int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
2110+
struct ib_udata *udata)
21132111
{
2112+
struct ib_pd *pd = qp->pd;
21142113
struct c4iw_dev *rhp;
2115-
struct c4iw_qp *qhp;
2114+
struct c4iw_qp *qhp = to_c4iw_qp(qp);
21162115
struct c4iw_pd *php;
21172116
struct c4iw_cq *schp;
21182117
struct c4iw_cq *rchp;
@@ -2124,44 +2123,36 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
21242123
struct c4iw_mm_entry *sq_key_mm, *rq_key_mm = NULL, *sq_db_key_mm;
21252124
struct c4iw_mm_entry *rq_db_key_mm = NULL, *ma_sync_key_mm = NULL;
21262125

2127-
pr_debug("ib_pd %p\n", pd);
2128-
21292126
if (attrs->qp_type != IB_QPT_RC || attrs->create_flags)
2130-
return ERR_PTR(-EOPNOTSUPP);
2127+
return -EOPNOTSUPP;
21312128

21322129
php = to_c4iw_pd(pd);
21332130
rhp = php->rhp;
21342131
schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid);
21352132
rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid);
21362133
if (!schp || !rchp)
2137-
return ERR_PTR(-EINVAL);
2134+
return -EINVAL;
21382135

21392136
if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE)
2140-
return ERR_PTR(-EINVAL);
2137+
return -EINVAL;
21412138

21422139
if (!attrs->srq) {
21432140
if (attrs->cap.max_recv_wr > rhp->rdev.hw_queue.t4_max_rq_size)
2144-
return ERR_PTR(-E2BIG);
2141+
return -E2BIG;
21452142
rqsize = attrs->cap.max_recv_wr + 1;
21462143
if (rqsize < 8)
21472144
rqsize = 8;
21482145
}
21492146

21502147
if (attrs->cap.max_send_wr > rhp->rdev.hw_queue.t4_max_sq_size)
2151-
return ERR_PTR(-E2BIG);
2148+
return -E2BIG;
21522149
sqsize = attrs->cap.max_send_wr + 1;
21532150
if (sqsize < 8)
21542151
sqsize = 8;
21552152

2156-
qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
2157-
if (!qhp)
2158-
return ERR_PTR(-ENOMEM);
2159-
21602153
qhp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);
2161-
if (!qhp->wr_waitp) {
2162-
ret = -ENOMEM;
2163-
goto err_free_qhp;
2164-
}
2154+
if (!qhp->wr_waitp)
2155+
return -ENOMEM;
21652156

21662157
qhp->wq.sq.size = sqsize;
21672158
qhp->wq.sq.memsize =
@@ -2339,7 +2330,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
23392330
qhp->wq.sq.qid, qhp->wq.sq.size, qhp->wq.sq.memsize,
23402331
attrs->cap.max_send_wr, qhp->wq.rq.qid, qhp->wq.rq.size,
23412332
qhp->wq.rq.memsize, attrs->cap.max_recv_wr);
2342-
return &qhp->ibqp;
2333+
return 0;
23432334
err_free_ma_sync_key:
23442335
kfree(ma_sync_key_mm);
23452336
err_free_rq_db_key:
@@ -2359,9 +2350,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
23592350
ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !attrs->srq);
23602351
err_free_wr_wait:
23612352
c4iw_put_wr_wait(qhp->wr_waitp);
2362-
err_free_qhp:
2363-
kfree(qhp);
2364-
return ERR_PTR(ret);
2353+
return ret;
23652354
}
23662355

23672356
int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,

‎drivers/infiniband/hw/efa/efa.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,8 @@ int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
132132
int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
133133
int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
134134
int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
135-
struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
136-
struct ib_qp_init_attr *init_attr,
137-
struct ib_udata *udata);
135+
int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
136+
struct ib_udata *udata);
138137
int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
139138
int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
140139
struct ib_udata *udata);

‎drivers/infiniband/hw/efa/efa_main.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ static const struct ib_device_ops efa_dev_ops = {
271271
INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),
272272
INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq),
273273
INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd),
274+
INIT_RDMA_OBJ_SIZE(ib_qp, efa_qp, ibqp),
274275
INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext),
275276
};
276277

0 commit comments

Comments
 (0)