Skip to content

Commit 79bccd7

Browse files
Shiraz Saleemrleon
Shiraz Saleem
authored and
rleon
committed
RDMA/mana_ib: Add port statistics support
Implement alloc_hw_port_stats and get_hw_stats APIs to support querying MANA VF port level statistics from rdma stat tool. Example output from rdma stat tool: $rdma statistic show link mana_0/1 -p link mana_0/1 requester_timeout 45 requester_oos_nak 0 requester_rnr_nak 0 responder_rnr_nak 0 responder_oos 0 responder_dup_request 0 requester_implicit_nak 0 requester_readresp_psn_mismatch 0 nak_inv_req 0 nak_access_error 0 nak_opp_error 0 nak_inv_read 0 responder_local_len_error 0 requestor_local_prot_error 0 responder_rem_access_error 0 responder_local_qp_error 0 responder_malformed_wqe 0 general_hw_error 6 requester_rnr_nak_retries_exceeded 0 requester_retries_exceeded 5 total_fatal_error 6 received_cnps 0 num_qps_congested 0 rate_inc_events 0 num_qps_recovered 0 current_rate 100000 Signed-off-by: Shiraz Saleem <[email protected]> Signed-off-by: Konstantin Taranov <[email protected]> Link: https://patch.msgid.link/[email protected] Reviewed-by: Long Li <[email protected]> Signed-off-by: Leon Romanovsky <[email protected]>
1 parent cd3c5dd commit 79bccd7

File tree

5 files changed

+194
-1
lines changed

5 files changed

+194
-1
lines changed

‎drivers/infiniband/hw/mana/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
# SPDX-License-Identifier: GPL-2.0-only
22
obj-$(CONFIG_MANA_INFINIBAND) += mana_ib.o
33

4-
mana_ib-y := device.o main.o wq.o qp.o cq.o mr.o ah.o wr.o
4+
mana_ib-y := device.o main.o wq.o qp.o cq.o mr.o ah.o wr.o counters.o

‎drivers/infiniband/hw/mana/counters.c‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/*
3+
* Copyright (c) 2024, Microsoft Corporation. All rights reserved.
4+
*/
5+
6+
#include "counters.h"
7+
8+
static const struct rdma_stat_desc mana_ib_port_stats_desc[] = {
9+
[MANA_IB_REQUESTER_TIMEOUT].name = "requester_timeout",
10+
[MANA_IB_REQUESTER_OOS_NAK].name = "requester_oos_nak",
11+
[MANA_IB_REQUESTER_RNR_NAK].name = "requester_rnr_nak",
12+
[MANA_IB_RESPONDER_RNR_NAK].name = "responder_rnr_nak",
13+
[MANA_IB_RESPONDER_OOS].name = "responder_oos",
14+
[MANA_IB_RESPONDER_DUP_REQUEST].name = "responder_dup_request",
15+
[MANA_IB_REQUESTER_IMPLICIT_NAK].name = "requester_implicit_nak",
16+
[MANA_IB_REQUESTER_READRESP_PSN_MISMATCH].name = "requester_readresp_psn_mismatch",
17+
[MANA_IB_NAK_INV_REQ].name = "nak_inv_req",
18+
[MANA_IB_NAK_ACCESS_ERR].name = "nak_access_error",
19+
[MANA_IB_NAK_OPP_ERR].name = "nak_opp_error",
20+
[MANA_IB_NAK_INV_READ].name = "nak_inv_read",
21+
[MANA_IB_RESPONDER_LOCAL_LEN_ERR].name = "responder_local_len_error",
22+
[MANA_IB_REQUESTOR_LOCAL_PROT_ERR].name = "requestor_local_prot_error",
23+
[MANA_IB_RESPONDER_REM_ACCESS_ERR].name = "responder_rem_access_error",
24+
[MANA_IB_RESPONDER_LOCAL_QP_ERR].name = "responder_local_qp_error",
25+
[MANA_IB_RESPONDER_MALFORMED_WQE].name = "responder_malformed_wqe",
26+
[MANA_IB_GENERAL_HW_ERR].name = "general_hw_error",
27+
[MANA_IB_REQUESTER_RNR_NAK_RETRIES_EXCEEDED].name = "requester_rnr_nak_retries_exceeded",
28+
[MANA_IB_REQUESTER_RETRIES_EXCEEDED].name = "requester_retries_exceeded",
29+
[MANA_IB_TOTAL_FATAL_ERR].name = "total_fatal_error",
30+
[MANA_IB_RECEIVED_CNPS].name = "received_cnps",
31+
[MANA_IB_NUM_QPS_CONGESTED].name = "num_qps_congested",
32+
[MANA_IB_RATE_INC_EVENTS].name = "rate_inc_events",
33+
[MANA_IB_NUM_QPS_RECOVERED].name = "num_qps_recovered",
34+
[MANA_IB_CURRENT_RATE].name = "current_rate",
35+
};
36+
37+
struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,
38+
u32 port_num)
39+
{
40+
return rdma_alloc_hw_stats_struct(mana_ib_port_stats_desc,
41+
ARRAY_SIZE(mana_ib_port_stats_desc),
42+
RDMA_HW_STATS_DEFAULT_LIFESPAN);
43+
}
44+
45+
int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
46+
u32 port_num, int index)
47+
{
48+
struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,
49+
ib_dev);
50+
struct mana_rnic_query_vf_cntrs_resp resp = {};
51+
struct mana_rnic_query_vf_cntrs_req req = {};
52+
int err;
53+
54+
mana_gd_init_req_hdr(&req.hdr, MANA_IB_QUERY_VF_COUNTERS,
55+
sizeof(req), sizeof(resp));
56+
req.hdr.dev_id = mdev->gdma_dev->dev_id;
57+
req.adapter = mdev->adapter_handle;
58+
59+
err = mana_gd_send_request(mdev_to_gc(mdev), sizeof(req), &req,
60+
sizeof(resp), &resp);
61+
if (err) {
62+
ibdev_err(&mdev->ib_dev, "Failed to query vf counters err %d",
63+
err);
64+
return err;
65+
}
66+
67+
stats->value[MANA_IB_REQUESTER_TIMEOUT] = resp.requester_timeout;
68+
stats->value[MANA_IB_REQUESTER_OOS_NAK] = resp.requester_oos_nak;
69+
stats->value[MANA_IB_REQUESTER_RNR_NAK] = resp.requester_rnr_nak;
70+
stats->value[MANA_IB_RESPONDER_RNR_NAK] = resp.responder_rnr_nak;
71+
stats->value[MANA_IB_RESPONDER_OOS] = resp.responder_oos;
72+
stats->value[MANA_IB_RESPONDER_DUP_REQUEST] = resp.responder_dup_request;
73+
stats->value[MANA_IB_REQUESTER_IMPLICIT_NAK] =
74+
resp.requester_implicit_nak;
75+
stats->value[MANA_IB_REQUESTER_READRESP_PSN_MISMATCH] =
76+
resp.requester_readresp_psn_mismatch;
77+
stats->value[MANA_IB_NAK_INV_REQ] = resp.nak_inv_req;
78+
stats->value[MANA_IB_NAK_ACCESS_ERR] = resp.nak_access_err;
79+
stats->value[MANA_IB_NAK_OPP_ERR] = resp.nak_opp_err;
80+
stats->value[MANA_IB_NAK_INV_READ] = resp.nak_inv_read;
81+
stats->value[MANA_IB_RESPONDER_LOCAL_LEN_ERR] =
82+
resp.responder_local_len_err;
83+
stats->value[MANA_IB_REQUESTOR_LOCAL_PROT_ERR] =
84+
resp.requestor_local_prot_err;
85+
stats->value[MANA_IB_RESPONDER_REM_ACCESS_ERR] =
86+
resp.responder_rem_access_err;
87+
stats->value[MANA_IB_RESPONDER_LOCAL_QP_ERR] =
88+
resp.responder_local_qp_err;
89+
stats->value[MANA_IB_RESPONDER_MALFORMED_WQE] =
90+
resp.responder_malformed_wqe;
91+
stats->value[MANA_IB_GENERAL_HW_ERR] = resp.general_hw_err;
92+
stats->value[MANA_IB_REQUESTER_RNR_NAK_RETRIES_EXCEEDED] =
93+
resp.requester_rnr_nak_retries_exceeded;
94+
stats->value[MANA_IB_REQUESTER_RETRIES_EXCEEDED] =
95+
resp.requester_retries_exceeded;
96+
stats->value[MANA_IB_TOTAL_FATAL_ERR] = resp.total_fatal_err;
97+
98+
stats->value[MANA_IB_RECEIVED_CNPS] = resp.received_cnps;
99+
stats->value[MANA_IB_NUM_QPS_CONGESTED] = resp.num_qps_congested;
100+
stats->value[MANA_IB_RATE_INC_EVENTS] = resp.rate_inc_events;
101+
stats->value[MANA_IB_NUM_QPS_RECOVERED] = resp.num_qps_recovered;
102+
stats->value[MANA_IB_CURRENT_RATE] = resp.current_rate;
103+
104+
return ARRAY_SIZE(mana_ib_port_stats_desc);
105+
}

‎drivers/infiniband/hw/mana/counters.h‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/* SPDX-License-Identifier: GPL-2.0-only */
2+
/*
3+
* Copyright (c) 2024 Microsoft Corporation. All rights reserved.
4+
*/
5+
6+
#ifndef _COUNTERS_H_
7+
#define _COUNTERS_H_
8+
9+
#include "mana_ib.h"
10+
11+
enum mana_ib_port_counters {
12+
MANA_IB_REQUESTER_TIMEOUT,
13+
MANA_IB_REQUESTER_OOS_NAK,
14+
MANA_IB_REQUESTER_RNR_NAK,
15+
MANA_IB_RESPONDER_RNR_NAK,
16+
MANA_IB_RESPONDER_OOS,
17+
MANA_IB_RESPONDER_DUP_REQUEST,
18+
MANA_IB_REQUESTER_IMPLICIT_NAK,
19+
MANA_IB_REQUESTER_READRESP_PSN_MISMATCH,
20+
MANA_IB_NAK_INV_REQ,
21+
MANA_IB_NAK_ACCESS_ERR,
22+
MANA_IB_NAK_OPP_ERR,
23+
MANA_IB_NAK_INV_READ,
24+
MANA_IB_RESPONDER_LOCAL_LEN_ERR,
25+
MANA_IB_REQUESTOR_LOCAL_PROT_ERR,
26+
MANA_IB_RESPONDER_REM_ACCESS_ERR,
27+
MANA_IB_RESPONDER_LOCAL_QP_ERR,
28+
MANA_IB_RESPONDER_MALFORMED_WQE,
29+
MANA_IB_GENERAL_HW_ERR,
30+
MANA_IB_REQUESTER_RNR_NAK_RETRIES_EXCEEDED,
31+
MANA_IB_REQUESTER_RETRIES_EXCEEDED,
32+
MANA_IB_TOTAL_FATAL_ERR,
33+
MANA_IB_RECEIVED_CNPS,
34+
MANA_IB_NUM_QPS_CONGESTED,
35+
MANA_IB_RATE_INC_EVENTS,
36+
MANA_IB_NUM_QPS_RECOVERED,
37+
MANA_IB_CURRENT_RATE,
38+
};
39+
40+
struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,
41+
u32 port_num);
42+
int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
43+
u32 port_num, int index);
44+
#endif /* _COUNTERS_H_ */

‎drivers/infiniband/hw/mana/device.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ static const struct ib_device_ops mana_ib_dev_ops = {
5959
ib_ind_table),
6060
};
6161

62+
static const struct ib_device_ops mana_ib_stats_ops = {
63+
.alloc_hw_port_stats = mana_ib_alloc_hw_port_stats,
64+
.get_hw_stats = mana_ib_get_hw_stats,
65+
};
66+
6267
static int mana_ib_probe(struct auxiliary_device *adev,
6368
const struct auxiliary_device_id *id)
6469
{
@@ -124,6 +129,8 @@ static int mana_ib_probe(struct auxiliary_device *adev,
124129
goto deregister_device;
125130
}
126131

132+
ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
133+
127134
ret = mana_ib_create_eqs(dev);
128135
if (ret) {
129136
ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);

‎drivers/infiniband/hw/mana/mana_ib.h‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
#include <net/mana/mana.h>
1717
#include "shadow_queue.h"
18+
#include "counters.h"
1819

1920
#define PAGE_SZ_BM \
2021
(SZ_4K | SZ_8K | SZ_16K | SZ_32K | SZ_64K | SZ_128K | SZ_256K | \
@@ -205,6 +206,7 @@ enum mana_ib_command_code {
205206
MANA_IB_CREATE_RC_QP = 0x3000a,
206207
MANA_IB_DESTROY_RC_QP = 0x3000b,
207208
MANA_IB_SET_QP_STATE = 0x3000d,
209+
MANA_IB_QUERY_VF_COUNTERS = 0x30022,
208210
};
209211

210212
struct mana_ib_query_adapter_caps_req {
@@ -476,6 +478,41 @@ struct mana_rdma_cqe {
476478
};
477479
}; /* HW DATA */
478480

481+
struct mana_rnic_query_vf_cntrs_req {
482+
struct gdma_req_hdr hdr;
483+
mana_handle_t adapter;
484+
}; /* HW Data */
485+
486+
struct mana_rnic_query_vf_cntrs_resp {
487+
struct gdma_resp_hdr hdr;
488+
u64 requester_timeout;
489+
u64 requester_oos_nak;
490+
u64 requester_rnr_nak;
491+
u64 responder_rnr_nak;
492+
u64 responder_oos;
493+
u64 responder_dup_request;
494+
u64 requester_implicit_nak;
495+
u64 requester_readresp_psn_mismatch;
496+
u64 nak_inv_req;
497+
u64 nak_access_err;
498+
u64 nak_opp_err;
499+
u64 nak_inv_read;
500+
u64 responder_local_len_err;
501+
u64 requestor_local_prot_err;
502+
u64 responder_rem_access_err;
503+
u64 responder_local_qp_err;
504+
u64 responder_malformed_wqe;
505+
u64 general_hw_err;
506+
u64 requester_rnr_nak_retries_exceeded;
507+
u64 requester_retries_exceeded;
508+
u64 total_fatal_err;
509+
u64 received_cnps;
510+
u64 num_qps_congested;
511+
u64 rate_inc_events;
512+
u64 num_qps_recovered;
513+
u64 current_rate;
514+
}; /* HW Data */
515+
479516
static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
480517
{
481518
return mdev->gdma_dev->gdma_context;

0 commit comments

Comments
 (0)