1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2018 Netronome Systems, Inc. */
3
4 #include <linux/bitops.h>
5 #include <linux/kernel.h>
6 #include <linux/log2.h>
7
8 #include "../nfpcore/nfp_cpp.h"
9 #include "../nfpcore/nfp_nffw.h"
10 #include "../nfp_app.h"
11 #include "../nfp_abi.h"
12 #include "../nfp_main.h"
13 #include "../nfp_net.h"
14 #include "main.h"
15
16 #define NFP_NUM_PRIOS_SYM_NAME "_abi_pci_dscp_num_prio_%u"
17 #define NFP_NUM_BANDS_SYM_NAME "_abi_pci_dscp_num_band_%u"
18 #define NFP_ACT_MASK_SYM_NAME "_abi_nfd_out_q_actions_%u"
19
20 #define NFP_RED_SUPPORT_SYM_NAME "_abi_nfd_out_red_offload_%u"
21
22 #define NFP_QLVL_SYM_NAME "_abi_nfd_out_q_lvls_%u%s"
23 #define NFP_QLVL_STRIDE 16
24 #define NFP_QLVL_BLOG_BYTES 0
25 #define NFP_QLVL_BLOG_PKTS 4
26 #define NFP_QLVL_THRS 8
27 #define NFP_QLVL_ACT 12
28
29 #define NFP_QMSTAT_SYM_NAME "_abi_nfdqm%u_stats%s"
30 #define NFP_QMSTAT_STRIDE 32
31 #define NFP_QMSTAT_NON_STO 0
32 #define NFP_QMSTAT_STO 8
33 #define NFP_QMSTAT_DROP 16
34 #define NFP_QMSTAT_ECN 24
35
36 #define NFP_Q_STAT_SYM_NAME "_abi_nfd_rxq_stats%u%s"
37 #define NFP_Q_STAT_STRIDE 16
38 #define NFP_Q_STAT_PKTS 0
39 #define NFP_Q_STAT_BYTES 8
40
41 #define NFP_NET_ABM_MBOX_CMD NFP_NET_CFG_MBOX_SIMPLE_CMD
42 #define NFP_NET_ABM_MBOX_RET NFP_NET_CFG_MBOX_SIMPLE_RET
43 #define NFP_NET_ABM_MBOX_DATALEN NFP_NET_CFG_MBOX_SIMPLE_VAL
44 #define NFP_NET_ABM_MBOX_RESERVED (NFP_NET_CFG_MBOX_SIMPLE_VAL + 4)
45 #define NFP_NET_ABM_MBOX_DATA (NFP_NET_CFG_MBOX_SIMPLE_VAL + 8)
46
47 static int
nfp_abm_ctrl_stat(struct nfp_abm_link * alink,const struct nfp_rtsym * sym,unsigned int stride,unsigned int offset,unsigned int band,unsigned int queue,bool is_u64,u64 * res)48 nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym,
49 unsigned int stride, unsigned int offset, unsigned int band,
50 unsigned int queue, bool is_u64, u64 *res)
51 {
52 struct nfp_cpp *cpp = alink->abm->app->cpp;
53 u64 val, sym_offset;
54 unsigned int qid;
55 u32 val32;
56 int err;
57
58 qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
59
60 sym_offset = qid * stride + offset;
61 if (is_u64)
62 err = __nfp_rtsym_readq(cpp, sym, 3, 0, sym_offset, &val);
63 else
64 err = __nfp_rtsym_readl(cpp, sym, 3, 0, sym_offset, &val32);
65 if (err) {
66 nfp_err(cpp, "RED offload reading stat failed on vNIC %d band %d queue %d (+ %d)\n",
67 alink->id, band, queue, alink->queue_base);
68 return err;
69 }
70
71 *res = is_u64 ? val : val32;
72 return 0;
73 }
74
__nfp_abm_ctrl_set_q_lvl(struct nfp_abm * abm,unsigned int id,u32 val)75 int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val)
76 {
77 struct nfp_cpp *cpp = abm->app->cpp;
78 u64 sym_offset;
79 int err;
80
81 __clear_bit(id, abm->threshold_undef);
82 if (abm->thresholds[id] == val)
83 return 0;
84
85 sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_THRS;
86 err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, val);
87 if (err) {
88 nfp_err(cpp,
89 "RED offload setting level failed on subqueue %d\n",
90 id);
91 return err;
92 }
93
94 abm->thresholds[id] = val;
95 return 0;
96 }
97
nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link * alink,unsigned int band,unsigned int queue,u32 val)98 int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int band,
99 unsigned int queue, u32 val)
100 {
101 unsigned int threshold;
102
103 threshold = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
104
105 return __nfp_abm_ctrl_set_q_lvl(alink->abm, threshold, val);
106 }
107
__nfp_abm_ctrl_set_q_act(struct nfp_abm * abm,unsigned int id,enum nfp_abm_q_action act)108 int __nfp_abm_ctrl_set_q_act(struct nfp_abm *abm, unsigned int id,
109 enum nfp_abm_q_action act)
110 {
111 struct nfp_cpp *cpp = abm->app->cpp;
112 u64 sym_offset;
113 int err;
114
115 if (abm->actions[id] == act)
116 return 0;
117
118 sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_ACT;
119 err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, act);
120 if (err) {
121 nfp_err(cpp,
122 "RED offload setting action failed on subqueue %d\n",
123 id);
124 return err;
125 }
126
127 abm->actions[id] = act;
128 return 0;
129 }
130
nfp_abm_ctrl_set_q_act(struct nfp_abm_link * alink,unsigned int band,unsigned int queue,enum nfp_abm_q_action act)131 int nfp_abm_ctrl_set_q_act(struct nfp_abm_link *alink, unsigned int band,
132 unsigned int queue, enum nfp_abm_q_action act)
133 {
134 unsigned int qid;
135
136 qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
137
138 return __nfp_abm_ctrl_set_q_act(alink->abm, qid, act);
139 }
140
nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link * alink,unsigned int queue)141 u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int queue)
142 {
143 unsigned int band;
144 u64 val, sum = 0;
145
146 for (band = 0; band < alink->abm->num_bands; band++) {
147 if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
148 NFP_QMSTAT_STRIDE, NFP_QMSTAT_NON_STO,
149 band, queue, true, &val))
150 return 0;
151 sum += val;
152 }
153
154 return sum;
155 }
156
nfp_abm_ctrl_stat_sto(struct nfp_abm_link * alink,unsigned int queue)157 u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int queue)
158 {
159 unsigned int band;
160 u64 val, sum = 0;
161
162 for (band = 0; band < alink->abm->num_bands; band++) {
163 if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
164 NFP_QMSTAT_STRIDE, NFP_QMSTAT_STO,
165 band, queue, true, &val))
166 return 0;
167 sum += val;
168 }
169
170 return sum;
171 }
172
173 static int
nfp_abm_ctrl_stat_basic(struct nfp_abm_link * alink,unsigned int band,unsigned int queue,unsigned int off,u64 * val)174 nfp_abm_ctrl_stat_basic(struct nfp_abm_link *alink, unsigned int band,
175 unsigned int queue, unsigned int off, u64 *val)
176 {
177 if (!nfp_abm_has_prio(alink->abm)) {
178 if (!band) {
179 unsigned int id = alink->queue_base + queue;
180
181 *val = nn_readq(alink->vnic,
182 NFP_NET_CFG_RXR_STATS(id) + off);
183 } else {
184 *val = 0;
185 }
186
187 return 0;
188 } else {
189 return nfp_abm_ctrl_stat(alink, alink->abm->q_stats,
190 NFP_Q_STAT_STRIDE, off, band, queue,
191 true, val);
192 }
193 }
194
nfp_abm_ctrl_read_q_stats(struct nfp_abm_link * alink,unsigned int band,unsigned int queue,struct nfp_alink_stats * stats)195 int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int band,
196 unsigned int queue, struct nfp_alink_stats *stats)
197 {
198 int err;
199
200 err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_PKTS,
201 &stats->tx_pkts);
202 if (err)
203 return err;
204
205 err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_BYTES,
206 &stats->tx_bytes);
207 if (err)
208 return err;
209
210 err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, NFP_QLVL_STRIDE,
211 NFP_QLVL_BLOG_BYTES, band, queue, false,
212 &stats->backlog_bytes);
213 if (err)
214 return err;
215
216 err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls,
217 NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS,
218 band, queue, false, &stats->backlog_pkts);
219 if (err)
220 return err;
221
222 err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
223 NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
224 band, queue, true, &stats->drops);
225 if (err)
226 return err;
227
228 return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
229 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
230 band, queue, true, &stats->overlimits);
231 }
232
nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link * alink,unsigned int band,unsigned int queue,struct nfp_alink_xstats * xstats)233 int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink,
234 unsigned int band, unsigned int queue,
235 struct nfp_alink_xstats *xstats)
236 {
237 int err;
238
239 err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
240 NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
241 band, queue, true, &xstats->pdrop);
242 if (err)
243 return err;
244
245 return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
246 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
247 band, queue, true, &xstats->ecn_marked);
248 }
249
nfp_abm_ctrl_qm_enable(struct nfp_abm * abm)250 int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm)
251 {
252 return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE,
253 NULL, 0, NULL, 0);
254 }
255
nfp_abm_ctrl_qm_disable(struct nfp_abm * abm)256 int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm)
257 {
258 return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_DISABLE,
259 NULL, 0, NULL, 0);
260 }
261
nfp_abm_ctrl_prio_map_update(struct nfp_abm_link * alink,u32 * packed)262 int nfp_abm_ctrl_prio_map_update(struct nfp_abm_link *alink, u32 *packed)
263 {
264 const u32 cmd = NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET;
265 struct nfp_net *nn = alink->vnic;
266 unsigned int i;
267 int err;
268
269 err = nfp_net_mbox_lock(nn, alink->abm->prio_map_len);
270 if (err)
271 return err;
272
273 /* Write data_len and wipe reserved */
274 nn_writeq(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATALEN,
275 alink->abm->prio_map_len);
276
277 for (i = 0; i < alink->abm->prio_map_len; i += sizeof(u32))
278 nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATA + i,
279 packed[i / sizeof(u32)]);
280
281 err = nfp_net_mbox_reconfig_and_unlock(nn, cmd);
282 if (err)
283 nfp_err(alink->abm->app->cpp,
284 "setting DSCP -> VQ map failed with error %d\n", err);
285 return err;
286 }
287
nfp_abm_ctrl_prio_check_params(struct nfp_abm_link * alink)288 static int nfp_abm_ctrl_prio_check_params(struct nfp_abm_link *alink)
289 {
290 struct nfp_abm *abm = alink->abm;
291 struct nfp_net *nn = alink->vnic;
292 unsigned int min_mbox_sz;
293
294 if (!nfp_abm_has_prio(alink->abm))
295 return 0;
296
297 min_mbox_sz = NFP_NET_ABM_MBOX_DATA + alink->abm->prio_map_len;
298 if (nn->tlv_caps.mbox_len < min_mbox_sz) {
299 nfp_err(abm->app->pf->cpp, "vNIC mailbox too small for prio offload: %u, need: %u\n",
300 nn->tlv_caps.mbox_len, min_mbox_sz);
301 return -EINVAL;
302 }
303
304 return 0;
305 }
306
nfp_abm_ctrl_read_params(struct nfp_abm_link * alink)307 int nfp_abm_ctrl_read_params(struct nfp_abm_link *alink)
308 {
309 alink->queue_base = nn_readl(alink->vnic, NFP_NET_CFG_START_RXQ);
310 alink->queue_base /= alink->vnic->stride_rx;
311
312 return nfp_abm_ctrl_prio_check_params(alink);
313 }
314
nfp_abm_ctrl_prio_map_size(struct nfp_abm * abm)315 static unsigned int nfp_abm_ctrl_prio_map_size(struct nfp_abm *abm)
316 {
317 unsigned int size;
318
319 size = roundup_pow_of_two(order_base_2(abm->num_bands));
320 size = DIV_ROUND_UP(size * abm->num_prios, BITS_PER_BYTE);
321 size = round_up(size, sizeof(u32));
322
323 return size;
324 }
325
326 static const struct nfp_rtsym *
nfp_abm_ctrl_find_rtsym(struct nfp_pf * pf,const char * name,unsigned int size)327 nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size)
328 {
329 const struct nfp_rtsym *sym;
330
331 sym = nfp_rtsym_lookup(pf->rtbl, name);
332 if (!sym) {
333 nfp_err(pf->cpp, "Symbol '%s' not found\n", name);
334 return ERR_PTR(-ENOENT);
335 }
336 if (nfp_rtsym_size(sym) != size) {
337 nfp_err(pf->cpp,
338 "Symbol '%s' wrong size: expected %u got %llu\n",
339 name, size, nfp_rtsym_size(sym));
340 return ERR_PTR(-EINVAL);
341 }
342
343 return sym;
344 }
345
346 static const struct nfp_rtsym *
nfp_abm_ctrl_find_q_rtsym(struct nfp_abm * abm,const char * name_fmt,size_t size)347 nfp_abm_ctrl_find_q_rtsym(struct nfp_abm *abm, const char *name_fmt,
348 size_t size)
349 {
350 char pf_symbol[64];
351
352 size = array3_size(size, abm->num_bands, NFP_NET_MAX_RX_RINGS);
353 snprintf(pf_symbol, sizeof(pf_symbol), name_fmt,
354 abm->pf_id, nfp_abm_has_prio(abm) ? "_per_band" : "");
355
356 return nfp_abm_ctrl_find_rtsym(abm->app->pf, pf_symbol, size);
357 }
358
nfp_abm_ctrl_find_addrs(struct nfp_abm * abm)359 int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm)
360 {
361 struct nfp_pf *pf = abm->app->pf;
362 const struct nfp_rtsym *sym;
363 int res;
364
365 abm->pf_id = nfp_cppcore_pcie_unit(pf->cpp);
366
367 /* Check if Qdisc offloads are supported */
368 res = nfp_pf_rtsym_read_optional(pf, NFP_RED_SUPPORT_SYM_NAME, 1);
369 if (res < 0)
370 return res;
371 abm->red_support = res;
372
373 /* Read count of prios and prio bands */
374 res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_BANDS_SYM_NAME, 1);
375 if (res < 0)
376 return res;
377 abm->num_bands = res;
378
379 res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_PRIOS_SYM_NAME, 1);
380 if (res < 0)
381 return res;
382 abm->num_prios = res;
383
384 /* Read available actions */
385 res = nfp_pf_rtsym_read_optional(pf, NFP_ACT_MASK_SYM_NAME,
386 BIT(NFP_ABM_ACT_MARK_DROP));
387 if (res < 0)
388 return res;
389 abm->action_mask = res;
390
391 abm->prio_map_len = nfp_abm_ctrl_prio_map_size(abm);
392 abm->dscp_mask = GENMASK(7, 8 - order_base_2(abm->num_prios));
393
394 /* Check values are sane, U16_MAX is arbitrarily chosen as max */
395 if (!is_power_of_2(abm->num_bands) || !is_power_of_2(abm->num_prios) ||
396 abm->num_bands > U16_MAX || abm->num_prios > U16_MAX ||
397 (abm->num_bands == 1) != (abm->num_prios == 1)) {
398 nfp_err(pf->cpp,
399 "invalid priomap description num bands: %u and num prios: %u\n",
400 abm->num_bands, abm->num_prios);
401 return -EINVAL;
402 }
403
404 /* Find level and stat symbols */
405 if (!abm->red_support)
406 return 0;
407
408 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QLVL_SYM_NAME,
409 NFP_QLVL_STRIDE);
410 if (IS_ERR(sym))
411 return PTR_ERR(sym);
412 abm->q_lvls = sym;
413
414 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QMSTAT_SYM_NAME,
415 NFP_QMSTAT_STRIDE);
416 if (IS_ERR(sym))
417 return PTR_ERR(sym);
418 abm->qm_stats = sym;
419
420 if (nfp_abm_has_prio(abm)) {
421 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_Q_STAT_SYM_NAME,
422 NFP_Q_STAT_STRIDE);
423 if (IS_ERR(sym))
424 return PTR_ERR(sym);
425 abm->q_stats = sym;
426 }
427
428 return 0;
429 }
430