1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
3 #include <net/sch_generic.h>
4
5 #include <net/pkt_cls.h>
6 #include "en.h"
7 #include "params.h"
8 #include "../qos.h"
9 #include "en/htb.h"
10
11 struct qos_sq_callback_params {
12 struct mlx5e_priv *priv;
13 struct mlx5e_channels *chs;
14 };
15
mlx5e_qos_bytes_rate_check(struct mlx5_core_dev * mdev,u64 nbytes)16 int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes)
17 {
18 if (nbytes < BYTES_IN_MBIT) {
19 qos_warn(mdev, "Input rate (%llu Bytes/sec) below minimum supported (%u Bytes/sec)\n",
20 nbytes, BYTES_IN_MBIT);
21 return -EINVAL;
22 }
23 return 0;
24 }
25
mlx5e_qos_bytes2mbits(struct mlx5_core_dev * mdev,u64 nbytes)26 static u32 mlx5e_qos_bytes2mbits(struct mlx5_core_dev *mdev, u64 nbytes)
27 {
28 return div_u64(nbytes, BYTES_IN_MBIT);
29 }
30
mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev * mdev)31 int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev)
32 {
33 return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev));
34 }
35
36 /* TX datapath API */
37
mlx5e_qid_from_qos(struct mlx5e_channels * chs,u16 qid)38 u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid)
39 {
40 /* These channel params are safe to access from the datapath, because:
41 * 1. This function is called only after checking selq->htb_maj_id != 0,
42 * and the number of queues can't change while HTB offload is active.
43 * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for
44 * mlx5e_select_queue to finish while holding priv->state_lock,
45 * preventing other code from changing the number of queues.
46 */
47 bool is_ptp = MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS);
48
49 return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid;
50 }
51
52 /* SQ lifecycle */
53
mlx5e_get_qos_sq(struct mlx5e_priv * priv,int qid)54 static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid)
55 {
56 struct mlx5e_params *params = &priv->channels.params;
57 struct mlx5e_txqsq __rcu **qos_sqs;
58 struct mlx5e_channel *c;
59 int ix;
60
61 ix = qid % params->num_channels;
62 qid /= params->num_channels;
63 c = priv->channels.c[ix];
64
65 qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
66 return mlx5e_state_dereference(priv, qos_sqs[qid]);
67 }
68
mlx5e_open_qos_sq(struct mlx5e_priv * priv,struct mlx5e_channels * chs,u16 node_qid,u32 hw_id)69 int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs,
70 u16 node_qid, u32 hw_id)
71 {
72 struct mlx5e_create_cq_param ccp = {};
73 struct mlx5e_txqsq __rcu **qos_sqs;
74 struct mlx5e_sq_param param_sq;
75 struct mlx5e_cq_param param_cq;
76 int txq_ix, ix, qid, err = 0;
77 struct mlx5e_params *params;
78 struct mlx5e_channel *c;
79 struct mlx5e_txqsq *sq;
80
81 params = &chs->params;
82
83 txq_ix = mlx5e_qid_from_qos(chs, node_qid);
84
85 WARN_ON(node_qid > priv->htb_max_qos_sqs);
86 if (node_qid == priv->htb_max_qos_sqs) {
87 struct mlx5e_sq_stats *stats, **stats_list = NULL;
88
89 if (priv->htb_max_qos_sqs == 0) {
90 stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev),
91 sizeof(*stats_list),
92 GFP_KERNEL);
93 if (!stats_list)
94 return -ENOMEM;
95 }
96 stats = kzalloc(sizeof(*stats), GFP_KERNEL);
97 if (!stats) {
98 kvfree(stats_list);
99 return -ENOMEM;
100 }
101 if (stats_list)
102 WRITE_ONCE(priv->htb_qos_sq_stats, stats_list);
103 WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats);
104 /* Order htb_max_qos_sqs increment after writing the array pointer.
105 * Pairs with smp_load_acquire in en_stats.c.
106 */
107 smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1);
108 }
109
110 ix = node_qid % params->num_channels;
111 qid = node_qid / params->num_channels;
112 c = chs->c[ix];
113
114 qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
115 sq = kzalloc(sizeof(*sq), GFP_KERNEL);
116
117 if (!sq)
118 return -ENOMEM;
119
120 mlx5e_build_create_cq_param(&ccp, c);
121
122 memset(¶m_sq, 0, sizeof(param_sq));
123 memset(¶m_cq, 0, sizeof(param_cq));
124 mlx5e_build_sq_param(priv->mdev, params, ¶m_sq);
125 mlx5e_build_tx_cq_param(priv->mdev, params, ¶m_cq);
126 err = mlx5e_open_cq(priv, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq);
127 if (err)
128 goto err_free_sq;
129 err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params,
130 ¶m_sq, sq, 0, hw_id,
131 priv->htb_qos_sq_stats[node_qid]);
132 if (err)
133 goto err_close_cq;
134
135 rcu_assign_pointer(qos_sqs[qid], sq);
136
137 return 0;
138
139 err_close_cq:
140 mlx5e_close_cq(&sq->cq);
141 err_free_sq:
142 kfree(sq);
143 return err;
144 }
145
mlx5e_open_qos_sq_cb_wrapper(void * data,u16 node_qid,u32 hw_id)146 static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id)
147 {
148 struct qos_sq_callback_params *cb_params = data;
149
150 return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id);
151 }
152
mlx5e_activate_qos_sq(void * data,u16 node_qid,u32 hw_id)153 int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id)
154 {
155 struct mlx5e_priv *priv = data;
156 struct mlx5e_txqsq *sq;
157 u16 qid;
158
159 sq = mlx5e_get_qos_sq(priv, node_qid);
160
161 qid = mlx5e_qid_from_qos(&priv->channels, node_qid);
162
163 /* If it's a new queue, it will be marked as started at this point.
164 * Stop it before updating txq2sq.
165 */
166 mlx5e_tx_disable_queue(netdev_get_tx_queue(priv->netdev, qid));
167
168 priv->txq2sq[qid] = sq;
169
170 /* Make the change to txq2sq visible before the queue is started.
171 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
172 * which pairs with this barrier.
173 */
174 smp_wmb();
175
176 qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid);
177 mlx5e_activate_txqsq(sq);
178
179 return 0;
180 }
181
mlx5e_deactivate_qos_sq(struct mlx5e_priv * priv,u16 qid)182 void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid)
183 {
184 struct mlx5e_txqsq *sq;
185
186 sq = mlx5e_get_qos_sq(priv, qid);
187 if (!sq) /* Handle the case when the SQ failed to open. */
188 return;
189
190 qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid);
191 mlx5e_deactivate_txqsq(sq);
192
193 priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL;
194
195 /* Make the change to txq2sq visible before the queue is started again.
196 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
197 * which pairs with this barrier.
198 */
199 smp_wmb();
200 }
201
mlx5e_close_qos_sq(struct mlx5e_priv * priv,u16 qid)202 void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid)
203 {
204 struct mlx5e_txqsq __rcu **qos_sqs;
205 struct mlx5e_params *params;
206 struct mlx5e_channel *c;
207 struct mlx5e_txqsq *sq;
208 int ix;
209
210 params = &priv->channels.params;
211
212 ix = qid % params->num_channels;
213 qid /= params->num_channels;
214 c = priv->channels.c[ix];
215 qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
216 sq = rcu_replace_pointer(qos_sqs[qid], NULL, lockdep_is_held(&priv->state_lock));
217 if (!sq) /* Handle the case when the SQ failed to open. */
218 return;
219
220 synchronize_rcu(); /* Sync with NAPI. */
221
222 mlx5e_close_txqsq(sq);
223 mlx5e_close_cq(&sq->cq);
224 kfree(sq);
225 }
226
mlx5e_qos_close_queues(struct mlx5e_channel * c)227 void mlx5e_qos_close_queues(struct mlx5e_channel *c)
228 {
229 struct mlx5e_txqsq __rcu **qos_sqs;
230 int i;
231
232 qos_sqs = rcu_replace_pointer(c->qos_sqs, NULL, lockdep_is_held(&c->priv->state_lock));
233 if (!qos_sqs)
234 return;
235 synchronize_rcu(); /* Sync with NAPI. */
236
237 for (i = 0; i < c->qos_sqs_size; i++) {
238 struct mlx5e_txqsq *sq;
239
240 sq = mlx5e_state_dereference(c->priv, qos_sqs[i]);
241 if (!sq) /* Handle the case when the SQ failed to open. */
242 continue;
243
244 mlx5e_close_txqsq(sq);
245 mlx5e_close_cq(&sq->cq);
246 kfree(sq);
247 }
248
249 kvfree(qos_sqs);
250 }
251
mlx5e_qos_close_all_queues(struct mlx5e_channels * chs)252 void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs)
253 {
254 int i;
255
256 for (i = 0; i < chs->num; i++)
257 mlx5e_qos_close_queues(chs->c[i]);
258 }
259
mlx5e_qos_alloc_queues(struct mlx5e_priv * priv,struct mlx5e_channels * chs)260 int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
261 {
262 u16 qos_sqs_size;
263 int i;
264
265 qos_sqs_size = DIV_ROUND_UP(mlx5e_qos_max_leaf_nodes(priv->mdev), chs->num);
266
267 for (i = 0; i < chs->num; i++) {
268 struct mlx5e_txqsq **sqs;
269
270 sqs = kvcalloc(qos_sqs_size, sizeof(struct mlx5e_txqsq *), GFP_KERNEL);
271 if (!sqs)
272 goto err_free;
273
274 WRITE_ONCE(chs->c[i]->qos_sqs_size, qos_sqs_size);
275 smp_wmb(); /* Pairs with mlx5e_napi_poll. */
276 rcu_assign_pointer(chs->c[i]->qos_sqs, sqs);
277 }
278
279 return 0;
280
281 err_free:
282 while (--i >= 0) {
283 struct mlx5e_txqsq **sqs;
284
285 sqs = rcu_replace_pointer(chs->c[i]->qos_sqs, NULL,
286 lockdep_is_held(&priv->state_lock));
287
288 synchronize_rcu(); /* Sync with NAPI. */
289 kvfree(sqs);
290 }
291 return -ENOMEM;
292 }
293
mlx5e_qos_open_queues(struct mlx5e_priv * priv,struct mlx5e_channels * chs)294 int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
295 {
296 struct qos_sq_callback_params callback_params;
297 int err;
298
299 err = mlx5e_qos_alloc_queues(priv, chs);
300 if (err)
301 return err;
302
303 callback_params.priv = priv;
304 callback_params.chs = chs;
305
306 err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params);
307 if (err) {
308 mlx5e_qos_close_all_queues(chs);
309 return err;
310 }
311
312 return 0;
313 }
314
mlx5e_qos_activate_queues(struct mlx5e_priv * priv)315 void mlx5e_qos_activate_queues(struct mlx5e_priv *priv)
316 {
317 mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv);
318 }
319
mlx5e_qos_deactivate_queues(struct mlx5e_channel * c)320 void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c)
321 {
322 struct mlx5e_params *params = &c->priv->channels.params;
323 struct mlx5e_txqsq __rcu **qos_sqs;
324 int i;
325
326 qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs);
327 if (!qos_sqs)
328 return;
329
330 for (i = 0; i < c->qos_sqs_size; i++) {
331 u16 qid = params->num_channels * i + c->ix;
332 struct mlx5e_txqsq *sq;
333
334 sq = mlx5e_state_dereference(c->priv, qos_sqs[i]);
335 if (!sq) /* Handle the case when the SQ failed to open. */
336 continue;
337
338 qos_dbg(c->mdev, "Deactivate QoS SQ qid %u\n", qid);
339 mlx5e_deactivate_txqsq(sq);
340
341 /* The queue is disabled, no synchronization with datapath is needed. */
342 c->priv->txq2sq[mlx5e_qid_from_qos(&c->priv->channels, qid)] = NULL;
343 }
344 }
345
mlx5e_qos_deactivate_all_queues(struct mlx5e_channels * chs)346 void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs)
347 {
348 int i;
349
350 for (i = 0; i < chs->num; i++)
351 mlx5e_qos_deactivate_queues(chs->c[i]);
352 }
353
mlx5e_reactivate_qos_sq(struct mlx5e_priv * priv,u16 qid,struct netdev_queue * txq)354 void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq)
355 {
356 qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid);
357 netdev_tx_reset_queue(txq);
358 netif_tx_start_queue(txq);
359 }
360
mlx5e_reset_qdisc(struct net_device * dev,u16 qid)361 void mlx5e_reset_qdisc(struct net_device *dev, u16 qid)
362 {
363 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid);
364 struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
365
366 if (!qdisc)
367 return;
368
369 spin_lock_bh(qdisc_lock(qdisc));
370 qdisc_reset(qdisc);
371 spin_unlock_bh(qdisc_lock(qdisc));
372 }
373
mlx5e_htb_setup_tc(struct mlx5e_priv * priv,struct tc_htb_qopt_offload * htb_qopt)374 int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt)
375 {
376 struct mlx5e_htb *htb = priv->htb;
377 int res;
378
379 if (!htb && htb_qopt->command != TC_HTB_CREATE)
380 return -EINVAL;
381
382 switch (htb_qopt->command) {
383 case TC_HTB_CREATE:
384 if (!mlx5_qos_is_supported(priv->mdev)) {
385 NL_SET_ERR_MSG_MOD(htb_qopt->extack,
386 "Missing QoS capabilities. Try disabling SRIOV or use a supported device.");
387 return -EOPNOTSUPP;
388 }
389 priv->htb = mlx5e_htb_alloc();
390 htb = priv->htb;
391 if (!htb)
392 return -ENOMEM;
393 res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv);
394 if (res) {
395 mlx5e_htb_free(htb);
396 priv->htb = NULL;
397 }
398 return res;
399 case TC_HTB_DESTROY:
400 mlx5e_htb_cleanup(htb);
401 mlx5e_htb_free(htb);
402 priv->htb = NULL;
403 return 0;
404 case TC_HTB_LEAF_ALLOC_QUEUE:
405 res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid,
406 htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack);
407 if (res < 0)
408 return res;
409 htb_qopt->qid = res;
410 return 0;
411 case TC_HTB_LEAF_TO_INNER:
412 return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid,
413 htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack);
414 case TC_HTB_LEAF_DEL:
415 return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack);
416 case TC_HTB_LEAF_DEL_LAST:
417 case TC_HTB_LEAF_DEL_LAST_FORCE:
418 return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid,
419 htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE,
420 htb_qopt->extack);
421 case TC_HTB_NODE_MODIFY:
422 return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil,
423 htb_qopt->extack);
424 case TC_HTB_LEAF_QUERY_QUEUE:
425 res = mlx5e_htb_get_txq_by_classid(htb, htb_qopt->classid);
426 if (res < 0)
427 return res;
428 htb_qopt->qid = res;
429 return 0;
430 default:
431 return -EOPNOTSUPP;
432 }
433 }
434
435 struct mlx5e_mqprio_rl {
436 struct mlx5_core_dev *mdev;
437 u32 root_id;
438 u32 *leaves_id;
439 u8 num_tc;
440 };
441
mlx5e_mqprio_rl_alloc(void)442 struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_alloc(void)
443 {
444 return kvzalloc(sizeof(struct mlx5e_mqprio_rl), GFP_KERNEL);
445 }
446
mlx5e_mqprio_rl_free(struct mlx5e_mqprio_rl * rl)447 void mlx5e_mqprio_rl_free(struct mlx5e_mqprio_rl *rl)
448 {
449 kvfree(rl);
450 }
451
mlx5e_mqprio_rl_init(struct mlx5e_mqprio_rl * rl,struct mlx5_core_dev * mdev,u8 num_tc,u64 max_rate[])452 int mlx5e_mqprio_rl_init(struct mlx5e_mqprio_rl *rl, struct mlx5_core_dev *mdev, u8 num_tc,
453 u64 max_rate[])
454 {
455 int err;
456 int tc;
457
458 if (!mlx5_qos_is_supported(mdev)) {
459 qos_warn(mdev, "Missing QoS capabilities. Try disabling SRIOV or use a supported device.");
460 return -EOPNOTSUPP;
461 }
462 if (num_tc > mlx5e_qos_max_leaf_nodes(mdev))
463 return -EINVAL;
464
465 rl->mdev = mdev;
466 rl->num_tc = num_tc;
467 rl->leaves_id = kvcalloc(num_tc, sizeof(*rl->leaves_id), GFP_KERNEL);
468 if (!rl->leaves_id)
469 return -ENOMEM;
470
471 err = mlx5_qos_create_root_node(mdev, &rl->root_id);
472 if (err)
473 goto err_free_leaves;
474
475 qos_dbg(mdev, "Root created, id %#x\n", rl->root_id);
476
477 for (tc = 0; tc < num_tc; tc++) {
478 u32 max_average_bw;
479
480 max_average_bw = mlx5e_qos_bytes2mbits(mdev, max_rate[tc]);
481 err = mlx5_qos_create_leaf_node(mdev, rl->root_id, 0, max_average_bw,
482 &rl->leaves_id[tc]);
483 if (err)
484 goto err_destroy_leaves;
485
486 qos_dbg(mdev, "Leaf[%d] created, id %#x, max average bw %u Mbits/sec\n",
487 tc, rl->leaves_id[tc], max_average_bw);
488 }
489 return 0;
490
491 err_destroy_leaves:
492 while (--tc >= 0)
493 mlx5_qos_destroy_node(mdev, rl->leaves_id[tc]);
494 mlx5_qos_destroy_node(mdev, rl->root_id);
495 err_free_leaves:
496 kvfree(rl->leaves_id);
497 return err;
498 }
499
mlx5e_mqprio_rl_cleanup(struct mlx5e_mqprio_rl * rl)500 void mlx5e_mqprio_rl_cleanup(struct mlx5e_mqprio_rl *rl)
501 {
502 int tc;
503
504 for (tc = 0; tc < rl->num_tc; tc++)
505 mlx5_qos_destroy_node(rl->mdev, rl->leaves_id[tc]);
506 mlx5_qos_destroy_node(rl->mdev, rl->root_id);
507 kvfree(rl->leaves_id);
508 }
509
mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl * rl,int tc,u32 * hw_id)510 int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_id)
511 {
512 if (tc >= rl->num_tc)
513 return -EINVAL;
514
515 *hw_id = rl->leaves_id[tc];
516 return 0;
517 }
518
519